From bf8c622bcd1cef35c19dd91445bb8d212a2cb633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89mile=20R=C3=A9?= Date: Mon, 11 May 2026 14:23:19 +0400 Subject: [PATCH] Extract webinspect package for logo discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the Google Favicons API with HTML head tag parsing to find higher-quality logos (SVG, apple-touch-icon, large PNG icons, msapplication-TileImage). The new pkg/webinspect package parses a website's DOM tree and is extensible for future resource extraction (footer links, etc.). Signed-off-by: Émile Ré --- cmd/common-third-parties-import/main.go | 29 ++-- pkg/webinspect/logo.go | 172 ++++++++++++++++++++++++ pkg/webinspect/logo_test.go | 163 ++++++++++++++++++++++ pkg/webinspect/parse.go | 110 +++++++++++++++ 4 files changed, 463 insertions(+), 11 deletions(-) create mode 100644 pkg/webinspect/logo.go create mode 100644 pkg/webinspect/logo_test.go create mode 100644 pkg/webinspect/parse.go diff --git a/cmd/common-third-parties-import/main.go b/cmd/common-third-parties-import/main.go index 11c515ce4..7d94f9d2b 100644 --- a/cmd/common-third-parties-import/main.go +++ b/cmd/common-third-parties-import/main.go @@ -16,9 +16,10 @@ // packages/vendors/data.json. It is idempotent: re-running upserts on conflict // (lower(name)) so existing rows keep their id and created_at. // -// When -fetch-logos is set, the tool also fetches favicons from Google's -// favicon service and stores them in S3 as public files, linking them to each -// common third party via logo_file_id. +// When -fetch-logos is set, the tool inspects each third party's website to +// find the best available logo (SVG icon, apple-touch-icon, etc.) and stores +// it in S3 as a public file, linking it to each common third party via +// logo_file_id. package main import ( @@ -42,6 +43,7 @@ import ( "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/filemanager" "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/webinspect" ) type thirdPartyData struct { @@ -254,18 +256,23 @@ func fetchAndStoreLogos( continue } - parsedURL, err := url.Parse(*tp.WebsiteURL) + pageInfo, err := webinspect.Parse(ctx, httpClient, *tp.WebsiteURL) if err != nil { - fmt.Fprintf(os.Stderr, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err) + fmt.Fprintf(os.Stderr, "warning: cannot inspect page for %q, skipping logo: %v\n", tp.Name, err) failed++ continue } - faviconURL := fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=64", parsedURL.Hostname()) - - resp, err := httpClient.Get(faviconURL) + logoURL, err := webinspect.FindLogoURL(pageInfo) if err != nil { - fmt.Fprintf(os.Stderr, "warning: cannot fetch favicon for %q: %v\n", tp.Name, err) + fmt.Fprintf(os.Stderr, "warning: cannot find logo for %q: %v\n", tp.Name, err) + failed++ + continue + } + + resp, err := httpClient.Get(logoURL) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: cannot fetch logo for %q: %v\n", tp.Name, err) failed++ continue } @@ -274,7 +281,7 @@ func fetchAndStoreLogos( _ = resp.Body.Close() if err != nil || resp.StatusCode != http.StatusOK || len(body) == 0 { - fmt.Fprintf(os.Stderr, "warning: bad favicon response for %q (status %d)\n", tp.Name, resp.StatusCode) + fmt.Fprintf(os.Stderr, "warning: bad logo response for %q (status %d)\n", tp.Name, resp.StatusCode) failed++ continue } @@ -297,7 +304,7 @@ func fetchAndStoreLogos( OrganizationID: gid.Nil, BucketName: bucket, MimeType: contentType, - FileName: parsedURL.Hostname() + "-favicon.png", + FileName: tp.Name + "-logo" + webinspect.ExtensionForMIME(contentType), FileKey: objectKey.String(), FileSize: int64(len(body)), Visibility: coredata.FileVisibilityPublic, diff --git a/pkg/webinspect/logo.go b/pkg/webinspect/logo.go new file mode 100644 index 000000000..193432ba0 --- /dev/null +++ b/pkg/webinspect/logo.go @@ -0,0 +1,172 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package webinspect + +import ( + "fmt" + "strconv" + "strings" + + "golang.org/x/net/html" +) + +func FindLogoURL(info *PageInfo) (string, error) { + head := findElement(info.Root, "head") + if head == nil { + return "", fmt.Errorf("cannot find logo: no head element") + } + + var ( + svgIcon string + appleTouchIcon string + appleTouchSize int + largestIcon string + largestSize int + msTileImage string + ) + + for _, n := range findAllIn(head, "link") { + rel := strings.ToLower(attrVal(n, "rel")) + href := attrVal(n, "href") + if href == "" { + continue + } + + switch { + case rel == "icon" && attrVal(n, "type") == "image/svg+xml": + svgIcon = href + case strings.Contains(rel, "apple-touch-icon"): + size := parseSizeAttr(attrVal(n, "sizes")) + if appleTouchIcon == "" || size > appleTouchSize { + appleTouchIcon = href + appleTouchSize = size + } + case rel == "icon": + size := parseSizeAttr(attrVal(n, "sizes")) + if largestIcon == "" || size > largestSize { + largestIcon = href + largestSize = size + } + } + } + + for _, n := range findAllIn(head, "meta") { + name := strings.ToLower(attrVal(n, "name")) + content := attrVal(n, "content") + if name == "msapplication-tileimage" && content != "" { + msTileImage = content + } + } + + candidates := []string{ + svgIcon, + appleTouchIcon, + largestIcon, + msTileImage, + } + + for _, href := range candidates { + if href != "" { + return info.ResolveHref(href), nil + } + } + + return "", fmt.Errorf("cannot find logo") +} + +func parseSizeAttr(sizes string) int { + if sizes == "" || strings.ToLower(sizes) == "any" { + return 0 + } + + parts := strings.SplitN(sizes, "x", 2) + if len(parts) == 0 { + return 0 + } + + w, err := strconv.Atoi(parts[0]) + if err != nil { + return 0 + } + + return w +} + +func ExtensionForMIME(contentType string) string { + ct := strings.ToLower(contentType) + if idx := strings.Index(ct, ";"); idx != -1 { + ct = ct[:idx] + } + ct = strings.TrimSpace(ct) + + switch ct { + case "image/svg+xml": + return ".svg" + case "image/png": + return ".png" + case "image/jpeg": + return ".jpg" + case "image/gif": + return ".gif" + case "image/webp": + return ".webp" + case "image/x-icon", "image/vnd.microsoft.icon": + return ".ico" + default: + return ".png" + } +} + +// HeadLinks returns all nodes from whose rel attribute +// contains the given value (case-insensitive partial match). +func (p *PageInfo) HeadLinks(rel string) []*html.Node { + head := findElement(p.Root, "head") + if head == nil { + return nil + } + + rel = strings.ToLower(rel) + var matches []*html.Node + + for _, n := range findAllIn(head, "link") { + if strings.Contains(strings.ToLower(attrVal(n, "rel")), rel) { + matches = append(matches, n) + } + } + + return matches +} + +// HeadMeta returns the content attribute of the first tag in +// whose name attribute matches (case-insensitive). +func (p *PageInfo) HeadMeta(name string) (string, bool) { + head := findElement(p.Root, "head") + if head == nil { + return "", false + } + + name = strings.ToLower(name) + + for _, n := range findAllIn(head, "meta") { + if strings.EqualFold(attrVal(n, "name"), name) { + content := attrVal(n, "content") + if content != "" { + return content, true + } + } + } + + return "", false +} diff --git a/pkg/webinspect/logo_test.go b/pkg/webinspect/logo_test.go new file mode 100644 index 000000000..77b06d8b3 --- /dev/null +++ b/pkg/webinspect/logo_test.go @@ -0,0 +1,163 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package webinspect_test + +import ( + "net/url" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/webinspect" +) + +func parseTestHTML(t *testing.T, rawURL string, body string) *webinspect.PageInfo { + t.Helper() + + u, err := url.Parse(rawURL) + require.NoError(t, err) + + info, err := webinspect.ParseHTML(u, strings.NewReader(body)) + require.NoError(t, err) + + return info +} + +func TestFindLogoURL_SVGPreferred(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://example.com", ` + + + + `) + + got, err := webinspect.FindLogoURL(info) + require.NoError(t, err) + assert.Equal(t, "https://example.com/favicon.svg", got) +} + +func TestFindLogoURL_AppleTouchIconSecond(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://example.com", ` + + + `) + + got, err := webinspect.FindLogoURL(info) + require.NoError(t, err) + assert.Equal(t, "https://example.com/apple-touch-icon.png", got) +} + +func TestFindLogoURL_AppleTouchIconLargest(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://example.com", ` + + + + `) + + got, err := webinspect.FindLogoURL(info) + require.NoError(t, err) + assert.Equal(t, "https://example.com/touch-180.png", got) +} + +func TestFindLogoURL_LargestIcon(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://example.com", ` + + + + `) + + got, err := webinspect.FindLogoURL(info) + require.NoError(t, err) + assert.Equal(t, "https://example.com/icon-192.png", got) +} + +func TestFindLogoURL_MsTileImage(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://example.com", ` + + `) + + got, err := webinspect.FindLogoURL(info) + require.NoError(t, err) + assert.Equal(t, "https://example.com/mstile-144.png", got) +} + +func TestFindLogoURL_RelativeHrefResolved(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://cdn.example.com/app/", ` + + `) + + got, err := webinspect.FindLogoURL(info) + require.NoError(t, err) + assert.Equal(t, "https://cdn.example.com/assets/logo.svg", got) +} + +func TestFindLogoURL_NoHead(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://example.com", `

no head

`) + + _, err := webinspect.FindLogoURL(info) + assert.Error(t, err) +} + +func TestFindLogoURL_NothingFound(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://example.com", ``) + + _, err := webinspect.FindLogoURL(info) + assert.Error(t, err) +} + +func TestExtensionForMIME(t *testing.T) { + t.Parallel() + + tests := []struct { + contentType string + expected string + }{ + {"image/svg+xml", ".svg"}, + {"image/png", ".png"}, + {"image/jpeg", ".jpg"}, + {"image/gif", ".gif"}, + {"image/webp", ".webp"}, + {"image/x-icon", ".ico"}, + {"image/vnd.microsoft.icon", ".ico"}, + {"image/png; charset=utf-8", ".png"}, + {"application/octet-stream", ".png"}, + } + + for _, tt := range tests { + t.Run( + tt.contentType, + func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.expected, webinspect.ExtensionForMIME(tt.contentType)) + }, + ) + } +} diff --git a/pkg/webinspect/parse.go b/pkg/webinspect/parse.go new file mode 100644 index 000000000..6fb46dbe9 --- /dev/null +++ b/pkg/webinspect/parse.go @@ -0,0 +1,110 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package webinspect + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + + "golang.org/x/net/html" +) + +type PageInfo struct { + URL *url.URL + Root *html.Node +} + +func Parse(ctx context.Context, client *http.Client, websiteURL string) (*PageInfo, error) { + parsed, err := url.Parse(websiteURL) + if err != nil { + return nil, fmt.Errorf("cannot parse website URL: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, websiteURL, nil) + if err != nil { + return nil, fmt.Errorf("cannot create request: %w", err) + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("cannot fetch page: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("cannot fetch page: status %d", resp.StatusCode) + } + + return ParseHTML(parsed, resp.Body) +} + +func ParseHTML(baseURL *url.URL, r io.Reader) (*PageInfo, error) { + root, err := html.Parse(r) + if err != nil { + return nil, fmt.Errorf("cannot parse HTML: %w", err) + } + + return &PageInfo{URL: baseURL, Root: root}, nil +} + +func (p *PageInfo) ResolveHref(href string) string { + ref, err := url.Parse(href) + if err != nil { + return href + } + + return p.URL.ResolveReference(ref).String() +} + +func findElement(n *html.Node, tag string) *html.Node { + if n.Type == html.ElementNode && n.Data == tag { + return n + } + + for c := n.FirstChild; c != nil; c = c.NextSibling { + if found := findElement(c, tag); found != nil { + return found + } + } + + return nil +} + +func findAllIn(parent *html.Node, tag string) []*html.Node { + var nodes []*html.Node + + for c := parent.FirstChild; c != nil; c = c.NextSibling { + if c.Type == html.ElementNode && c.Data == tag { + nodes = append(nodes, c) + } + + nodes = append(nodes, findAllIn(c, tag)...) + } + + return nodes +} + +func attrVal(n *html.Node, key string) string { + for _, a := range n.Attr { + if a.Key == key { + return a.Val + } + } + + return "" +}