Extract webinspect package for logo discovery

Replace the Google Favicons API with HTML head tag parsing to find higher-quality logos (SVG, apple-touch-icon, large PNG icons, msapplication-TileImage). The new pkg/webinspect package parses a website's DOM tree and is extensible for future resource extraction (footer links, etc.). Signed-off-by: Émile Ré <emile@getprobo.com>
2026-05-11 14:23:19 +04:00
parent 684fadea3e
commit bf8c622bcd
4 changed files with 463 additions and 11 deletions
--- a/cmd/common-third-parties-import/main.go
+++ b/cmd/common-third-parties-import/main.go
@@ -16,9 +16,10 @@
 // packages/vendors/data.json. It is idempotent: re-running upserts on conflict
 // (lower(name)) so existing rows keep their id and created_at.
 //
-// When -fetch-logos is set, the tool also fetches favicons from Google's
-// favicon service and stores them in S3 as public files, linking them to each
-// common third party via logo_file_id.
+// When -fetch-logos is set, the tool inspects each third party's website to
+// find the best available logo (SVG icon, apple-touch-icon, etc.) and stores
+// it in S3 as a public file, linking it to each common third party via
+// logo_file_id.
 package main

 import (
@@ -42,6 +43,7 @@ import (
 	"go.probo.inc/probo/pkg/coredata"
 	"go.probo.inc/probo/pkg/filemanager"
 	"go.probo.inc/probo/pkg/gid"
+	"go.probo.inc/probo/pkg/webinspect"
 )

 type thirdPartyData struct {
@@ -254,18 +256,23 @@ func fetchAndStoreLogos(
 			continue
 		}

-		parsedURL, err := url.Parse(*tp.WebsiteURL)
+		pageInfo, err := webinspect.Parse(ctx, httpClient, *tp.WebsiteURL)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err)
+			fmt.Fprintf(os.Stderr, "warning: cannot inspect page for %q, skipping logo: %v\n", tp.Name, err)
 			failed++
 			continue
 		}

-		faviconURL := fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=64", parsedURL.Hostname())
-
-		resp, err := httpClient.Get(faviconURL)
+		logoURL, err := webinspect.FindLogoURL(pageInfo)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "warning: cannot fetch favicon for %q: %v\n", tp.Name, err)
+			fmt.Fprintf(os.Stderr, "warning: cannot find logo for %q: %v\n", tp.Name, err)
+			failed++
+			continue
+		}
+
+		resp, err := httpClient.Get(logoURL)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "warning: cannot fetch logo for %q: %v\n", tp.Name, err)
 			failed++
 			continue
 		}
@@ -274,7 +281,7 @@ func fetchAndStoreLogos(
 		_ = resp.Body.Close()

 		if err != nil || resp.StatusCode != http.StatusOK || len(body) == 0 {
-			fmt.Fprintf(os.Stderr, "warning: bad favicon response for %q (status %d)\n", tp.Name, resp.StatusCode)
+			fmt.Fprintf(os.Stderr, "warning: bad logo response for %q (status %d)\n", tp.Name, resp.StatusCode)
 			failed++
 			continue
 		}
@@ -297,7 +304,7 @@ func fetchAndStoreLogos(
 			OrganizationID: gid.Nil,
 			BucketName:     bucket,
 			MimeType:       contentType,
-			FileName:       parsedURL.Hostname() + "-favicon.png",
+			FileName:       tp.Name + "-logo" + webinspect.ExtensionForMIME(contentType),
 			FileKey:        objectKey.String(),
 			FileSize:       int64(len(body)),
 			Visibility:     coredata.FileVisibilityPublic,
--- a/pkg/webinspect/logo.go
+++ b/pkg/webinspect/logo.go
@@ -0,0 +1,172 @@
+// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+package webinspect
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+func FindLogoURL(info *PageInfo) (string, error) {
+	head := findElement(info.Root, "head")
+	if head == nil {
+		return "", fmt.Errorf("cannot find logo: no head element")
+	}
+
+	var (
+		svgIcon        string
+		appleTouchIcon string
+		appleTouchSize int
+		largestIcon    string
+		largestSize    int
+		msTileImage    string
+	)
+
+	for _, n := range findAllIn(head, "link") {
+		rel := strings.ToLower(attrVal(n, "rel"))
+		href := attrVal(n, "href")
+		if href == "" {
+			continue
+		}
+
+		switch {
+		case rel == "icon" && attrVal(n, "type") == "image/svg+xml":
+			svgIcon = href
+		case strings.Contains(rel, "apple-touch-icon"):
+			size := parseSizeAttr(attrVal(n, "sizes"))
+			if appleTouchIcon == "" || size > appleTouchSize {
+				appleTouchIcon = href
+				appleTouchSize = size
+			}
+		case rel == "icon":
+			size := parseSizeAttr(attrVal(n, "sizes"))
+			if largestIcon == "" || size > largestSize {
+				largestIcon = href
+				largestSize = size
+			}
+		}
+	}
+
+	for _, n := range findAllIn(head, "meta") {
+		name := strings.ToLower(attrVal(n, "name"))
+		content := attrVal(n, "content")
+		if name == "msapplication-tileimage" && content != "" {
+			msTileImage = content
+		}
+	}
+
+	candidates := []string{
+		svgIcon,
+		appleTouchIcon,
+		largestIcon,
+		msTileImage,
+	}
+
+	for _, href := range candidates {
+		if href != "" {
+			return info.ResolveHref(href), nil
+		}
+	}
+
+	return "", fmt.Errorf("cannot find logo")
+}
+
+func parseSizeAttr(sizes string) int {
+	if sizes == "" || strings.ToLower(sizes) == "any" {
+		return 0
+	}
+
+	parts := strings.SplitN(sizes, "x", 2)
+	if len(parts) == 0 {
+		return 0
+	}
+
+	w, err := strconv.Atoi(parts[0])
+	if err != nil {
+		return 0
+	}
+
+	return w
+}
+
+func ExtensionForMIME(contentType string) string {
+	ct := strings.ToLower(contentType)
+	if idx := strings.Index(ct, ";"); idx != -1 {
+		ct = ct[:idx]
+	}
+	ct = strings.TrimSpace(ct)
+
+	switch ct {
+	case "image/svg+xml":
+		return ".svg"
+	case "image/png":
+		return ".png"
+	case "image/jpeg":
+		return ".jpg"
+	case "image/gif":
+		return ".gif"
+	case "image/webp":
+		return ".webp"
+	case "image/x-icon", "image/vnd.microsoft.icon":
+		return ".ico"
+	default:
+		return ".png"
+	}
+}
+
+// HeadLinks returns all <link> nodes from <head> whose rel attribute
+// contains the given value (case-insensitive partial match).
+func (p *PageInfo) HeadLinks(rel string) []*html.Node {
+	head := findElement(p.Root, "head")
+	if head == nil {
+		return nil
+	}
+
+	rel = strings.ToLower(rel)
+	var matches []*html.Node
+
+	for _, n := range findAllIn(head, "link") {
+		if strings.Contains(strings.ToLower(attrVal(n, "rel")), rel) {
+			matches = append(matches, n)
+		}
+	}
+
+	return matches
+}
+
+// HeadMeta returns the content attribute of the first <meta> tag in <head>
+// whose name attribute matches (case-insensitive).
+func (p *PageInfo) HeadMeta(name string) (string, bool) {
+	head := findElement(p.Root, "head")
+	if head == nil {
+		return "", false
+	}
+
+	name = strings.ToLower(name)
+
+	for _, n := range findAllIn(head, "meta") {
+		if strings.EqualFold(attrVal(n, "name"), name) {
+			content := attrVal(n, "content")
+			if content != "" {
+				return content, true
+			}
+		}
+	}
+
+	return "", false
+}
--- a/pkg/webinspect/logo_test.go
+++ b/pkg/webinspect/logo_test.go
@@ -0,0 +1,163 @@
+// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+package webinspect_test
+
+import (
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.probo.inc/probo/pkg/webinspect"
+)
+
+func parseTestHTML(t *testing.T, rawURL string, body string) *webinspect.PageInfo {
+	t.Helper()
+
+	u, err := url.Parse(rawURL)
+	require.NoError(t, err)
+
+	info, err := webinspect.ParseHTML(u, strings.NewReader(body))
+	require.NoError(t, err)
+
+	return info
+}
+
+func TestFindLogoURL_SVGPreferred(t *testing.T) {
+	t.Parallel()
+
+	info := parseTestHTML(t, "https://example.com", `<html><head>
+		<link rel="icon" type="image/svg+xml" href="/favicon.svg">
+		<link rel="apple-touch-icon" href="/apple-touch-icon.png">
+		<link rel="icon" sizes="192x192" href="/icon-192.png">
+	</head><body></body></html>`)
+
+	got, err := webinspect.FindLogoURL(info)
+	require.NoError(t, err)
+	assert.Equal(t, "https://example.com/favicon.svg", got)
+}
+
+func TestFindLogoURL_AppleTouchIconSecond(t *testing.T) {
+	t.Parallel()
+
+	info := parseTestHTML(t, "https://example.com", `<html><head>
+		<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
+		<link rel="icon" sizes="32x32" href="/favicon-32.png">
+	</head><body></body></html>`)
+
+	got, err := webinspect.FindLogoURL(info)
+	require.NoError(t, err)
+	assert.Equal(t, "https://example.com/apple-touch-icon.png", got)
+}
+
+func TestFindLogoURL_AppleTouchIconLargest(t *testing.T) {
+	t.Parallel()
+
+	info := parseTestHTML(t, "https://example.com", `<html><head>
+		<link rel="apple-touch-icon" sizes="120x120" href="/touch-120.png">
+		<link rel="apple-touch-icon" sizes="180x180" href="/touch-180.png">
+		<link rel="apple-touch-icon" sizes="152x152" href="/touch-152.png">
+	</head><body></body></html>`)
+
+	got, err := webinspect.FindLogoURL(info)
+	require.NoError(t, err)
+	assert.Equal(t, "https://example.com/touch-180.png", got)
+}
+
+func TestFindLogoURL_LargestIcon(t *testing.T) {
+	t.Parallel()
+
+	info := parseTestHTML(t, "https://example.com", `<html><head>
+		<link rel="icon" sizes="32x32" href="/icon-32.png">
+		<link rel="icon" sizes="192x192" href="/icon-192.png">
+		<link rel="icon" sizes="16x16" href="/icon-16.png">
+	</head><body></body></html>`)
+
+	got, err := webinspect.FindLogoURL(info)
+	require.NoError(t, err)
+	assert.Equal(t, "https://example.com/icon-192.png", got)
+}
+
+func TestFindLogoURL_MsTileImage(t *testing.T) {
+	t.Parallel()
+
+	info := parseTestHTML(t, "https://example.com", `<html><head>
+		<meta name="msapplication-TileImage" content="/mstile-144.png">
+	</head><body></body></html>`)
+
+	got, err := webinspect.FindLogoURL(info)
+	require.NoError(t, err)
+	assert.Equal(t, "https://example.com/mstile-144.png", got)
+}
+
+func TestFindLogoURL_RelativeHrefResolved(t *testing.T) {
+	t.Parallel()
+
+	info := parseTestHTML(t, "https://cdn.example.com/app/", `<html><head>
+		<link rel="icon" type="image/svg+xml" href="../assets/logo.svg">
+	</head><body></body></html>`)
+
+	got, err := webinspect.FindLogoURL(info)
+	require.NoError(t, err)
+	assert.Equal(t, "https://cdn.example.com/assets/logo.svg", got)
+}
+
+func TestFindLogoURL_NoHead(t *testing.T) {
+	t.Parallel()
+
+	info := parseTestHTML(t, "https://example.com", `<html><body><p>no head</p></body></html>`)
+
+	_, err := webinspect.FindLogoURL(info)
+	assert.Error(t, err)
+}
+
+func TestFindLogoURL_NothingFound(t *testing.T) {
+	t.Parallel()
+
+	info := parseTestHTML(t, "https://example.com", `<html><head></head><body></body></html>`)
+
+	_, err := webinspect.FindLogoURL(info)
+	assert.Error(t, err)
+}
+
+func TestExtensionForMIME(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		contentType string
+		expected    string
+	}{
+		{"image/svg+xml", ".svg"},
+		{"image/png", ".png"},
+		{"image/jpeg", ".jpg"},
+		{"image/gif", ".gif"},
+		{"image/webp", ".webp"},
+		{"image/x-icon", ".ico"},
+		{"image/vnd.microsoft.icon", ".ico"},
+		{"image/png; charset=utf-8", ".png"},
+		{"application/octet-stream", ".png"},
+	}
+
+	for _, tt := range tests {
+		t.Run(
+			tt.contentType,
+			func(t *testing.T) {
+				t.Parallel()
+				assert.Equal(t, tt.expected, webinspect.ExtensionForMIME(tt.contentType))
+			},
+		)
+	}
+}
--- a/pkg/webinspect/parse.go
+++ b/pkg/webinspect/parse.go
@@ -0,0 +1,110 @@
+// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+package webinspect
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
+	"golang.org/x/net/html"
+)
+
+type PageInfo struct {
+	URL  *url.URL
+	Root *html.Node
+}
+
+func Parse(ctx context.Context, client *http.Client, websiteURL string) (*PageInfo, error) {
+	parsed, err := url.Parse(websiteURL)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse website URL: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, websiteURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create request: %w", err)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cannot fetch page: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("cannot fetch page: status %d", resp.StatusCode)
+	}
+
+	return ParseHTML(parsed, resp.Body)
+}
+
+func ParseHTML(baseURL *url.URL, r io.Reader) (*PageInfo, error) {
+	root, err := html.Parse(r)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse HTML: %w", err)
+	}
+
+	return &PageInfo{URL: baseURL, Root: root}, nil
+}
+
+func (p *PageInfo) ResolveHref(href string) string {
+	ref, err := url.Parse(href)
+	if err != nil {
+		return href
+	}
+
+	return p.URL.ResolveReference(ref).String()
+}
+
+func findElement(n *html.Node, tag string) *html.Node {
+	if n.Type == html.ElementNode && n.Data == tag {
+		return n
+	}
+
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		if found := findElement(c, tag); found != nil {
+			return found
+		}
+	}
+
+	return nil
+}
+
+func findAllIn(parent *html.Node, tag string) []*html.Node {
+	var nodes []*html.Node
+
+	for c := parent.FirstChild; c != nil; c = c.NextSibling {
+		if c.Type == html.ElementNode && c.Data == tag {
+			nodes = append(nodes, c)
+		}
+
+		nodes = append(nodes, findAllIn(c, tag)...)
+	}
+
+	return nodes
+}
+
+func attrVal(n *html.Node, key string) string {
+	for _, a := range n.Attr {
+		if a.Key == key {
+			return a.Val
+		}
+	}
+
+	return ""
+}