From a3f60968cf9060011c72b9d73c316f81e453439d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89mile=20R=C3=A9?= Date: Thu, 14 May 2026 18:42:42 +0400 Subject: [PATCH] Remove logo fetching logic and unused webinspect package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Émile Ré --- .../common_third_parties.go | 235 +----------------- pkg/webinspect/logo.go | 180 -------------- pkg/webinspect/logo_test.go | 163 ------------ pkg/webinspect/parse.go | 111 --------- 4 files changed, 1 insertion(+), 688 deletions(-) delete mode 100644 pkg/webinspect/logo.go delete mode 100644 pkg/webinspect/logo_test.go delete mode 100644 pkg/webinspect/parse.go diff --git a/pkg/proboctl/seed/common-third-parties/common_third_parties.go b/pkg/proboctl/seed/common-third-parties/common_third_parties.go index f3ef88d49..9aeceef75 100644 --- a/pkg/proboctl/seed/common-third-parties/common_third_parties.go +++ b/pkg/proboctl/seed/common-third-parties/common_third_parties.go @@ -15,30 +15,19 @@ package commonthirdparties import ( - "bytes" "context" "encoding/json" "fmt" "io" - "net/http" - "net/url" "os" "time" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/spf13/cobra" - "go.gearno.de/crypto/uuid" - "go.gearno.de/kit/httpclient" "go.gearno.de/kit/pg" "go.probo.inc/probo/pkg/coredata" - "go.probo.inc/probo/pkg/filemanager" "go.probo.inc/probo/pkg/gid" "go.probo.inc/probo/pkg/proboctl/cmdutil" "go.probo.inc/probo/pkg/slug" - "go.probo.inc/probo/pkg/version" - "go.probo.inc/probo/pkg/webinspect" ) type thirdPartyData struct { @@ -62,16 +51,7 @@ type thirdPartyData struct { } func NewCmdCommonThirdParties(f *cmdutil.Factory) *cobra.Command { - var ( - flagData string - flagFetchLogos bool - flagS3Bucket string - flagS3Endpoint string - flagS3Region string - flagS3AccessKey string - flagS3SecretKey string - flagS3UsePathStyle bool - ) + var flagData string cmd := &cobra.Command{ Use: "common-third-parties", @@ -82,11 +62,6 @@ func NewCmdCommonThirdParties(f *cmdutil.Factory) *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { out := f.IOStreams.Out errOut := f.IOStreams.ErrOut - - if flagFetchLogos && flagS3Bucket == "" { - return fmt.Errorf("set --s3-bucket or AWS_S3_BUCKET when using --fetch-logos") - } - ctx := cmd.Context() thirdParties, err := loadThirdParties(flagData) @@ -177,211 +152,16 @@ func NewCmdCommonThirdParties(f *cmdutil.Factory) *cobra.Command { _, _ = fmt.Fprintf(out, "seeded %d third parties (%d inserted, %d updated)\n", len(thirdParties), inserted, updated) _, _ = fmt.Fprintf(out, "seeded %d domains (%d inserted, %d updated)\n", domainsInserted+domainsUpdated, domainsInserted, domainsUpdated) - if flagFetchLogos { - if err := fetchAndStoreLogos( - ctx, out, errOut, pgClient, thirdParties, - flagS3Bucket, flagS3Endpoint, flagS3Region, flagS3AccessKey, flagS3SecretKey, flagS3UsePathStyle, - ); err != nil { - return fmt.Errorf("cannot fetch logos: %w", err) - } - } - return nil }, } cmd.Flags().StringVar(&flagData, "data", "", "Path to the third-party data.json file") _ = cmd.MarkFlagRequired("data") - cmd.Flags().BoolVar(&flagFetchLogos, "fetch-logos", false, "Fetch favicons and store them in S3") - cmd.Flags().StringVar(&flagS3Bucket, "s3-bucket", os.Getenv("AWS_S3_BUCKET"), "S3 bucket name (default: AWS_S3_BUCKET env)") - cmd.Flags().StringVar(&flagS3Endpoint, "s3-endpoint", os.Getenv("AWS_ENDPOINT_URL"), "S3 endpoint URL (default: AWS_ENDPOINT_URL env)") - cmd.Flags().StringVar(&flagS3Region, "s3-region", os.Getenv("AWS_REGION"), "S3 region (default: AWS_REGION env)") - cmd.Flags().StringVar(&flagS3AccessKey, "s3-access-key", os.Getenv("AWS_ACCESS_KEY_ID"), "S3 access key ID (default: AWS_ACCESS_KEY_ID env)") - cmd.Flags().StringVar(&flagS3SecretKey, "s3-secret-key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "S3 secret access key (default: AWS_SECRET_ACCESS_KEY env)") - cmd.Flags().BoolVar(&flagS3UsePathStyle, "s3-path-style", false, "Use S3 path-style addressing") return cmd } -func fetchAndStoreLogos( - ctx context.Context, - out, errOut io.Writer, - pgClient *pg.Client, - thirdParties []thirdPartyData, - bucket, endpoint, region, accessKey, secretKey string, - usePathStyle bool, -) error { - s3Client := newS3Client(endpoint, region, accessKey, secretKey, usePathStyle) - fileMgr := filemanager.NewService(s3Client) - httpClient := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection()) - httpClient.Transport = &userAgentTransport{ - next: httpClient.Transport, - ua: version.UserAgent("proboctl"), - } - scope := coredata.NewScope(gid.NilTenant) - - var fetched, skipped, failed int - - for _, tp := range thirdParties { - if tp.WebsiteURL == nil || *tp.WebsiteURL == "" { - skipped++ - continue - } - - var party coredata.CommonThirdParty - if err := pgClient.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { - return party.LoadByName(ctx, conn, tp.Name) - }); err != nil { - _, _ = fmt.Fprintf(errOut, "warning: cannot load %q, skipping logo: %v\n", tp.Name, err) - failed++ - continue - } - - if party.LogoFileID != nil { - skipped++ - continue - } - - var logoURL string - pageInfo, err := webinspect.Parse(ctx, httpClient, *tp.WebsiteURL) - if err != nil { - _, _ = fmt.Fprintf(errOut, "warning: cannot inspect page for %q, trying default apple-touch-icon: %v\n", tp.Name, err) - } else { - logoURL, err = webinspect.FindLogoURL(pageInfo) - if err != nil { - _, _ = fmt.Fprintf(errOut, "warning: cannot find logo for %q, trying default apple-touch-icon: %v\n", tp.Name, err) - } - } - - parsed, err := url.Parse(*tp.WebsiteURL) - if err != nil { - _, _ = fmt.Fprintf(errOut, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err) - failed++ - continue - } - - var candidateURLs []string - if logoURL != "" { - candidateURLs = append(candidateURLs, logoURL) - } - base := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host) - candidateURLs = append( - candidateURLs, - base+"/apple-touch-icon.png", - base+"/apple-touch-icon-precomposed.png", - "https://logo.debounce.com/"+parsed.Host, - ) - - var ( - body []byte - contentType string - ) - for _, candidate := range candidateURLs { - resp, err := httpClient.Get(candidate) - if err != nil { - continue - } - - b, err := io.ReadAll(resp.Body) - _ = resp.Body.Close() - - if err != nil || resp.StatusCode != http.StatusOK || len(b) == 0 { - continue - } - - body = b - contentType = resp.Header.Get("Content-Type") - break - } - - if len(body) == 0 { - _, _ = fmt.Fprintf(errOut, "warning: cannot fetch logo for %q from any candidate URL\n", tp.Name) - failed++ - continue - } - - if contentType == "" { - contentType = "image/png" - } - - objectKey, err := uuid.NewV7() - if err != nil { - return fmt.Errorf("cannot generate object key: %w", err) - } - - now := time.Now() - fileID := gid.New(gid.NilTenant, coredata.FileEntityType) - - fileRecord := &coredata.File{ - ID: fileID, - OrganizationID: gid.Nil, - BucketName: bucket, - MimeType: contentType, - FileName: tp.Name + "-logo" + webinspect.ExtensionForMIME(contentType), - FileKey: objectKey.String(), - FileSize: int64(len(body)), - Visibility: coredata.FileVisibilityPublic, - CreatedAt: now, - UpdatedAt: now, - } - - if _, err := fileMgr.PutFile(ctx, fileRecord, bytes.NewReader(body), map[string]string{ - "type": "common-third-party-logo", - "common-third-party-id": party.ID.String(), - }); err != nil { - _, _ = fmt.Fprintf(errOut, "warning: cannot upload logo for %q to S3: %v\n", tp.Name, err) - failed++ - continue - } - - if err := pgClient.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { - if err := fileRecord.Insert(ctx, tx, scope); err != nil { - return fmt.Errorf("cannot insert file record: %w", err) - } - - party.LogoFileID = &fileID - party.UpdatedAt = now - if err := party.UpdateLogoFileID(ctx, tx); err != nil { - return fmt.Errorf("cannot update logo_file_id: %w", err) - } - - return nil - }); err != nil { - _, _ = fmt.Fprintf(errOut, "warning: cannot store logo for %q: %v\n", tp.Name, err) - failed++ - continue - } - - fetched++ - _, _ = fmt.Fprintf(out, " fetched logo for %q\n", tp.Name) - } - - _, _ = fmt.Fprintf(out, "logos: %d fetched, %d skipped, %d failed\n", fetched, skipped, failed) - return nil -} - -func newS3Client(endpoint, region, accessKey, secretKey string, usePathStyle bool) *s3.Client { - if region == "" { - region = "us-east-2" - } - - cfg := aws.Config{ - Region: region, - } - - if accessKey != "" && secretKey != "" { - cfg.Credentials = credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") - } - - if endpoint != "" { - cfg.BaseEndpoint = &endpoint - } - - return s3.NewFromConfig(cfg, func(o *s3.Options) { - o.UsePathStyle = usePathStyle - }) -} - func loadThirdParties(path string) ([]thirdPartyData, error) { f, err := os.Open(path) if err != nil { @@ -413,16 +193,3 @@ func parseCategory(errOut io.Writer, tp thirdPartyData) coredata.ThirdPartyCateg return c } - -type userAgentTransport struct { - next http.RoundTripper - ua string -} - -func (t *userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) { - req = req.Clone(req.Context()) - req.Header.Set("User-Agent", t.ua) - req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8") - req.Header.Set("Accept-Language", "en-US,en;q=0.5") - return t.next.RoundTrip(req) -} diff --git a/pkg/webinspect/logo.go b/pkg/webinspect/logo.go deleted file mode 100644 index 2bba7c162..000000000 --- a/pkg/webinspect/logo.go +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright (c) 2026 Probo Inc . -// -// Permission to use, copy, modify, and/or distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -// PERFORMANCE OF THIS SOFTWARE. - -package webinspect - -import ( - "fmt" - "strconv" - "strings" - - "golang.org/x/net/html" -) - -func FindLogoURL(info *PageInfo) (string, error) { - head := findElement(info.Root, "head") - if head == nil { - return "", fmt.Errorf("cannot find logo: no head element") - } - - var ( - svgIcon string - appleTouchIcon string - appleTouchSize int - largestIcon string - largestSize int - msTileImage string - ) - - for _, n := range findAllIn(head, "link") { - rel := strings.ToLower(attrVal(n, "rel")) - href := attrVal(n, "href") - if href == "" { - continue - } - - switch { - case strings.Contains(rel, "icon") && !strings.Contains(rel, "apple-touch-icon") && attrVal(n, "type") == "image/svg+xml": - svgIcon = href - case strings.Contains(rel, "apple-touch-icon"): - size := parseSizeAttr(attrVal(n, "sizes")) - if appleTouchIcon == "" || size > appleTouchSize { - appleTouchIcon = href - appleTouchSize = size - } - case strings.Contains(rel, "icon") && !strings.Contains(rel, "apple-touch-icon"): - size := parseSizeAttr(attrVal(n, "sizes")) - if largestIcon == "" || size > largestSize { - largestIcon = href - largestSize = size - } - } - } - - for _, n := range findAllIn(head, "meta") { - name := strings.ToLower(attrVal(n, "name")) - content := attrVal(n, "content") - if name == "msapplication-tileimage" && content != "" { - msTileImage = content - } - } - - candidates := []string{ - svgIcon, - appleTouchIcon, - largestIcon, - msTileImage, - } - - for _, href := range candidates { - if href != "" { - return info.ResolveHref(href), nil - } - } - - return "", fmt.Errorf("cannot find logo") -} - -func parseSizeAttr(sizes string) int { - if sizes == "" || strings.EqualFold(sizes, "any") { - return 0 - } - - best := 0 - for token := range strings.FieldsSeq(sizes) { - token = strings.ToLower(token) - parts := strings.SplitN(token, "x", 2) - if len(parts) != 2 { - continue - } - - w, err := strconv.Atoi(parts[0]) - if err != nil { - continue - } - - if w > best { - best = w - } - } - - return best -} - -func ExtensionForMIME(contentType string) string { - ct := strings.ToLower(contentType) - if idx := strings.Index(ct, ";"); idx != -1 { - ct = ct[:idx] - } - ct = strings.TrimSpace(ct) - - switch ct { - case "image/svg+xml": - return ".svg" - case "image/png": - return ".png" - case "image/jpeg": - return ".jpg" - case "image/gif": - return ".gif" - case "image/webp": - return ".webp" - case "image/x-icon", "image/vnd.microsoft.icon": - return ".ico" - default: - return ".png" - } -} - -// HeadLinks returns all nodes from whose rel attribute -// contains the given value (case-insensitive partial match). -func (p *PageInfo) HeadLinks(rel string) []*html.Node { - head := findElement(p.Root, "head") - if head == nil { - return nil - } - - rel = strings.ToLower(rel) - var matches []*html.Node - - for _, n := range findAllIn(head, "link") { - if strings.Contains(strings.ToLower(attrVal(n, "rel")), rel) { - matches = append(matches, n) - } - } - - return matches -} - -// HeadMeta returns the content attribute of the first tag in -// whose name attribute matches (case-insensitive). -func (p *PageInfo) HeadMeta(name string) (string, bool) { - head := findElement(p.Root, "head") - if head == nil { - return "", false - } - - name = strings.ToLower(name) - - for _, n := range findAllIn(head, "meta") { - if strings.EqualFold(attrVal(n, "name"), name) { - content := attrVal(n, "content") - if content != "" { - return content, true - } - } - } - - return "", false -} diff --git a/pkg/webinspect/logo_test.go b/pkg/webinspect/logo_test.go deleted file mode 100644 index 77b06d8b3..000000000 --- a/pkg/webinspect/logo_test.go +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright (c) 2026 Probo Inc . -// -// Permission to use, copy, modify, and/or distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -// PERFORMANCE OF THIS SOFTWARE. - -package webinspect_test - -import ( - "net/url" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.probo.inc/probo/pkg/webinspect" -) - -func parseTestHTML(t *testing.T, rawURL string, body string) *webinspect.PageInfo { - t.Helper() - - u, err := url.Parse(rawURL) - require.NoError(t, err) - - info, err := webinspect.ParseHTML(u, strings.NewReader(body)) - require.NoError(t, err) - - return info -} - -func TestFindLogoURL_SVGPreferred(t *testing.T) { - t.Parallel() - - info := parseTestHTML(t, "https://example.com", ` - - - - `) - - got, err := webinspect.FindLogoURL(info) - require.NoError(t, err) - assert.Equal(t, "https://example.com/favicon.svg", got) -} - -func TestFindLogoURL_AppleTouchIconSecond(t *testing.T) { - t.Parallel() - - info := parseTestHTML(t, "https://example.com", ` - - - `) - - got, err := webinspect.FindLogoURL(info) - require.NoError(t, err) - assert.Equal(t, "https://example.com/apple-touch-icon.png", got) -} - -func TestFindLogoURL_AppleTouchIconLargest(t *testing.T) { - t.Parallel() - - info := parseTestHTML(t, "https://example.com", ` - - - - `) - - got, err := webinspect.FindLogoURL(info) - require.NoError(t, err) - assert.Equal(t, "https://example.com/touch-180.png", got) -} - -func TestFindLogoURL_LargestIcon(t *testing.T) { - t.Parallel() - - info := parseTestHTML(t, "https://example.com", ` - - - - `) - - got, err := webinspect.FindLogoURL(info) - require.NoError(t, err) - assert.Equal(t, "https://example.com/icon-192.png", got) -} - -func TestFindLogoURL_MsTileImage(t *testing.T) { - t.Parallel() - - info := parseTestHTML(t, "https://example.com", ` - - `) - - got, err := webinspect.FindLogoURL(info) - require.NoError(t, err) - assert.Equal(t, "https://example.com/mstile-144.png", got) -} - -func TestFindLogoURL_RelativeHrefResolved(t *testing.T) { - t.Parallel() - - info := parseTestHTML(t, "https://cdn.example.com/app/", ` - - `) - - got, err := webinspect.FindLogoURL(info) - require.NoError(t, err) - assert.Equal(t, "https://cdn.example.com/assets/logo.svg", got) -} - -func TestFindLogoURL_NoHead(t *testing.T) { - t.Parallel() - - info := parseTestHTML(t, "https://example.com", `

no head

`) - - _, err := webinspect.FindLogoURL(info) - assert.Error(t, err) -} - -func TestFindLogoURL_NothingFound(t *testing.T) { - t.Parallel() - - info := parseTestHTML(t, "https://example.com", ``) - - _, err := webinspect.FindLogoURL(info) - assert.Error(t, err) -} - -func TestExtensionForMIME(t *testing.T) { - t.Parallel() - - tests := []struct { - contentType string - expected string - }{ - {"image/svg+xml", ".svg"}, - {"image/png", ".png"}, - {"image/jpeg", ".jpg"}, - {"image/gif", ".gif"}, - {"image/webp", ".webp"}, - {"image/x-icon", ".ico"}, - {"image/vnd.microsoft.icon", ".ico"}, - {"image/png; charset=utf-8", ".png"}, - {"application/octet-stream", ".png"}, - } - - for _, tt := range tests { - t.Run( - tt.contentType, - func(t *testing.T) { - t.Parallel() - assert.Equal(t, tt.expected, webinspect.ExtensionForMIME(tt.contentType)) - }, - ) - } -} diff --git a/pkg/webinspect/parse.go b/pkg/webinspect/parse.go deleted file mode 100644 index 8f66fe2ca..000000000 --- a/pkg/webinspect/parse.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (c) 2026 Probo Inc . -// -// Permission to use, copy, modify, and/or distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -// PERFORMANCE OF THIS SOFTWARE. - -package webinspect - -import ( - "context" - "fmt" - "io" - "net/http" - "net/url" - - "golang.org/x/net/html" -) - -type PageInfo struct { - URL *url.URL - Root *html.Node -} - -func Parse(ctx context.Context, client *http.Client, websiteURL string) (*PageInfo, error) { - parsed, err := url.Parse(websiteURL) - if err != nil { - return nil, fmt.Errorf("cannot parse website URL: %w", err) - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, websiteURL, nil) - if err != nil { - return nil, fmt.Errorf("cannot create request: %w", err) - } - - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("cannot fetch page: %w", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("cannot fetch page: status %d", resp.StatusCode) - } - - const maxHTMLSize = 10 << 20 // 10 MiB - return ParseHTML(parsed, io.LimitReader(resp.Body, maxHTMLSize)) -} - -func ParseHTML(baseURL *url.URL, r io.Reader) (*PageInfo, error) { - root, err := html.Parse(r) - if err != nil { - return nil, fmt.Errorf("cannot parse HTML: %w", err) - } - - return &PageInfo{URL: baseURL, Root: root}, nil -} - -func (p *PageInfo) ResolveHref(href string) string { - ref, err := url.Parse(href) - if err != nil { - return href - } - - return p.URL.ResolveReference(ref).String() -} - -func findElement(n *html.Node, tag string) *html.Node { - if n.Type == html.ElementNode && n.Data == tag { - return n - } - - for c := n.FirstChild; c != nil; c = c.NextSibling { - if found := findElement(c, tag); found != nil { - return found - } - } - - return nil -} - -func findAllIn(parent *html.Node, tag string) []*html.Node { - var nodes []*html.Node - - for c := parent.FirstChild; c != nil; c = c.NextSibling { - if c.Type == html.ElementNode && c.Data == tag { - nodes = append(nodes, c) - } - - nodes = append(nodes, findAllIn(c, tag)...) - } - - return nodes -} - -func attrVal(n *html.Node, key string) string { - for _, a := range n.Attr { - if a.Key == key { - return a.Val - } - } - - return "" -}