From f7dd08d432af52d66359b524ca0b2bc25b53a93b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89mile=20R=C3=A9?= Date: Mon, 11 May 2026 19:23:54 +0400 Subject: [PATCH] Optimize website logo get MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Émile Ré --- cmd/common-third-parties-import/main.go | 112 +++++++++++++++++++----- cmd/geoloc-import/main.go | 9 ++ packages/vendors/data.json | 6 +- 3 files changed, 104 insertions(+), 23 deletions(-) diff --git a/cmd/common-third-parties-import/main.go b/cmd/common-third-parties-import/main.go index 7d94f9d2b..d02774a9e 100644 --- a/cmd/common-third-parties-import/main.go +++ b/cmd/common-third-parties-import/main.go @@ -25,6 +25,7 @@ package main import ( "bytes" "context" + "crypto/tls" "encoding/json" "flag" "fmt" @@ -39,10 +40,12 @@ import ( "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "go.gearno.de/crypto/uuid" + "go.gearno.de/kit/httpclient" "go.gearno.de/kit/pg" "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/filemanager" "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/version" "go.probo.inc/probo/pkg/webinspect" ) @@ -231,7 +234,28 @@ func fetchAndStoreLogos( ) error { s3Client := newS3Client(endpoint, region, accessKey, secretKey, usePathStyle) fileMgr := filemanager.NewService(s3Client) - httpClient := &http.Client{Timeout: 10 * time.Second} + tlsConfig := &tls.Config{ + CipherSuites: []uint16{ + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_AES_128_GCM_SHA256, + tls.VersionTLS13, + tls.VersionTLS10, + }, + } + httpClient := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection()) + httpClient.Transport = &userAgentTransport{ + next: &http.Transport{ + TLSHandshakeTimeout: 30 * time.Second, + DisableKeepAlives: false, + TLSClientConfig: tlsConfig, + DialTLS: func(network, addr string) (net.Conn, error) { + return tls.Dial(network, addr, tlsConfig) + }, + }, + ua: version.UserAgent("common-third-parties-import"), + } scope := coredata.NewScope(gid.NilTenant) var fetched, skipped, failed int @@ -256,37 +280,63 @@ func fetchAndStoreLogos( continue } + var logoURL string pageInfo, err := webinspect.Parse(ctx, httpClient, *tp.WebsiteURL) if err != nil { - fmt.Fprintf(os.Stderr, "warning: cannot inspect page for %q, skipping logo: %v\n", tp.Name, err) - failed++ - continue + fmt.Fprintf(os.Stderr, "warning: cannot inspect page for %q, trying default apple-touch-icon: %v\n", tp.Name, err) + } else { + logoURL, err = webinspect.FindLogoURL(pageInfo) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: cannot find logo for %q, trying default apple-touch-icon: %v\n", tp.Name, err) + } } - logoURL, err := webinspect.FindLogoURL(pageInfo) + parsed, err := url.Parse(*tp.WebsiteURL) if err != nil { - fmt.Fprintf(os.Stderr, "warning: cannot find logo for %q: %v\n", tp.Name, err) + fmt.Fprintf(os.Stderr, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err) failed++ continue } - resp, err := httpClient.Get(logoURL) - if err != nil { - fmt.Fprintf(os.Stderr, "warning: cannot fetch logo for %q: %v\n", tp.Name, err) + var candidateURLs []string + if logoURL != "" { + candidateURLs = append(candidateURLs, logoURL) + } + base := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host) + candidateURLs = append(candidateURLs, + base+"/apple-touch-icon.png", + base+"/apple-touch-icon-precomposed.png", + "https://logo.debounce.com/"+parsed.Host, + ) + + var ( + body []byte + contentType string + ) + for _, candidate := range candidateURLs { + resp, err := httpClient.Get(candidate) + if err != nil { + continue + } + + b, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if err != nil || resp.StatusCode != http.StatusOK || len(b) == 0 { + continue + } + + body = b + contentType = resp.Header.Get("Content-Type") + break + } + + if len(body) == 0 { + fmt.Fprintf(os.Stderr, "warning: cannot fetch logo for %q from any candidate URL\n", tp.Name) failed++ continue } - body, err := io.ReadAll(resp.Body) - _ = resp.Body.Close() - - if err != nil || resp.StatusCode != http.StatusOK || len(body) == 0 { - fmt.Fprintf(os.Stderr, "warning: bad logo response for %q (status %d)\n", tp.Name, resp.StatusCode) - failed++ - continue - } - - contentType := resp.Header.Get("Content-Type") if contentType == "" { contentType = "image/png" } @@ -407,7 +457,16 @@ func newPgClientFromDSN(dsn string) (*pg.Client, error) { return nil, fmt.Errorf("cannot parse DSN: %w", err) } - opts := []pg.Option{pg.WithUnsecureTLS()} + var opts []pg.Option + + switch u.Query().Get("sslmode") { + case "", "disable": + // plain connection, no TLS + case "require", "prefer": + opts = append(opts, pg.WithUnsecureTLS()) + default: + return nil, fmt.Errorf("unsupported sslmode %q (only disable, require, prefer are supported)", u.Query().Get("sslmode")) + } if u.Host != "" { host := u.Host @@ -430,3 +489,16 @@ func newPgClientFromDSN(dsn string) (*pg.Client, error) { return pg.NewClient(opts...) } + +type userAgentTransport struct { + next http.RoundTripper + ua string +} + +func (t *userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + req.Header.Set("User-Agent", t.ua) + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.5") + return t.next.RoundTrip(req) +} diff --git a/cmd/geoloc-import/main.go b/cmd/geoloc-import/main.go index a8e1bf452..933cc4c5c 100644 --- a/cmd/geoloc-import/main.go +++ b/cmd/geoloc-import/main.go @@ -79,6 +79,15 @@ func newPgClientFromDSN(dsn string) (*pg.Client, error) { var opts []pg.Option + switch u.Query().Get("sslmode") { + case "", "disable": + // plain connection, no TLS + case "require", "prefer": + opts = append(opts, pg.WithUnsecureTLS()) + default: + return nil, fmt.Errorf("unsupported sslmode %q (only disable, require, prefer are supported)", u.Query().Get("sslmode")) + } + if u.Host != "" { host := u.Host if u.Port() == "" { diff --git a/packages/vendors/data.json b/packages/vendors/data.json index c672290f4..8bedf9ca6 100644 --- a/packages/vendors/data.json +++ b/packages/vendors/data.json @@ -400,7 +400,7 @@ "name": "Hubspot", "legalName": "Hubspot, Inc.", "headquarterAddress": "2 Canal Park Cambridge, MA 02141. United States", - "websiteUrl": "https://hubspot.com", + "websiteUrl": "https://www.hubspot.com", "privacyPolicyUrl": "https://legal.hubspot.com/privacy-policy", "securityPageUrl": "https://legal.hubspot.com/security/", "dataProcessingAgreementUrl": "https://legal.hubspot.com/dpa", @@ -1355,7 +1355,7 @@ "name": "Google", "legalName": "Google LLC", "headquarterAddress": "1600 Amphitheatre Parkway, Mountain View, CA 94043, US", - "websiteUrl": "https://google.com", + "websiteUrl": "https://www.google.com", "category": "CLOUD_PROVIDER", "certifications": [ "ISO 27001", @@ -2276,7 +2276,7 @@ "name": "Okta", "headquarterAddress": "100 First Plaza, San Francisco, CA 94105, US", "legalName": "Okta, Inc.", - "websiteUrl": "https://okta.com", + "websiteUrl": "https://www.okta.com", "privacyPolicyUrl": "https://www.okta.com/privacy-policy/", "termsOfServiceUrl": "https://www.okta.com/terms-of-service/", "category": "IDENTITY_PROVIDER",