rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize website logo get

Browse Source 
Signed-off-by: Émile Ré <emile@getprobo.com>
This commit is contained in:
Émile Ré
2026-05-11 19:23:54 +04:00
parent 44072b158a
commit f7dd08d432
3 changed files with 104 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
cmd/common-third-parties-import/main.go
View File

@@ -25,6 +25,7 @@ package main
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"flag"
"fmt"
@@ -39,10 +40,12 @@ import (
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"go.gearno.de/crypto/uuid"
"go.gearno.de/kit/httpclient"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/filemanager"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/version"
"go.probo.inc/probo/pkg/webinspect"
)
@@ -231,7 +234,28 @@ func fetchAndStoreLogos(
) error {
s3Client := newS3Client(endpoint, region, accessKey, secretKey, usePathStyle)
fileMgr := filemanager.NewService(s3Client)
httpClient := &http.Client{Timeout: 10 * time.Second}
tlsConfig := &tls.Config{
CipherSuites: []uint16{
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_AES_128_GCM_SHA256,
tls.VersionTLS13,
tls.VersionTLS10,
},
}
httpClient := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
httpClient.Transport = &userAgentTransport{
next: &http.Transport{
TLSHandshakeTimeout: 30 * time.Second,
DisableKeepAlives: false,
TLSClientConfig: tlsConfig,
DialTLS: func(network, addr string) (net.Conn, error) {
return tls.Dial(network, addr, tlsConfig)
},
},
ua: version.UserAgent("common-third-parties-import"),
}
scope := coredata.NewScope(gid.NilTenant)
var fetched, skipped, failed int
@@ -256,37 +280,63 @@ func fetchAndStoreLogos(
continue
}
var logoURL string
pageInfo, err := webinspect.Parse(ctx, httpClient, *tp.WebsiteURL)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot inspect page for %q, skipping logo: %v\n", tp.Name, err)
failed++
continue
fmt.Fprintf(os.Stderr, "warning: cannot inspect page for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
} else {
logoURL, err = webinspect.FindLogoURL(pageInfo)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot find logo for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
}
}
logoURL, err := webinspect.FindLogoURL(pageInfo)
parsed, err := url.Parse(*tp.WebsiteURL)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot find logo for %q: %v\n", tp.Name, err)
fmt.Fprintf(os.Stderr, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err)
failed++
continue
}
resp, err := httpClient.Get(logoURL)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot fetch logo for %q: %v\n", tp.Name, err)
var candidateURLs []string
if logoURL != "" {
candidateURLs = append(candidateURLs, logoURL)
}
base := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
candidateURLs = append(candidateURLs,
base+"/apple-touch-icon.png",
base+"/apple-touch-icon-precomposed.png",
"https://logo.debounce.com/"+parsed.Host,
)
var (
body []byte
contentType string
)
for _, candidate := range candidateURLs {
resp, err := httpClient.Get(candidate)
if err != nil {
continue
}
b, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil || resp.StatusCode != http.StatusOK || len(b) == 0 {
continue
}
body = b
contentType = resp.Header.Get("Content-Type")
break
}
if len(body) == 0 {
fmt.Fprintf(os.Stderr, "warning: cannot fetch logo for %q from any candidate URL\n", tp.Name)
failed++
continue
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil || resp.StatusCode != http.StatusOK || len(body) == 0 {
fmt.Fprintf(os.Stderr, "warning: bad logo response for %q (status %d)\n", tp.Name, resp.StatusCode)
failed++
continue
}
contentType := resp.Header.Get("Content-Type")
if contentType == "" {
contentType = "image/png"
}
@@ -407,7 +457,16 @@ func newPgClientFromDSN(dsn string) (*pg.Client, error) {
return nil, fmt.Errorf("cannot parse DSN: %w", err)
}
opts := []pg.Option{pg.WithUnsecureTLS()}
var opts []pg.Option
switch u.Query().Get("sslmode") {
case "", "disable":
// plain connection, no TLS
case "require", "prefer":
opts = append(opts, pg.WithUnsecureTLS())
default:
return nil, fmt.Errorf("unsupported sslmode %q (only disable, require, prefer are supported)", u.Query().Get("sslmode"))
}
if u.Host != "" {
host := u.Host
@@ -430,3 +489,16 @@ func newPgClientFromDSN(dsn string) (*pg.Client, error) {
return pg.NewClient(opts...)
}
type userAgentTransport struct {
next http.RoundTripper
ua string
}
func (t *userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
req = req.Clone(req.Context())
req.Header.Set("User-Agent", t.ua)
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
return t.next.RoundTrip(req)
}

9
cmd/geoloc-import/main.go
View File

@@ -79,6 +79,15 @@ func newPgClientFromDSN(dsn string) (*pg.Client, error) {
var opts []pg.Option
switch u.Query().Get("sslmode") {
case "", "disable":
// plain connection, no TLS
case "require", "prefer":
opts = append(opts, pg.WithUnsecureTLS())
default:
return nil, fmt.Errorf("unsupported sslmode %q (only disable, require, prefer are supported)", u.Query().Get("sslmode"))
}
if u.Host != "" {
host := u.Host
if u.Port() == "" {

6
packages/vendors/data.json vendored
View File

@@ -400,7 +400,7 @@
"name": "Hubspot",
"legalName": "Hubspot, Inc.",
"headquarterAddress": "2 Canal Park Cambridge, MA 02141. United States",
"websiteUrl": "https://hubspot.com",
"websiteUrl": "https://www.hubspot.com",
"privacyPolicyUrl": "https://legal.hubspot.com/privacy-policy",
"securityPageUrl": "https://legal.hubspot.com/security/",
"dataProcessingAgreementUrl": "https://legal.hubspot.com/dpa",
@@ -1355,7 +1355,7 @@
"name": "Google",
"legalName": "Google LLC",
"headquarterAddress": "1600 Amphitheatre Parkway, Mountain View, CA 94043, US",
"websiteUrl": "https://google.com",
"websiteUrl": "https://www.google.com",
"category": "CLOUD_PROVIDER",
"certifications": [
"ISO 27001",
@@ -2276,7 +2276,7 @@
"name": "Okta",
"headquarterAddress": "100 First Plaza, San Francisco, CA 94105, US",
"legalName": "Okta, Inc.",
"websiteUrl": "https://okta.com",
"websiteUrl": "https://www.okta.com",
"privacyPolicyUrl": "https://www.okta.com/privacy-policy/",
"termsOfServiceUrl": "https://www.okta.com/terms-of-service/",
"category": "IDENTITY_PROVIDER",