rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove logo fetching logic and unused webinspect package

Browse Source 
Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-05-14 18:42:42 +04:00
parent 48a494461e
commit a3f60968cf
4 changed files with 1 additions and 688 deletions
Download Patch File Download Diff File Expand all files Collapse all files

235
pkg/proboctl/seed/common-third-parties/common_third_parties.go
View File

@@ -15,30 +15,19 @@
package commonthirdparties
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/spf13/cobra"
"go.gearno.de/crypto/uuid"
"go.gearno.de/kit/httpclient"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/filemanager"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/proboctl/cmdutil"
"go.probo.inc/probo/pkg/slug"
"go.probo.inc/probo/pkg/version"
"go.probo.inc/probo/pkg/webinspect"
)
type thirdPartyData struct {
@@ -62,16 +51,7 @@ type thirdPartyData struct {
}
func NewCmdCommonThirdParties(f *cmdutil.Factory) *cobra.Command {
var (
flagData string
flagFetchLogos bool
flagS3Bucket string
flagS3Endpoint string
flagS3Region string
flagS3AccessKey string
flagS3SecretKey string
flagS3UsePathStyle bool
)
var flagData string
cmd := &cobra.Command{
Use: "common-third-parties",
@@ -82,11 +62,6 @@ func NewCmdCommonThirdParties(f *cmdutil.Factory) *cobra.Command {
RunE: func(cmd *cobra.Command, args []string) error {
out := f.IOStreams.Out
errOut := f.IOStreams.ErrOut
if flagFetchLogos && flagS3Bucket == "" {
return fmt.Errorf("set --s3-bucket or AWS_S3_BUCKET when using --fetch-logos")
}
ctx := cmd.Context()
thirdParties, err := loadThirdParties(flagData)
@@ -177,211 +152,16 @@ func NewCmdCommonThirdParties(f *cmdutil.Factory) *cobra.Command {
_, _ = fmt.Fprintf(out, "seeded %d third parties (%d inserted, %d updated)\n", len(thirdParties), inserted, updated)
_, _ = fmt.Fprintf(out, "seeded %d domains (%d inserted, %d updated)\n", domainsInserted+domainsUpdated, domainsInserted, domainsUpdated)
if flagFetchLogos {
if err := fetchAndStoreLogos(
ctx, out, errOut, pgClient, thirdParties,
flagS3Bucket, flagS3Endpoint, flagS3Region, flagS3AccessKey, flagS3SecretKey, flagS3UsePathStyle,
); err != nil {
return fmt.Errorf("cannot fetch logos: %w", err)
}
}
return nil
},
}
cmd.Flags().StringVar(&flagData, "data", "", "Path to the third-party data.json file")
_ = cmd.MarkFlagRequired("data")
cmd.Flags().BoolVar(&flagFetchLogos, "fetch-logos", false, "Fetch favicons and store them in S3")
cmd.Flags().StringVar(&flagS3Bucket, "s3-bucket", os.Getenv("AWS_S3_BUCKET"), "S3 bucket name (default: AWS_S3_BUCKET env)")
cmd.Flags().StringVar(&flagS3Endpoint, "s3-endpoint", os.Getenv("AWS_ENDPOINT_URL"), "S3 endpoint URL (default: AWS_ENDPOINT_URL env)")
cmd.Flags().StringVar(&flagS3Region, "s3-region", os.Getenv("AWS_REGION"), "S3 region (default: AWS_REGION env)")
cmd.Flags().StringVar(&flagS3AccessKey, "s3-access-key", os.Getenv("AWS_ACCESS_KEY_ID"), "S3 access key ID (default: AWS_ACCESS_KEY_ID env)")
cmd.Flags().StringVar(&flagS3SecretKey, "s3-secret-key", os.Getenv("AWS_SECRET_ACCESS_KEY"), "S3 secret access key (default: AWS_SECRET_ACCESS_KEY env)")
cmd.Flags().BoolVar(&flagS3UsePathStyle, "s3-path-style", false, "Use S3 path-style addressing")
return cmd
}
func fetchAndStoreLogos(
ctx context.Context,
out, errOut io.Writer,
pgClient *pg.Client,
thirdParties []thirdPartyData,
bucket, endpoint, region, accessKey, secretKey string,
usePathStyle bool,
) error {
s3Client := newS3Client(endpoint, region, accessKey, secretKey, usePathStyle)
fileMgr := filemanager.NewService(s3Client)
httpClient := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
httpClient.Transport = &userAgentTransport{
next: httpClient.Transport,
ua: version.UserAgent("proboctl"),
}
scope := coredata.NewScope(gid.NilTenant)
var fetched, skipped, failed int
for _, tp := range thirdParties {
if tp.WebsiteURL == nil || *tp.WebsiteURL == "" {
skipped++
continue
}
var party coredata.CommonThirdParty
if err := pgClient.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
return party.LoadByName(ctx, conn, tp.Name)
}); err != nil {
_, _ = fmt.Fprintf(errOut, "warning: cannot load %q, skipping logo: %v\n", tp.Name, err)
failed++
continue
}
if party.LogoFileID != nil {
skipped++
continue
}
var logoURL string
pageInfo, err := webinspect.Parse(ctx, httpClient, *tp.WebsiteURL)
if err != nil {
_, _ = fmt.Fprintf(errOut, "warning: cannot inspect page for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
} else {
logoURL, err = webinspect.FindLogoURL(pageInfo)
if err != nil {
_, _ = fmt.Fprintf(errOut, "warning: cannot find logo for %q, trying default apple-touch-icon: %v\n", tp.Name, err)
}
}
parsed, err := url.Parse(*tp.WebsiteURL)
if err != nil {
_, _ = fmt.Fprintf(errOut, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err)
failed++
continue
}
var candidateURLs []string
if logoURL != "" {
candidateURLs = append(candidateURLs, logoURL)
}
base := fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
candidateURLs = append(
candidateURLs,
base+"/apple-touch-icon.png",
base+"/apple-touch-icon-precomposed.png",
"https://logo.debounce.com/"+parsed.Host,
)
var (
body []byte
contentType string
)
for _, candidate := range candidateURLs {
resp, err := httpClient.Get(candidate)
if err != nil {
continue
}
b, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil || resp.StatusCode != http.StatusOK || len(b) == 0 {
continue
}
body = b
contentType = resp.Header.Get("Content-Type")
break
}
if len(body) == 0 {
_, _ = fmt.Fprintf(errOut, "warning: cannot fetch logo for %q from any candidate URL\n", tp.Name)
failed++
continue
}
if contentType == "" {
contentType = "image/png"
}
objectKey, err := uuid.NewV7()
if err != nil {
return fmt.Errorf("cannot generate object key: %w", err)
}
now := time.Now()
fileID := gid.New(gid.NilTenant, coredata.FileEntityType)
fileRecord := &coredata.File{
ID: fileID,
OrganizationID: gid.Nil,
BucketName: bucket,
MimeType: contentType,
FileName: tp.Name + "-logo" + webinspect.ExtensionForMIME(contentType),
FileKey: objectKey.String(),
FileSize: int64(len(body)),
Visibility: coredata.FileVisibilityPublic,
CreatedAt: now,
UpdatedAt: now,
}
if _, err := fileMgr.PutFile(ctx, fileRecord, bytes.NewReader(body), map[string]string{
"type": "common-third-party-logo",
"common-third-party-id": party.ID.String(),
}); err != nil {
_, _ = fmt.Fprintf(errOut, "warning: cannot upload logo for %q to S3: %v\n", tp.Name, err)
failed++
continue
}
if err := pgClient.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
if err := fileRecord.Insert(ctx, tx, scope); err != nil {
return fmt.Errorf("cannot insert file record: %w", err)
}
party.LogoFileID = &fileID
party.UpdatedAt = now
if err := party.UpdateLogoFileID(ctx, tx); err != nil {
return fmt.Errorf("cannot update logo_file_id: %w", err)
}
return nil
}); err != nil {
_, _ = fmt.Fprintf(errOut, "warning: cannot store logo for %q: %v\n", tp.Name, err)
failed++
continue
}
fetched++
_, _ = fmt.Fprintf(out, " fetched logo for %q\n", tp.Name)
}
_, _ = fmt.Fprintf(out, "logos: %d fetched, %d skipped, %d failed\n", fetched, skipped, failed)
return nil
}
func newS3Client(endpoint, region, accessKey, secretKey string, usePathStyle bool) *s3.Client {
if region == "" {
region = "us-east-2"
}
cfg := aws.Config{
Region: region,
}
if accessKey != "" && secretKey != "" {
cfg.Credentials = credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")
}
if endpoint != "" {
cfg.BaseEndpoint = &endpoint
}
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = usePathStyle
})
}
func loadThirdParties(path string) ([]thirdPartyData, error) {
f, err := os.Open(path)
if err != nil {
@@ -413,16 +193,3 @@ func parseCategory(errOut io.Writer, tp thirdPartyData) coredata.ThirdPartyCateg
return c
}
type userAgentTransport struct {
next http.RoundTripper
ua string
}
func (t *userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
req = req.Clone(req.Context())
req.Header.Set("User-Agent", t.ua)
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
return t.next.RoundTrip(req)
}

180
pkg/webinspect/logo.go
View File

@@ -1,180 +0,0 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package webinspect
import (
"fmt"
"strconv"
"strings"
"golang.org/x/net/html"
)
func FindLogoURL(info *PageInfo) (string, error) {
head := findElement(info.Root, "head")
if head == nil {
return "", fmt.Errorf("cannot find logo: no head element")
}
var (
svgIcon string
appleTouchIcon string
appleTouchSize int
largestIcon string
largestSize int
msTileImage string
)
for _, n := range findAllIn(head, "link") {
rel := strings.ToLower(attrVal(n, "rel"))
href := attrVal(n, "href")
if href == "" {
continue
}
switch {
case strings.Contains(rel, "icon") && !strings.Contains(rel, "apple-touch-icon") && attrVal(n, "type") == "image/svg+xml":
svgIcon = href
case strings.Contains(rel, "apple-touch-icon"):
size := parseSizeAttr(attrVal(n, "sizes"))
if appleTouchIcon == "" || size > appleTouchSize {
appleTouchIcon = href
appleTouchSize = size
}
case strings.Contains(rel, "icon") && !strings.Contains(rel, "apple-touch-icon"):
size := parseSizeAttr(attrVal(n, "sizes"))
if largestIcon == "" || size > largestSize {
largestIcon = href
largestSize = size
}
}
}
for _, n := range findAllIn(head, "meta") {
name := strings.ToLower(attrVal(n, "name"))
content := attrVal(n, "content")
if name == "msapplication-tileimage" && content != "" {
msTileImage = content
}
}
candidates := []string{
svgIcon,
appleTouchIcon,
largestIcon,
msTileImage,
}
for _, href := range candidates {
if href != "" {
return info.ResolveHref(href), nil
}
}
return "", fmt.Errorf("cannot find logo")
}
func parseSizeAttr(sizes string) int {
if sizes == "" || strings.EqualFold(sizes, "any") {
return 0
}
best := 0
for token := range strings.FieldsSeq(sizes) {
token = strings.ToLower(token)
parts := strings.SplitN(token, "x", 2)
if len(parts) != 2 {
continue
}
w, err := strconv.Atoi(parts[0])
if err != nil {
continue
}
if w > best {
best = w
}
}
return best
}
func ExtensionForMIME(contentType string) string {
ct := strings.ToLower(contentType)
if idx := strings.Index(ct, ";"); idx != -1 {
ct = ct[:idx]
}
ct = strings.TrimSpace(ct)
switch ct {
case "image/svg+xml":
return ".svg"
case "image/png":
return ".png"
case "image/jpeg":
return ".jpg"
case "image/gif":
return ".gif"
case "image/webp":
return ".webp"
case "image/x-icon", "image/vnd.microsoft.icon":
return ".ico"
default:
return ".png"
}
}
// HeadLinks returns all <link> nodes from <head> whose rel attribute
// contains the given value (case-insensitive partial match).
func (p *PageInfo) HeadLinks(rel string) []*html.Node {
head := findElement(p.Root, "head")
if head == nil {
return nil
}
rel = strings.ToLower(rel)
var matches []*html.Node
for _, n := range findAllIn(head, "link") {
if strings.Contains(strings.ToLower(attrVal(n, "rel")), rel) {
matches = append(matches, n)
}
}
return matches
}
// HeadMeta returns the content attribute of the first <meta> tag in <head>
// whose name attribute matches (case-insensitive).
func (p *PageInfo) HeadMeta(name string) (string, bool) {
head := findElement(p.Root, "head")
if head == nil {
return "", false
}
name = strings.ToLower(name)
for _, n := range findAllIn(head, "meta") {
if strings.EqualFold(attrVal(n, "name"), name) {
content := attrVal(n, "content")
if content != "" {
return content, true
}
}
}
return "", false
}

163
pkg/webinspect/logo_test.go
View File

@@ -1,163 +0,0 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package webinspect_test
import (
"net/url"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.probo.inc/probo/pkg/webinspect"
)
func parseTestHTML(t *testing.T, rawURL string, body string) *webinspect.PageInfo {
t.Helper()
u, err := url.Parse(rawURL)
require.NoError(t, err)
info, err := webinspect.ParseHTML(u, strings.NewReader(body))
require.NoError(t, err)
return info
}
func TestFindLogoURL_SVGPreferred(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://example.com", `<html><head>
<link rel="icon" type="image/svg+xml" href="/favicon.svg">
<link rel="apple-touch-icon" href="/apple-touch-icon.png">
<link rel="icon" sizes="192x192" href="/icon-192.png">
</head><body></body></html>`)
got, err := webinspect.FindLogoURL(info)
require.NoError(t, err)
assert.Equal(t, "https://example.com/favicon.svg", got)
}
func TestFindLogoURL_AppleTouchIconSecond(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://example.com", `<html><head>
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
<link rel="icon" sizes="32x32" href="/favicon-32.png">
</head><body></body></html>`)
got, err := webinspect.FindLogoURL(info)
require.NoError(t, err)
assert.Equal(t, "https://example.com/apple-touch-icon.png", got)
}
func TestFindLogoURL_AppleTouchIconLargest(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://example.com", `<html><head>
<link rel="apple-touch-icon" sizes="120x120" href="/touch-120.png">
<link rel="apple-touch-icon" sizes="180x180" href="/touch-180.png">
<link rel="apple-touch-icon" sizes="152x152" href="/touch-152.png">
</head><body></body></html>`)
got, err := webinspect.FindLogoURL(info)
require.NoError(t, err)
assert.Equal(t, "https://example.com/touch-180.png", got)
}
func TestFindLogoURL_LargestIcon(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://example.com", `<html><head>
<link rel="icon" sizes="32x32" href="/icon-32.png">
<link rel="icon" sizes="192x192" href="/icon-192.png">
<link rel="icon" sizes="16x16" href="/icon-16.png">
</head><body></body></html>`)
got, err := webinspect.FindLogoURL(info)
require.NoError(t, err)
assert.Equal(t, "https://example.com/icon-192.png", got)
}
func TestFindLogoURL_MsTileImage(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://example.com", `<html><head>
<meta name="msapplication-TileImage" content="/mstile-144.png">
</head><body></body></html>`)
got, err := webinspect.FindLogoURL(info)
require.NoError(t, err)
assert.Equal(t, "https://example.com/mstile-144.png", got)
}
func TestFindLogoURL_RelativeHrefResolved(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://cdn.example.com/app/", `<html><head>
<link rel="icon" type="image/svg+xml" href="../assets/logo.svg">
</head><body></body></html>`)
got, err := webinspect.FindLogoURL(info)
require.NoError(t, err)
assert.Equal(t, "https://cdn.example.com/assets/logo.svg", got)
}
func TestFindLogoURL_NoHead(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://example.com", `<html><body><p>no head</p></body></html>`)
_, err := webinspect.FindLogoURL(info)
assert.Error(t, err)
}
func TestFindLogoURL_NothingFound(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://example.com", `<html><head></head><body></body></html>`)
_, err := webinspect.FindLogoURL(info)
assert.Error(t, err)
}
func TestExtensionForMIME(t *testing.T) {
t.Parallel()
tests := []struct {
contentType string
expected string
}{
{"image/svg+xml", ".svg"},
{"image/png", ".png"},
{"image/jpeg", ".jpg"},
{"image/gif", ".gif"},
{"image/webp", ".webp"},
{"image/x-icon", ".ico"},
{"image/vnd.microsoft.icon", ".ico"},
{"image/png; charset=utf-8", ".png"},
{"application/octet-stream", ".png"},
}
for _, tt := range tests {
t.Run(
tt.contentType,
func(t *testing.T) {
t.Parallel()
assert.Equal(t, tt.expected, webinspect.ExtensionForMIME(tt.contentType))
},
)
}
}

111
pkg/webinspect/parse.go
View File

@@ -1,111 +0,0 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package webinspect
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"golang.org/x/net/html"
)
type PageInfo struct {
URL *url.URL
Root *html.Node
}
func Parse(ctx context.Context, client *http.Client, websiteURL string) (*PageInfo, error) {
parsed, err := url.Parse(websiteURL)
if err != nil {
return nil, fmt.Errorf("cannot parse website URL: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, websiteURL, nil)
if err != nil {
return nil, fmt.Errorf("cannot create request: %w", err)
}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("cannot fetch page: %w", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("cannot fetch page: status %d", resp.StatusCode)
}
const maxHTMLSize = 10 << 20 // 10 MiB
return ParseHTML(parsed, io.LimitReader(resp.Body, maxHTMLSize))
}
func ParseHTML(baseURL *url.URL, r io.Reader) (*PageInfo, error) {
root, err := html.Parse(r)
if err != nil {
return nil, fmt.Errorf("cannot parse HTML: %w", err)
}
return &PageInfo{URL: baseURL, Root: root}, nil
}
func (p *PageInfo) ResolveHref(href string) string {
ref, err := url.Parse(href)
if err != nil {
return href
}
return p.URL.ResolveReference(ref).String()
}
func findElement(n *html.Node, tag string) *html.Node {
if n.Type == html.ElementNode && n.Data == tag {
return n
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
if found := findElement(c, tag); found != nil {
return found
}
}
return nil
}
func findAllIn(parent *html.Node, tag string) []*html.Node {
var nodes []*html.Node
for c := parent.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.ElementNode && c.Data == tag {
nodes = append(nodes, c)
}
nodes = append(nodes, findAllIn(c, tag)...)
}
return nodes
}
func attrVal(n *html.Node, key string) string {
for _, a := range n.Attr {
if a.Key == key {
return a.Val
}
}
return ""
}