From 4a405ce16cdcc52a87e721a9ebd84e1c4ce92e55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89mile=20R=C3=A9?= Date: Mon, 11 May 2026 12:23:52 +0400 Subject: [PATCH] Self-host common third party logos via S3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fetch favicons at import time instead of calling Google's favicon service per page load. Logos are stored as public files in S3 and served through the existing /api/files/v1/{id} endpoint. Signed-off-by: Émile Ré --- .../dialogs/CommonThirdPartyCombobox.tsx | 4 +- cmd/common-third-parties-import/main.go | 217 +++++++++++++++++- pkg/coredata/common_third_party.go | 41 ++++ pkg/coredata/migrations/20260511T081055Z.sql | 15 ++ pkg/probod/probod.go | 2 +- .../v1/common_third_party_resolvers.go | 38 +++ .../v1/graphql/common_third_party.graphql | 1 + .../console/v1/types/common_third_party.go | 2 + pkg/thirdparty/service.go | 26 ++- 9 files changed, 338 insertions(+), 8 deletions(-) create mode 100644 pkg/coredata/migrations/20260511T081055Z.sql create mode 100644 pkg/server/api/console/v1/common_third_party_resolvers.go diff --git a/apps/console/src/pages/organizations/vendors/dialogs/CommonThirdPartyCombobox.tsx b/apps/console/src/pages/organizations/vendors/dialogs/CommonThirdPartyCombobox.tsx index 19cf81997..f74249244 100644 --- a/apps/console/src/pages/organizations/vendors/dialogs/CommonThirdPartyCombobox.tsx +++ b/apps/console/src/pages/organizations/vendors/dialogs/CommonThirdPartyCombobox.tsx @@ -12,7 +12,6 @@ // OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // PERFORMANCE OF THIS SOFTWARE. -import { faviconUrl } from "@probo/helpers"; import { Avatar, ComboboxItem } from "@probo/ui"; import type { PreloadedQuery } from "react-relay"; import { graphql, usePreloadedQuery } from "react-relay"; @@ -31,6 +30,7 @@ export const commonThirdPartiesQuery = graphql` id name websiteUrl + logoUrl ...CreateVendorDialog_commonThirdParty } } @@ -56,7 +56,7 @@ export function CommonThirdPartyCombobox({ > {thirdParty.name} diff --git a/cmd/common-third-parties-import/main.go b/cmd/common-third-parties-import/main.go index e71fb4040..f4c43ddc2 100644 --- a/cmd/common-third-parties-import/main.go +++ b/cmd/common-third-parties-import/main.go @@ -15,20 +15,32 @@ // Command common-third-parties-import seeds the common_third_parties table from // packages/vendors/data.json. It is idempotent: re-running upserts on conflict // (lower(name)) so existing rows keep their id and created_at. +// +// When -fetch-logos is set, the tool also fetches favicons from Google's +// favicon service and stores them in S3 as public files, linking them to each +// common third party via logo_file_id. package main import ( + "bytes" "context" "encoding/json" "flag" "fmt" + "io" "net" + "net/http" "net/url" "os" "time" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "go.gearno.de/crypto/uuid" "go.gearno.de/kit/pg" "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/filemanager" "go.probo.inc/probo/pkg/gid" ) @@ -61,8 +73,15 @@ func main() { func run() error { var ( - pgDSN string - dataPath string + pgDSN string + dataPath string + fetchLogos bool + s3Bucket string + s3Endpoint string + s3Region string + s3AccessKey string + s3SecretKey string + s3UsePathStyle bool ) flag.StringVar( @@ -77,12 +96,58 @@ func run() error { "", "Path to the third-party data.json file", ) + flag.BoolVar( + &fetchLogos, + "fetch-logos", + false, + "Fetch favicons from Google and store them in S3", + ) + flag.StringVar( + &s3Bucket, + "s3-bucket", + os.Getenv("AWS_S3_BUCKET"), + "S3 bucket name (default: AWS_S3_BUCKET env)", + ) + flag.StringVar( + &s3Endpoint, + "s3-endpoint", + os.Getenv("AWS_ENDPOINT_URL"), + "S3 endpoint URL (default: AWS_ENDPOINT_URL env)", + ) + flag.StringVar( + &s3Region, + "s3-region", + os.Getenv("AWS_REGION"), + "S3 region (default: AWS_REGION env)", + ) + flag.StringVar( + &s3AccessKey, + "s3-access-key", + os.Getenv("AWS_ACCESS_KEY_ID"), + "S3 access key ID (default: AWS_ACCESS_KEY_ID env)", + ) + flag.StringVar( + &s3SecretKey, + "s3-secret-key", + os.Getenv("AWS_SECRET_ACCESS_KEY"), + "S3 secret access key (default: AWS_SECRET_ACCESS_KEY env)", + ) + flag.BoolVar( + &s3UsePathStyle, + "s3-path-style", + false, + "Use S3 path-style addressing", + ) flag.Parse() if pgDSN == "" { return fmt.Errorf("set -pg-dsn or DATABASE_URL") } + if fetchLogos && s3Bucket == "" { + return fmt.Errorf("set -s3-bucket or AWS_S3_BUCKET when using -fetch-logos") + } + ctx := context.Background() thirdParties, err := loadThirdParties(dataPath) @@ -148,9 +213,157 @@ func run() error { fmt.Printf("imported %d rows (%d inserted, %d updated)\n", len(thirdParties), inserted, updated) + if fetchLogos { + if err := fetchAndStoreLogos(ctx, pgClient, thirdParties, s3Bucket, s3Endpoint, s3Region, s3AccessKey, s3SecretKey, s3UsePathStyle); err != nil { + return fmt.Errorf("cannot fetch logos: %w", err) + } + } + return nil } +func fetchAndStoreLogos( + ctx context.Context, + pgClient *pg.Client, + thirdParties []thirdPartyData, + bucket, endpoint, region, accessKey, secretKey string, + usePathStyle bool, +) error { + s3Client := newS3Client(endpoint, region, accessKey, secretKey, usePathStyle) + fileMgr := filemanager.NewService(s3Client) + httpClient := &http.Client{Timeout: 10 * time.Second} + scope := coredata.NewScope(gid.NilTenant) + + var fetched, skipped, failed int + + for _, tp := range thirdParties { + if tp.WebsiteURL == nil || *tp.WebsiteURL == "" { + skipped++ + continue + } + + var party coredata.CommonThirdParty + if err := pgClient.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { + return party.LoadByName(ctx, conn, tp.Name) + }); err != nil { + fmt.Fprintf(os.Stderr, "warning: cannot load %q, skipping logo: %v\n", tp.Name, err) + failed++ + continue + } + + if party.LogoFileID != nil { + skipped++ + continue + } + + parsedURL, err := url.Parse(*tp.WebsiteURL) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err) + failed++ + continue + } + + faviconURL := fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=64", parsedURL.Hostname()) + + resp, err := httpClient.Get(faviconURL) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: cannot fetch favicon for %q: %v\n", tp.Name, err) + failed++ + continue + } + + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if err != nil || resp.StatusCode != http.StatusOK || len(body) == 0 { + fmt.Fprintf(os.Stderr, "warning: bad favicon response for %q (status %d)\n", tp.Name, resp.StatusCode) + failed++ + continue + } + + contentType := resp.Header.Get("Content-Type") + if contentType == "" { + contentType = "image/png" + } + + objectKey, err := uuid.NewV7() + if err != nil { + return fmt.Errorf("cannot generate object key: %w", err) + } + + now := time.Now() + fileID := gid.New(gid.NilTenant, coredata.FileEntityType) + + fileRecord := &coredata.File{ + ID: fileID, + OrganizationID: gid.Nil, + BucketName: bucket, + MimeType: contentType, + FileName: parsedURL.Hostname() + "-favicon.png", + FileKey: objectKey.String(), + FileSize: int64(len(body)), + Visibility: coredata.FileVisibilityPublic, + CreatedAt: now, + UpdatedAt: now, + } + + if _, err := fileMgr.PutFile(ctx, fileRecord, bytes.NewReader(body), map[string]string{ + "type": "common-third-party-logo", + "common-third-party-id": party.ID.String(), + }); err != nil { + fmt.Fprintf(os.Stderr, "warning: cannot upload logo for %q to S3: %v\n", tp.Name, err) + failed++ + continue + } + + if err := pgClient.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + if err := fileRecord.Insert(ctx, tx, scope); err != nil { + return fmt.Errorf("cannot insert file record: %w", err) + } + + party.LogoFileID = &fileID + party.UpdatedAt = now + if err := party.UpdateLogoFileID(ctx, tx); err != nil { + return fmt.Errorf("cannot update logo_file_id: %w", err) + } + + return nil + }); err != nil { + fmt.Fprintf(os.Stderr, "warning: cannot store logo for %q: %v\n", tp.Name, err) + failed++ + continue + } + + fetched++ + fmt.Printf(" fetched logo for %q\n", tp.Name) + } + + fmt.Printf("logos: %d fetched, %d skipped, %d failed\n", fetched, skipped, failed) + return nil +} + +func newS3Client(endpoint, region, accessKey, secretKey string, usePathStyle bool) *s3.Client { + if region == "" { + region = "us-east-2" + } + + cfg := aws.Config{ + Region: region, + } + + if accessKey != "" && secretKey != "" { + cfg.Credentials = credentials.NewStaticCredentialsProvider(accessKey, secretKey, "") + } + + if endpoint != "" { + cfg.BaseEndpoint = &endpoint + } + + return s3.NewFromConfig(cfg, func(o *s3.Options) { + o.UsePathStyle = usePathStyle + }) +} + func loadThirdParties(path string) ([]thirdPartyData, error) { f, err := os.Open(path) if err != nil { diff --git a/pkg/coredata/common_third_party.go b/pkg/coredata/common_third_party.go index ee739e03b..3feb7526e 100644 --- a/pkg/coredata/common_third_party.go +++ b/pkg/coredata/common_third_party.go @@ -46,6 +46,7 @@ type ( TermsOfServiceURL *string `db:"terms_of_service_url"` SecurityPageURL *string `db:"security_page_url"` TrustPageURL *string `db:"trust_page_url"` + LogoFileID *gid.GID `db:"logo_file_id"` CreatedAt time.Time `db:"created_at"` UpdatedAt time.Time `db:"updated_at"` } @@ -78,6 +79,7 @@ SELECT terms_of_service_url, security_page_url, trust_page_url, + logo_file_id, created_at, updated_at FROM @@ -133,6 +135,7 @@ SELECT terms_of_service_url, security_page_url, trust_page_url, + logo_file_id, created_at, updated_at FROM @@ -187,6 +190,7 @@ INSERT INTO common_third_parties ( terms_of_service_url, security_page_url, trust_page_url, + logo_file_id, created_at, updated_at ) VALUES ( @@ -208,6 +212,7 @@ INSERT INTO common_third_parties ( @terms_of_service_url, @security_page_url, @trust_page_url, + @logo_file_id, @created_at, @updated_at ) @@ -232,6 +237,7 @@ INSERT INTO common_third_parties ( "terms_of_service_url": t.TermsOfServiceURL, "security_page_url": t.SecurityPageURL, "trust_page_url": t.TrustPageURL, + "logo_file_id": t.LogoFileID, "created_at": t.CreatedAt, "updated_at": t.UpdatedAt, } @@ -271,6 +277,7 @@ INSERT INTO common_third_parties ( terms_of_service_url, security_page_url, trust_page_url, + logo_file_id, created_at, updated_at ) VALUES ( @@ -292,6 +299,7 @@ INSERT INTO common_third_parties ( @terms_of_service_url, @security_page_url, @trust_page_url, + @logo_file_id, @created_at, @updated_at ) @@ -337,6 +345,7 @@ RETURNING (xmax = 0) AS inserted "terms_of_service_url": t.TermsOfServiceURL, "security_page_url": t.SecurityPageURL, "trust_page_url": t.TrustPageURL, + "logo_file_id": t.LogoFileID, "created_at": t.CreatedAt, "updated_at": t.UpdatedAt, } @@ -401,6 +410,7 @@ SELECT terms_of_service_url, security_page_url, trust_page_url, + logo_file_id, created_at, updated_at FROM @@ -430,3 +440,34 @@ LIMIT 20 return nil } + +func (t CommonThirdParty) UpdateLogoFileID( + ctx context.Context, + conn pg.Tx, +) error { + q := ` +UPDATE common_third_parties +SET + logo_file_id = @logo_file_id, + updated_at = @updated_at +WHERE + id = @id +` + + args := pgx.StrictNamedArgs{ + "id": t.ID, + "logo_file_id": t.LogoFileID, + "updated_at": t.UpdatedAt, + } + + result, err := conn.Exec(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot update common third party logo: %w", err) + } + + if result.RowsAffected() == 0 { + return ErrResourceNotFound + } + + return nil +} diff --git a/pkg/coredata/migrations/20260511T081055Z.sql b/pkg/coredata/migrations/20260511T081055Z.sql new file mode 100644 index 000000000..d77ce5ec0 --- /dev/null +++ b/pkg/coredata/migrations/20260511T081055Z.sql @@ -0,0 +1,15 @@ +-- Copyright (c) 2026 Probo Inc . +-- +-- Permission to use, copy, modify, and/or distribute this software for any +-- purpose with or without fee is hereby granted, provided that the above +-- copyright notice and this permission notice appear in all copies. +-- +-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +-- PERFORMANCE OF THIS SOFTWARE. + +ALTER TABLE common_third_parties ADD COLUMN logo_file_id TEXT REFERENCES files(id); diff --git a/pkg/probod/probod.go b/pkg/probod/probod.go index 4a1dc4e30..e5f3a60c8 100644 --- a/pkg/probod/probod.go +++ b/pkg/probod/probod.go @@ -520,7 +520,7 @@ func (impl *Implm) Run( l.Named("access-review"), ) - thirdPartyService := thirdparty.NewService(pgClient) + thirdPartyService := thirdparty.NewService(pgClient, fileService) serverHandler, err := server.NewServer( server.Config{ diff --git a/pkg/server/api/console/v1/common_third_party_resolvers.go b/pkg/server/api/console/v1/common_third_party_resolvers.go new file mode 100644 index 000000000..5da1efa4e --- /dev/null +++ b/pkg/server/api/console/v1/common_third_party_resolvers.go @@ -0,0 +1,38 @@ +package console_v1 + +// This file will be automatically regenerated based on the schema, any resolver +// implementations +// will be copied through when generating and any unknown code will be moved to the end. +// Code generated by github.com/99designs/gqlgen version v0.17.90 + +import ( + "context" + "time" + + "go.gearno.de/kit/log" + "go.probo.inc/probo/pkg/server/api/console/v1/schema" + "go.probo.inc/probo/pkg/server/api/console/v1/types" + "go.probo.inc/probo/pkg/server/gqlutils" +) + +// LogoURL is the resolver for the logoUrl field. +func (r *commonThirdPartyResolver) LogoURL(ctx context.Context, obj *types.CommonThirdParty) (*string, error) { + if obj.LogoFileID == nil { + return nil, nil + } + + logoURL, err := r.thirdParty.GenerateLogoURL(ctx, *obj.LogoFileID, 1*time.Hour) + if err != nil { + r.logger.ErrorCtx(ctx, "cannot generate common third party logo URL", log.Error(err)) + return nil, gqlutils.Internal(ctx) + } + + return logoURL, nil +} + +// CommonThirdParty returns schema.CommonThirdPartyResolver implementation. +func (r *Resolver) CommonThirdParty() schema.CommonThirdPartyResolver { + return &commonThirdPartyResolver{r} +} + +type commonThirdPartyResolver struct{ *Resolver } diff --git a/pkg/server/api/console/v1/graphql/common_third_party.graphql b/pkg/server/api/console/v1/graphql/common_third_party.graphql index e850d12ad..d76aaf500 100644 --- a/pkg/server/api/console/v1/graphql/common_third_party.graphql +++ b/pkg/server/api/console/v1/graphql/common_third_party.graphql @@ -31,4 +31,5 @@ type CommonThirdParty trustPageUrl: String statusPageUrl: String termsOfServiceUrl: String + logoUrl: String @goField(forceResolver: true) } diff --git a/pkg/server/api/console/v1/types/common_third_party.go b/pkg/server/api/console/v1/types/common_third_party.go index 3a345899a..859334a22 100644 --- a/pkg/server/api/console/v1/types/common_third_party.go +++ b/pkg/server/api/console/v1/types/common_third_party.go @@ -35,6 +35,7 @@ type CommonThirdParty struct { TrustPageURL *string `json:"trustPageUrl,omitempty"` StatusPageURL *string `json:"statusPageUrl,omitempty"` TermsOfServiceURL *string `json:"termsOfServiceUrl,omitempty"` + LogoFileID *gid.GID `json:"logoFileId,omitempty"` } func NewCommonThirdParty(c *coredata.CommonThirdParty) *CommonThirdParty { @@ -54,5 +55,6 @@ func NewCommonThirdParty(c *coredata.CommonThirdParty) *CommonThirdParty { TrustPageURL: c.TrustPageURL, StatusPageURL: c.StatusPageURL, TermsOfServiceURL: c.TermsOfServiceURL, + LogoFileID: c.LogoFileID, } } diff --git a/pkg/thirdparty/service.go b/pkg/thirdparty/service.go index 1bc721052..31ae7b1fe 100644 --- a/pkg/thirdparty/service.go +++ b/pkg/thirdparty/service.go @@ -17,17 +17,37 @@ package thirdparty import ( "context" "fmt" + "time" "go.gearno.de/kit/pg" "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/file" + "go.probo.inc/probo/pkg/gid" ) type Service struct { - pg *pg.Client + pg *pg.Client + file *file.Service } -func NewService(pgClient *pg.Client) *Service { - return &Service{pg: pgClient} +func NewService(pgClient *pg.Client, fileSvc *file.Service) *Service { + return &Service{ + pg: pgClient, + file: fileSvc, + } +} + +func (s *Service) GenerateLogoURL( + ctx context.Context, + logoFileID gid.GID, + expiresIn time.Duration, +) (*string, error) { + url, err := s.file.GetPublicFileURL(ctx, logoFileID, expiresIn) + if err != nil { + return nil, fmt.Errorf("cannot generate logo URL: %w", err) + } + + return &url, nil } func (s *Service) Search(ctx context.Context, name string) ([]*coredata.CommonThirdParty, error) {