rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Self-host common third party logos via S3

Browse Source 
Fetch favicons at import time instead of calling Google's favicon
service per page load. Logos are stored as public files in S3 and
served through the existing /api/files/v1/{id} endpoint.

Signed-off-by: Émile Ré <emile@getprobo.com>
This commit is contained in:
Émile Ré
2026-05-11 12:23:52 +04:00
parent 7099a3d702
commit 4a405ce16c
9 changed files with 338 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
apps/console/src/pages/organizations/vendors/dialogs/CommonThirdPartyCombobox.tsx vendored
View File

@@ -12,7 +12,6 @@
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
import { faviconUrl } from "@probo/helpers";
import { Avatar, ComboboxItem } from "@probo/ui";
import type { PreloadedQuery } from "react-relay";
import { graphql, usePreloadedQuery } from "react-relay";
@@ -31,6 +30,7 @@ export const commonThirdPartiesQuery = graphql`
id
name
websiteUrl
logoUrl
...CreateVendorDialog_commonThirdParty
}
}
@@ -56,7 +56,7 @@ export function CommonThirdPartyCombobox({
>
<Avatar
name={thirdParty.name}
src={faviconUrl(thirdParty.websiteUrl)}
src={thirdParty.logoUrl}
/>
{thirdParty.name}
</ComboboxItem>

217
cmd/common-third-parties-import/main.go
View File

@@ -15,20 +15,32 @@
// Command common-third-parties-import seeds the common_third_parties table from
// packages/vendors/data.json. It is idempotent: re-running upserts on conflict
// (lower(name)) so existing rows keep their id and created_at.
//
// When -fetch-logos is set, the tool also fetches favicons from Google's
// favicon service and stores them in S3 as public files, linking them to each
// common third party via logo_file_id.
package main
import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"go.gearno.de/crypto/uuid"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/filemanager"
"go.probo.inc/probo/pkg/gid"
)
@@ -61,8 +73,15 @@ func main() {
func run() error {
var (
pgDSN string
dataPath string
pgDSN string
dataPath string
fetchLogos bool
s3Bucket string
s3Endpoint string
s3Region string
s3AccessKey string
s3SecretKey string
s3UsePathStyle bool
)
flag.StringVar(
@@ -77,12 +96,58 @@ func run() error {
"",
"Path to the third-party data.json file",
)
flag.BoolVar(
&fetchLogos,
"fetch-logos",
false,
"Fetch favicons from Google and store them in S3",
)
flag.StringVar(
&s3Bucket,
"s3-bucket",
os.Getenv("AWS_S3_BUCKET"),
"S3 bucket name (default: AWS_S3_BUCKET env)",
)
flag.StringVar(
&s3Endpoint,
"s3-endpoint",
os.Getenv("AWS_ENDPOINT_URL"),
"S3 endpoint URL (default: AWS_ENDPOINT_URL env)",
)
flag.StringVar(
&s3Region,
"s3-region",
os.Getenv("AWS_REGION"),
"S3 region (default: AWS_REGION env)",
)
flag.StringVar(
&s3AccessKey,
"s3-access-key",
os.Getenv("AWS_ACCESS_KEY_ID"),
"S3 access key ID (default: AWS_ACCESS_KEY_ID env)",
)
flag.StringVar(
&s3SecretKey,
"s3-secret-key",
os.Getenv("AWS_SECRET_ACCESS_KEY"),
"S3 secret access key (default: AWS_SECRET_ACCESS_KEY env)",
)
flag.BoolVar(
&s3UsePathStyle,
"s3-path-style",
false,
"Use S3 path-style addressing",
)
flag.Parse()
if pgDSN == "" {
return fmt.Errorf("set -pg-dsn or DATABASE_URL")
}
if fetchLogos && s3Bucket == "" {
return fmt.Errorf("set -s3-bucket or AWS_S3_BUCKET when using -fetch-logos")
}
ctx := context.Background()
thirdParties, err := loadThirdParties(dataPath)
@@ -148,9 +213,157 @@ func run() error {
fmt.Printf("imported %d rows (%d inserted, %d updated)\n", len(thirdParties), inserted, updated)
if fetchLogos {
if err := fetchAndStoreLogos(ctx, pgClient, thirdParties, s3Bucket, s3Endpoint, s3Region, s3AccessKey, s3SecretKey, s3UsePathStyle); err != nil {
return fmt.Errorf("cannot fetch logos: %w", err)
}
}
return nil
}
func fetchAndStoreLogos(
ctx context.Context,
pgClient *pg.Client,
thirdParties []thirdPartyData,
bucket, endpoint, region, accessKey, secretKey string,
usePathStyle bool,
) error {
s3Client := newS3Client(endpoint, region, accessKey, secretKey, usePathStyle)
fileMgr := filemanager.NewService(s3Client)
httpClient := &http.Client{Timeout: 10 * time.Second}
scope := coredata.NewScope(gid.NilTenant)
var fetched, skipped, failed int
for _, tp := range thirdParties {
if tp.WebsiteURL == nil || *tp.WebsiteURL == "" {
skipped++
continue
}
var party coredata.CommonThirdParty
if err := pgClient.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
return party.LoadByName(ctx, conn, tp.Name)
}); err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot load %q, skipping logo: %v\n", tp.Name, err)
failed++
continue
}
if party.LogoFileID != nil {
skipped++
continue
}
parsedURL, err := url.Parse(*tp.WebsiteURL)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot parse URL for %q, skipping logo: %v\n", tp.Name, err)
failed++
continue
}
faviconURL := fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=64", parsedURL.Hostname())
resp, err := httpClient.Get(faviconURL)
if err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot fetch favicon for %q: %v\n", tp.Name, err)
failed++
continue
}
body, err := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if err != nil || resp.StatusCode != http.StatusOK || len(body) == 0 {
fmt.Fprintf(os.Stderr, "warning: bad favicon response for %q (status %d)\n", tp.Name, resp.StatusCode)
failed++
continue
}
contentType := resp.Header.Get("Content-Type")
if contentType == "" {
contentType = "image/png"
}
objectKey, err := uuid.NewV7()
if err != nil {
return fmt.Errorf("cannot generate object key: %w", err)
}
now := time.Now()
fileID := gid.New(gid.NilTenant, coredata.FileEntityType)
fileRecord := &coredata.File{
ID: fileID,
OrganizationID: gid.Nil,
BucketName: bucket,
MimeType: contentType,
FileName: parsedURL.Hostname() + "-favicon.png",
FileKey: objectKey.String(),
FileSize: int64(len(body)),
Visibility: coredata.FileVisibilityPublic,
CreatedAt: now,
UpdatedAt: now,
}
if _, err := fileMgr.PutFile(ctx, fileRecord, bytes.NewReader(body), map[string]string{
"type": "common-third-party-logo",
"common-third-party-id": party.ID.String(),
}); err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot upload logo for %q to S3: %v\n", tp.Name, err)
failed++
continue
}
if err := pgClient.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
if err := fileRecord.Insert(ctx, tx, scope); err != nil {
return fmt.Errorf("cannot insert file record: %w", err)
}
party.LogoFileID = &fileID
party.UpdatedAt = now
if err := party.UpdateLogoFileID(ctx, tx); err != nil {
return fmt.Errorf("cannot update logo_file_id: %w", err)
}
return nil
}); err != nil {
fmt.Fprintf(os.Stderr, "warning: cannot store logo for %q: %v\n", tp.Name, err)
failed++
continue
}
fetched++
fmt.Printf(" fetched logo for %q\n", tp.Name)
}
fmt.Printf("logos: %d fetched, %d skipped, %d failed\n", fetched, skipped, failed)
return nil
}
func newS3Client(endpoint, region, accessKey, secretKey string, usePathStyle bool) *s3.Client {
if region == "" {
region = "us-east-2"
}
cfg := aws.Config{
Region: region,
}
if accessKey != "" && secretKey != "" {
cfg.Credentials = credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")
}
if endpoint != "" {
cfg.BaseEndpoint = &endpoint
}
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = usePathStyle
})
}
func loadThirdParties(path string) ([]thirdPartyData, error) {
f, err := os.Open(path)
if err != nil {

41
pkg/coredata/common_third_party.go
View File

@@ -46,6 +46,7 @@ type (
TermsOfServiceURL *string `db:"terms_of_service_url"`
SecurityPageURL *string `db:"security_page_url"`
TrustPageURL *string `db:"trust_page_url"`
LogoFileID *gid.GID `db:"logo_file_id"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
}
@@ -78,6 +79,7 @@ SELECT
terms_of_service_url,
security_page_url,
trust_page_url,
logo_file_id,
created_at,
updated_at
FROM
@@ -133,6 +135,7 @@ SELECT
terms_of_service_url,
security_page_url,
trust_page_url,
logo_file_id,
created_at,
updated_at
FROM
@@ -187,6 +190,7 @@ INSERT INTO common_third_parties (
terms_of_service_url,
security_page_url,
trust_page_url,
logo_file_id,
created_at,
updated_at
) VALUES (
@@ -208,6 +212,7 @@ INSERT INTO common_third_parties (
@terms_of_service_url,
@security_page_url,
@trust_page_url,
@logo_file_id,
@created_at,
@updated_at
)
@@ -232,6 +237,7 @@ INSERT INTO common_third_parties (
"terms_of_service_url": t.TermsOfServiceURL,
"security_page_url": t.SecurityPageURL,
"trust_page_url": t.TrustPageURL,
"logo_file_id": t.LogoFileID,
"created_at": t.CreatedAt,
"updated_at": t.UpdatedAt,
}
@@ -271,6 +277,7 @@ INSERT INTO common_third_parties (
terms_of_service_url,
security_page_url,
trust_page_url,
logo_file_id,
created_at,
updated_at
) VALUES (
@@ -292,6 +299,7 @@ INSERT INTO common_third_parties (
@terms_of_service_url,
@security_page_url,
@trust_page_url,
@logo_file_id,
@created_at,
@updated_at
)
@@ -337,6 +345,7 @@ RETURNING (xmax = 0) AS inserted
"terms_of_service_url": t.TermsOfServiceURL,
"security_page_url": t.SecurityPageURL,
"trust_page_url": t.TrustPageURL,
"logo_file_id": t.LogoFileID,
"created_at": t.CreatedAt,
"updated_at": t.UpdatedAt,
}
@@ -401,6 +410,7 @@ SELECT
terms_of_service_url,
security_page_url,
trust_page_url,
logo_file_id,
created_at,
updated_at
FROM
@@ -430,3 +440,34 @@ LIMIT 20
return nil
}
func (t CommonThirdParty) UpdateLogoFileID(
ctx context.Context,
conn pg.Tx,
) error {
q := `
UPDATE common_third_parties
SET
logo_file_id = @logo_file_id,
updated_at = @updated_at
WHERE
id = @id
`
args := pgx.StrictNamedArgs{
"id": t.ID,
"logo_file_id": t.LogoFileID,
"updated_at": t.UpdatedAt,
}
result, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot update common third party logo: %w", err)
}
if result.RowsAffected() == 0 {
return ErrResourceNotFound
}
return nil
}

15
pkg/coredata/migrations/20260511T081055Z.sql Normal file
View File

@@ -0,0 +1,15 @@
-- Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
--
-- Permission to use, copy, modify, and/or distribute this software for any
-- purpose with or without fee is hereby granted, provided that the above
-- copyright notice and this permission notice appear in all copies.
--
-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-- PERFORMANCE OF THIS SOFTWARE.
ALTER TABLE common_third_parties ADD COLUMN logo_file_id TEXT REFERENCES files(id);

2
pkg/probod/probod.go
View File

@@ -520,7 +520,7 @@ func (impl *Implm) Run(
l.Named("access-review"),
)
thirdPartyService := thirdparty.NewService(pgClient)
thirdPartyService := thirdparty.NewService(pgClient, fileService)
serverHandler, err := server.NewServer(
server.Config{

38
pkg/server/api/console/v1/common_third_party_resolvers.go Normal file
View File

@@ -0,0 +1,38 @@
package console_v1
// This file will be automatically regenerated based on the schema, any resolver
// implementations
// will be copied through when generating and any unknown code will be moved to the end.
// Code generated by github.com/99designs/gqlgen version v0.17.90
import (
"context"
"time"
"go.gearno.de/kit/log"
"go.probo.inc/probo/pkg/server/api/console/v1/schema"
"go.probo.inc/probo/pkg/server/api/console/v1/types"
"go.probo.inc/probo/pkg/server/gqlutils"
)
// LogoURL is the resolver for the logoUrl field.
func (r *commonThirdPartyResolver) LogoURL(ctx context.Context, obj *types.CommonThirdParty) (*string, error) {
if obj.LogoFileID == nil {
return nil, nil
}
logoURL, err := r.thirdParty.GenerateLogoURL(ctx, *obj.LogoFileID, 1*time.Hour)
if err != nil {
r.logger.ErrorCtx(ctx, "cannot generate common third party logo URL", log.Error(err))
return nil, gqlutils.Internal(ctx)
}
return logoURL, nil
}
// CommonThirdParty returns schema.CommonThirdPartyResolver implementation.
func (r *Resolver) CommonThirdParty() schema.CommonThirdPartyResolver {
return &commonThirdPartyResolver{r}
}
type commonThirdPartyResolver struct{ *Resolver }

1
pkg/server/api/console/v1/graphql/common_third_party.graphql
View File

@@ -31,4 +31,5 @@ type CommonThirdParty
trustPageUrl: String
statusPageUrl: String
termsOfServiceUrl: String
logoUrl: String @goField(forceResolver: true)
}

2
pkg/server/api/console/v1/types/common_third_party.go
View File

@@ -35,6 +35,7 @@ type CommonThirdParty struct {
TrustPageURL *string `json:"trustPageUrl,omitempty"`
StatusPageURL *string `json:"statusPageUrl,omitempty"`
TermsOfServiceURL *string `json:"termsOfServiceUrl,omitempty"`
LogoFileID *gid.GID `json:"logoFileId,omitempty"`
}
func NewCommonThirdParty(c *coredata.CommonThirdParty) *CommonThirdParty {
@@ -54,5 +55,6 @@ func NewCommonThirdParty(c *coredata.CommonThirdParty) *CommonThirdParty {
TrustPageURL: c.TrustPageURL,
StatusPageURL: c.StatusPageURL,
TermsOfServiceURL: c.TermsOfServiceURL,
LogoFileID: c.LogoFileID,
}
}

26
pkg/thirdparty/service.go vendored
View File

@@ -17,17 +17,37 @@ package thirdparty
import (
"context"
"fmt"
"time"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/file"
"go.probo.inc/probo/pkg/gid"
)
type Service struct {
pg *pg.Client
pg *pg.Client
file *file.Service
}
func NewService(pgClient *pg.Client) *Service {
return &Service{pg: pgClient}
func NewService(pgClient *pg.Client, fileSvc *file.Service) *Service {
return &Service{
pg: pgClient,
file: fileSvc,
}
}
func (s *Service) GenerateLogoURL(
ctx context.Context,
logoFileID gid.GID,
expiresIn time.Duration,
) (*string, error) {
url, err := s.file.GetPublicFileURL(ctx, logoFileID, expiresIn)
if err != nil {
return nil, fmt.Errorf("cannot generate logo URL: %w", err)
}
return &url, nil
}
func (s *Service) Search(ctx context.Context, name string) ([]*coredata.CommonThirdParty, error) {