diff --git a/cmd/common-third-parties-import/main.go b/cmd/common-third-parties-import/main.go index 4f2a66681..bf65e039e 100644 --- a/cmd/common-third-parties-import/main.go +++ b/cmd/common-third-parties-import/main.go @@ -14,7 +14,7 @@ // Command common-third-parties-import seeds the common_third_parties table from // packages/thirdParties/data.json. It is idempotent: re-running upserts on conflict -// (lower(name)) so existing rows keep their id and created_at. +// (slug) so existing rows keep their id and created_at. // // When -fetch-logos is set, the tool inspects each third party's website to // find the best available logo (SVG icon, apple-touch-icon, etc.) and stores @@ -44,6 +44,7 @@ import ( "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/filemanager" "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/slug" "go.probo.inc/probo/pkg/version" "go.probo.inc/probo/pkg/webinspect" ) @@ -177,6 +178,7 @@ func run() error { party := coredata.CommonThirdParty{ ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType), Name: tp.Name, + Slug: slug.Make(tp.Name), Category: parseCategory(tp), HeadquarterAddress: tp.HeadquarterAddress, LegalName: tp.LegalName, diff --git a/cmd/common-tracker-patterns-import/main.go b/cmd/common-tracker-patterns-import/main.go index 96f3c6700..52775a609 100644 --- a/cmd/common-tracker-patterns-import/main.go +++ b/cmd/common-tracker-patterns-import/main.go @@ -19,16 +19,17 @@ // existing rows are updated on the unique constraint so ids and created_at // are preserved. // -// Entries are linked to the matching common_third_parties row (by -// case-insensitive name lookup on the platform key). Entries whose platform -// cannot be resolved are inserted with a NULL common_third_party_id. +// Entries are linked to the matching common_third_parties row via a +// three-step cascade: slug lookup, domain lookup, then auto-create. package main import ( "context" "encoding/json" + "errors" "flag" "fmt" + "math" "net" "net/url" "os" @@ -43,6 +44,7 @@ import ( "go.gearno.de/kit/pg" "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/slug" ) const ( @@ -68,6 +70,8 @@ type ( TrackerType string MatchType string ThirdPartyName *string + Domain string + Category string Description string MaxAgeSeconds *int Confidence float32 @@ -119,6 +123,7 @@ func run() error { fmt.Printf("importing %d common tracker patterns from Open Cookie Database\n", len(patterns)) var inserted, updated, skipped int + var partiesCreated int if err := pgClient.WithTx( ctx, @@ -127,26 +132,9 @@ func run() error { thirdPartyCache := make(map[string]*gid.GID) for _, p := range patterns { - var thirdPartyID *gid.GID - - if p.ThirdPartyName != nil { - if cached, ok := thirdPartyCache[*p.ThirdPartyName]; ok { - thirdPartyID = cached - } else { - var party coredata.CommonThirdParty - if err := party.LoadByName(ctx, tx, *p.ThirdPartyName); err != nil { - fmt.Fprintf( - os.Stderr, - "warning: cannot find third party %q for pattern %q, skipping link\n", - *p.ThirdPartyName, - p.Pattern, - ) - thirdPartyCache[*p.ThirdPartyName] = nil - } else { - thirdPartyCache[*p.ThirdPartyName] = &party.ID - thirdPartyID = &party.ID - } - } + thirdPartyID, err := resolveThirdParty(ctx, tx, p, thirdPartyCache, now, &partiesCreated) + if err != nil { + return err } trackerType, err := parseTrackerType(p.TrackerType) @@ -195,11 +183,12 @@ func run() error { } fmt.Printf( - "imported %d patterns (%d inserted, %d updated, %d skipped)\n", + "imported %d patterns (%d inserted, %d updated, %d skipped, %d third parties auto-created)\n", len(patterns)-skipped, inserted, updated, skipped, + partiesCreated, ) return nil @@ -271,6 +260,8 @@ func loadPatternsFromOCD(dir string) ([]trackerPatternData, error) { TrackerType: "COOKIE", MatchType: matchType, ThirdPartyName: new(platform), + Domain: e.Domain, + Category: e.Category, Description: e.Description, MaxAgeSeconds: parseRetentionPeriod(e.RetentionPeriod), Confidence: 1.0, @@ -282,6 +273,136 @@ func loadPatternsFromOCD(dir string) ([]trackerPatternData, error) { return patterns, nil } +func resolveThirdParty( + ctx context.Context, + tx pg.Tx, + p trackerPatternData, + cache map[string]*gid.GID, + now time.Time, + created *int, +) (*gid.GID, error) { + if p.ThirdPartyName == nil || *p.ThirdPartyName == "" { + return nil, nil + } + + platformSlug := slug.Make(*p.ThirdPartyName) + if platformSlug == "" { + return nil, nil + } + if cached, ok := cache[platformSlug]; ok { + return cached, nil + } + + var party coredata.CommonThirdParty + if err := party.LoadBySlug(ctx, tx, platformSlug); err != nil { + if !errors.Is(err, coredata.ErrResourceNotFound) { + return nil, fmt.Errorf("cannot look up common third party by slug %q: %w", platformSlug, err) + } + } else { + cache[platformSlug] = &party.ID + return &party.ID, nil + } + + domain := normalizeDomain(p.Domain) + if domain != "" { + var domainRow coredata.CommonThirdPartyDomain + if err := domainRow.LoadByDomain(ctx, tx, domain); err != nil { + if !errors.Is(err, coredata.ErrResourceNotFound) { + return nil, fmt.Errorf("cannot look up common third party by domain %q: %w", domain, err) + } + } else if err := party.LoadByID(ctx, tx, domainRow.CommonThirdPartyID); err == nil { + cache[platformSlug] = &party.ID + return &party.ID, nil + } + } + + party = coredata.CommonThirdParty{ + ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType), + Name: *p.ThirdPartyName, + Slug: platformSlug, + Category: mapOCDCategory(p.Category), + Certifications: []string{}, + CreatedAt: now, + UpdatedAt: now, + } + + if _, err := party.Upsert(ctx, tx); err != nil { + return nil, fmt.Errorf("cannot auto-create common third party %q: %w", *p.ThirdPartyName, err) + } + + if err := party.LoadBySlug(ctx, tx, platformSlug); err != nil { + return nil, fmt.Errorf("cannot reload auto-created common third party %q: %w", *p.ThirdPartyName, err) + } + + if domain != "" { + d := coredata.CommonThirdPartyDomain{ + ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyDomainEntityType), + CommonThirdPartyID: party.ID, + Domain: domain, + CreatedAt: now, + UpdatedAt: now, + } + if _, err := d.Upsert(ctx, tx); err != nil { + return nil, fmt.Errorf("cannot upsert domain %q for %q: %w", domain, *p.ThirdPartyName, err) + } + } + + *created++ + cache[platformSlug] = &party.ID + return &party.ID, nil +} + +var domainValidRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9.-]*[a-zA-Z]$`) + +func normalizeDomain(s string) string { + s = strings.TrimSpace(s) + + // Strip zero-width spaces and other Unicode control characters. + s = strings.Map(func(r rune) rune { + if r == '\u200b' || r == '\ufeff' { + return -1 + } + return r + }, s) + + // Take only the first domain when "or" separates multiple values, + // e.g. ".bing.com (3rd party) or .microsoft.com (3rd party)". + if idx := strings.Index(s, " or "); idx != -1 { + s = s[:idx] + } + // Handle values starting with "or ", e.g. "or demdex.net (3rd party)". + s = strings.TrimPrefix(s, "or ") + + if idx := strings.IndexByte(s, '('); idx != -1 { + s = strings.TrimSpace(s[:idx]) + } + + // Strip placeholders like "[account]". + if strings.ContainsAny(s, "[]") { + return "" + } + + s = strings.TrimPrefix(s, ".") + s = strings.TrimSuffix(s, ".") + s = strings.TrimSpace(s) + + if s == "" || !domainValidRe.MatchString(s) { + return "" + } + return s +} + +func mapOCDCategory(s string) coredata.ThirdPartyCategory { + switch strings.ToLower(strings.TrimSpace(s)) { + case "analytics": + return coredata.ThirdPartyCategoryAnalytics + case "marketing": + return coredata.ThirdPartyCategoryMarketing + default: + return coredata.ThirdPartyCategoryOther + } +} + var retentionRe = regexp.MustCompile(`(?i)^(\d+)\s+(second|seconds|sec|secs|minute|minutes|mins|min|hour|hours|day|days|week|weeks|month|months|year|years)`) func parseRetentionPeriod(s string) *int { @@ -332,7 +453,7 @@ func parseRetentionPeriod(s string) *int { return nil } - result := n * multiplier + result := min(n*multiplier, math.MaxInt32) return &result } diff --git a/pkg/coredata/common_third_party.go b/pkg/coredata/common_third_party.go index e3d429c58..c58883d1f 100644 --- a/pkg/coredata/common_third_party.go +++ b/pkg/coredata/common_third_party.go @@ -30,6 +30,7 @@ type ( CommonThirdParty struct { ID gid.GID `db:"id"` Name string `db:"name"` + Slug string `db:"slug"` Category ThirdPartyCategory `db:"category"` HeadquarterAddress *string `db:"headquarter_address"` LegalName *string `db:"legal_name"` @@ -62,6 +63,7 @@ func (t *CommonThirdParty) LoadByID( SELECT id, name, + slug, category, headquarter_address, legal_name, @@ -117,6 +119,7 @@ func (t *CommonThirdParty) LoadByName( SELECT id, name, + slug, category, headquarter_address, legal_name, @@ -163,6 +166,62 @@ LIMIT 1; return nil } +func (t *CommonThirdParty) LoadBySlug( + ctx context.Context, + conn pg.Querier, + slug string, +) error { + q := ` +SELECT + id, + name, + slug, + category, + headquarter_address, + legal_name, + website_url, + privacy_policy_url, + service_level_agreement_url, + service_software_agreement_url, + data_processing_agreement_url, + business_associate_agreement_url, + subprocessors_list_url, + certifications, + status_page_url, + terms_of_service_url, + security_page_url, + trust_page_url, + logo_file_id, + created_at, + updated_at +FROM + common_third_parties +WHERE + slug = @slug +LIMIT 1; +` + + args := pgx.StrictNamedArgs{"slug": slug} + + rows, err := conn.Query(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot query common third party by slug: %w", err) + } + defer rows.Close() + + row, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[CommonThirdParty]) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return ErrResourceNotFound + } + return fmt.Errorf("cannot collect common third party by slug: %w", err) + } + + *t = row + + return nil +} + func (t CommonThirdParty) Insert( ctx context.Context, conn pg.Tx, @@ -171,6 +230,7 @@ func (t CommonThirdParty) Insert( INSERT INTO common_third_parties ( id, name, + slug, category, headquarter_address, legal_name, @@ -192,6 +252,7 @@ INSERT INTO common_third_parties ( ) VALUES ( @id, @name, + @slug, @category, @headquarter_address, @legal_name, @@ -216,6 +277,7 @@ INSERT INTO common_third_parties ( args := pgx.StrictNamedArgs{ "id": t.ID, "name": t.Name, + "slug": t.Slug, "category": t.Category, "headquarter_address": t.HeadquarterAddress, "legal_name": t.LegalName, @@ -244,7 +306,7 @@ INSERT INTO common_third_parties ( return nil } -// Upsert inserts a row, or on lower(name) conflict updates every column except +// Upsert inserts a row, or on slug conflict updates every column except // id and created_at. Returns true if a new row was inserted, false if an // existing row was updated. func (t CommonThirdParty) Upsert( @@ -255,6 +317,7 @@ func (t CommonThirdParty) Upsert( INSERT INTO common_third_parties ( id, name, + slug, category, headquarter_address, legal_name, @@ -276,6 +339,7 @@ INSERT INTO common_third_parties ( ) VALUES ( @id, @name, + @slug, @category, @headquarter_address, @legal_name, @@ -295,7 +359,7 @@ INSERT INTO common_third_parties ( @created_at, @updated_at ) -ON CONFLICT (lower(name)) DO UPDATE +ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name, category = EXCLUDED.category, @@ -320,6 +384,7 @@ RETURNING (xmax = 0) AS inserted args := pgx.StrictNamedArgs{ "id": t.ID, "name": t.Name, + "slug": t.Slug, "category": t.Category, "headquarter_address": t.HeadquarterAddress, "legal_name": t.LegalName, @@ -384,6 +449,7 @@ func (t *CommonThirdParties) LoadAll( SELECT id, name, + slug, category, headquarter_address, legal_name, diff --git a/pkg/coredata/migrations/20260514T123727Z.sql b/pkg/coredata/migrations/20260514T123727Z.sql new file mode 100644 index 000000000..fd3c7dbbc --- /dev/null +++ b/pkg/coredata/migrations/20260514T123727Z.sql @@ -0,0 +1,32 @@ +-- Copyright (c) 2026 Probo Inc . +-- +-- Permission to use, copy, modify, and/or distribute this software for any +-- purpose with or without fee is hereby granted, provided that the above +-- copyright notice and this permission notice appear in all copies. +-- +-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +-- PERFORMANCE OF THIS SOFTWARE. + +ALTER TABLE common_third_parties ADD COLUMN slug TEXT; + +UPDATE common_third_parties +SET slug = lower( + trim(BOTH '-' FROM + regexp_replace( + regexp_replace( + regexp_replace(name, '[^a-zA-Z0-9 _-]', '', 'g'), + '[ _]+', '-', 'g'), + '-+', '-', 'g') + ) +); + +ALTER TABLE common_third_parties ALTER COLUMN slug SET NOT NULL; + +DROP INDEX common_third_parties_name_key; + +CREATE UNIQUE INDEX common_third_parties_slug_key ON common_third_parties (slug);