Add slug-based dedup and auto-create third parties from OCD

Add a slug column to common_third_parties for stable cross-source
deduplication. The OCD tracker patterns importer now resolves third
parties via a three-step cascade (slug lookup, domain fallback,
auto-create) instead of silently dropping unmatched links.

Signed-off-by: Émile Ré <emile@probo.com>

cmd/common-third-parties-import/main.go

@@ -14,7 +14,7 @@
 
 // Command common-third-parties-import seeds the common_third_parties table from
 // packages/thirdParties/data.json. It is idempotent: re-running upserts on conflict
-// (lower(name)) so existing rows keep their id and created_at.
+// (slug) so existing rows keep their id and created_at.
 //
 // When -fetch-logos is set, the tool inspects each third party's website to
 // find the best available logo (SVG icon, apple-touch-icon, etc.) and stores
@@ -44,6 +44,7 @@ import (
 	"go.probo.inc/probo/pkg/coredata"
 	"go.probo.inc/probo/pkg/filemanager"
 	"go.probo.inc/probo/pkg/gid"
+	"go.probo.inc/probo/pkg/slug"
 	"go.probo.inc/probo/pkg/version"
 	"go.probo.inc/probo/pkg/webinspect"
 )
@@ -177,6 +178,7 @@ func run() error {
 				party := coredata.CommonThirdParty{
 					ID:                            gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
 					Name:                          tp.Name,
+					Slug:                          slug.Make(tp.Name),
 					Category:                      parseCategory(tp),
 					HeadquarterAddress:            tp.HeadquarterAddress,
 					LegalName:                     tp.LegalName,

cmd/common-tracker-patterns-import/main.go

@@ -19,16 +19,17 @@
 // existing rows are updated on the unique constraint so ids and created_at
 // are preserved.
 //
-// Entries are linked to the matching common_third_parties row (by
-// case-insensitive name lookup on the platform key). Entries whose platform
-// cannot be resolved are inserted with a NULL common_third_party_id.
+// Entries are linked to the matching common_third_parties row via a
+// three-step cascade: slug lookup, domain lookup, then auto-create.
 package main
 
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
+	"math"
 	"net"
 	"net/url"
 	"os"
@@ -43,6 +44,7 @@ import (
 	"go.gearno.de/kit/pg"
 	"go.probo.inc/probo/pkg/coredata"
 	"go.probo.inc/probo/pkg/gid"
+	"go.probo.inc/probo/pkg/slug"
 )
 
 const (
@@ -68,6 +70,8 @@ type (
 		TrackerType    string
 		MatchType      string
 		ThirdPartyName *string
+		Domain         string
+		Category       string
 		Description    string
 		MaxAgeSeconds  *int
 		Confidence     float32
@@ -119,6 +123,7 @@ func run() error {
 	fmt.Printf("importing %d common tracker patterns from Open Cookie Database\n", len(patterns))
 
 	var inserted, updated, skipped int
+	var partiesCreated int
 
 	if err := pgClient.WithTx(
 		ctx,
@@ -127,26 +132,9 @@ func run() error {
 			thirdPartyCache := make(map[string]*gid.GID)
 
 			for _, p := range patterns {
-				var thirdPartyID *gid.GID
-
-				if p.ThirdPartyName != nil {
-					if cached, ok := thirdPartyCache[*p.ThirdPartyName]; ok {
-						thirdPartyID = cached
-					} else {
-						var party coredata.CommonThirdParty
-						if err := party.LoadByName(ctx, tx, *p.ThirdPartyName); err != nil {
-							fmt.Fprintf(
-								os.Stderr,
-								"warning: cannot find third party %q for pattern %q, skipping link\n",
-								*p.ThirdPartyName,
-								p.Pattern,
-							)
-							thirdPartyCache[*p.ThirdPartyName] = nil
-						} else {
-							thirdPartyCache[*p.ThirdPartyName] = &party.ID
-							thirdPartyID = &party.ID
-						}
-					}
+				thirdPartyID, err := resolveThirdParty(ctx, tx, p, thirdPartyCache, now, &partiesCreated)
+				if err != nil {
+					return err
 				}
 
 				trackerType, err := parseTrackerType(p.TrackerType)
@@ -195,11 +183,12 @@ func run() error {
 	}
 
 	fmt.Printf(
-		"imported %d patterns (%d inserted, %d updated, %d skipped)\n",
+		"imported %d patterns (%d inserted, %d updated, %d skipped, %d third parties auto-created)\n",
 		len(patterns)-skipped,
 		inserted,
 		updated,
 		skipped,
+		partiesCreated,
 	)
 
 	return nil
@@ -271,6 +260,8 @@ func loadPatternsFromOCD(dir string) ([]trackerPatternData, error) {
 					TrackerType:    "COOKIE",
 					MatchType:      matchType,
 					ThirdPartyName: new(platform),
+					Domain:         e.Domain,
+					Category:       e.Category,
 					Description:    e.Description,
 					MaxAgeSeconds:  parseRetentionPeriod(e.RetentionPeriod),
 					Confidence:     1.0,
@@ -282,6 +273,136 @@ func loadPatternsFromOCD(dir string) ([]trackerPatternData, error) {
 	return patterns, nil
 }
 
+func resolveThirdParty(
+	ctx context.Context,
+	tx pg.Tx,
+	p trackerPatternData,
+	cache map[string]*gid.GID,
+	now time.Time,
+	created *int,
+) (*gid.GID, error) {
+	if p.ThirdPartyName == nil || *p.ThirdPartyName == "" {
+		return nil, nil
+	}
+
+	platformSlug := slug.Make(*p.ThirdPartyName)
+	if platformSlug == "" {
+		return nil, nil
+	}
+	if cached, ok := cache[platformSlug]; ok {
+		return cached, nil
+	}
+
+	var party coredata.CommonThirdParty
+	if err := party.LoadBySlug(ctx, tx, platformSlug); err != nil {
+		if !errors.Is(err, coredata.ErrResourceNotFound) {
+			return nil, fmt.Errorf("cannot look up common third party by slug %q: %w", platformSlug, err)
+		}
+	} else {
+		cache[platformSlug] = &party.ID
+		return &party.ID, nil
+	}
+
+	domain := normalizeDomain(p.Domain)
+	if domain != "" {
+		var domainRow coredata.CommonThirdPartyDomain
+		if err := domainRow.LoadByDomain(ctx, tx, domain); err != nil {
+			if !errors.Is(err, coredata.ErrResourceNotFound) {
+				return nil, fmt.Errorf("cannot look up common third party by domain %q: %w", domain, err)
+			}
+		} else if err := party.LoadByID(ctx, tx, domainRow.CommonThirdPartyID); err == nil {
+			cache[platformSlug] = &party.ID
+			return &party.ID, nil
+		}
+	}
+
+	party = coredata.CommonThirdParty{
+		ID:             gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
+		Name:           *p.ThirdPartyName,
+		Slug:           platformSlug,
+		Category:       mapOCDCategory(p.Category),
+		Certifications: []string{},
+		CreatedAt:      now,
+		UpdatedAt:      now,
+	}
+
+	if _, err := party.Upsert(ctx, tx); err != nil {
+		return nil, fmt.Errorf("cannot auto-create common third party %q: %w", *p.ThirdPartyName, err)
+	}
+
+	if err := party.LoadBySlug(ctx, tx, platformSlug); err != nil {
+		return nil, fmt.Errorf("cannot reload auto-created common third party %q: %w", *p.ThirdPartyName, err)
+	}
+
+	if domain != "" {
+		d := coredata.CommonThirdPartyDomain{
+			ID:                 gid.New(gid.NilTenant, coredata.CommonThirdPartyDomainEntityType),
+			CommonThirdPartyID: party.ID,
+			Domain:             domain,
+			CreatedAt:          now,
+			UpdatedAt:          now,
+		}
+		if _, err := d.Upsert(ctx, tx); err != nil {
+			return nil, fmt.Errorf("cannot upsert domain %q for %q: %w", domain, *p.ThirdPartyName, err)
+		}
+	}
+
+	*created++
+	cache[platformSlug] = &party.ID
+	return &party.ID, nil
+}
+
+var domainValidRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9.-]*[a-zA-Z]$`)
+
+func normalizeDomain(s string) string {
+	s = strings.TrimSpace(s)
+
+	// Strip zero-width spaces and other Unicode control characters.
+	s = strings.Map(func(r rune) rune {
+		if r == '\u200b' || r == '\ufeff' {
+			return -1
+		}
+		return r
+	}, s)
+
+	// Take only the first domain when "or" separates multiple values,
+	// e.g. ".bing.com (3rd party) or .microsoft.com (3rd party)".
+	if idx := strings.Index(s, " or "); idx != -1 {
+		s = s[:idx]
+	}
+	// Handle values starting with "or ", e.g. "or demdex.net (3rd party)".
+	s = strings.TrimPrefix(s, "or ")
+
+	if idx := strings.IndexByte(s, '('); idx != -1 {
+		s = strings.TrimSpace(s[:idx])
+	}
+
+	// Strip placeholders like "[account]".
+	if strings.ContainsAny(s, "[]") {
+		return ""
+	}
+
+	s = strings.TrimPrefix(s, ".")
+	s = strings.TrimSuffix(s, ".")
+	s = strings.TrimSpace(s)
+
+	if s == "" || !domainValidRe.MatchString(s) {
+		return ""
+	}
+	return s
+}
+
+func mapOCDCategory(s string) coredata.ThirdPartyCategory {
+	switch strings.ToLower(strings.TrimSpace(s)) {
+	case "analytics":
+		return coredata.ThirdPartyCategoryAnalytics
+	case "marketing":
+		return coredata.ThirdPartyCategoryMarketing
+	default:
+		return coredata.ThirdPartyCategoryOther
+	}
+}
+
 var retentionRe = regexp.MustCompile(`(?i)^(\d+)\s+(second|seconds|sec|secs|minute|minutes|mins|min|hour|hours|day|days|week|weeks|month|months|year|years)`)
 
 func parseRetentionPeriod(s string) *int {
@@ -332,7 +453,7 @@ func parseRetentionPeriod(s string) *int {
 		return nil
 	}
 
-	result := n * multiplier
+	result := min(n*multiplier, math.MaxInt32)
 	return &result
 }
 

pkg/coredata/common_third_party.go

@@ -30,6 +30,7 @@ type (
 	CommonThirdParty struct {
 		ID                            gid.GID            `db:"id"`
 		Name                          string             `db:"name"`
+		Slug                          string             `db:"slug"`
 		Category                      ThirdPartyCategory `db:"category"`
 		HeadquarterAddress            *string            `db:"headquarter_address"`
 		LegalName                     *string            `db:"legal_name"`
@@ -62,6 +63,7 @@ func (t *CommonThirdParty) LoadByID(
 SELECT
     id,
     name,
+    slug,
     category,
     headquarter_address,
     legal_name,
@@ -117,6 +119,7 @@ func (t *CommonThirdParty) LoadByName(
 SELECT
     id,
     name,
+    slug,
     category,
     headquarter_address,
     legal_name,
@@ -163,6 +166,62 @@ LIMIT 1;
 	return nil
 }
 
+func (t *CommonThirdParty) LoadBySlug(
+	ctx context.Context,
+	conn pg.Querier,
+	slug string,
+) error {
+	q := `
+SELECT
+    id,
+    name,
+    slug,
+    category,
+    headquarter_address,
+    legal_name,
+    website_url,
+    privacy_policy_url,
+    service_level_agreement_url,
+    service_software_agreement_url,
+    data_processing_agreement_url,
+    business_associate_agreement_url,
+    subprocessors_list_url,
+    certifications,
+    status_page_url,
+    terms_of_service_url,
+    security_page_url,
+    trust_page_url,
+    logo_file_id,
+    created_at,
+    updated_at
+FROM
+    common_third_parties
+WHERE
+    slug = @slug
+LIMIT 1;
+`
+
+	args := pgx.StrictNamedArgs{"slug": slug}
+
+	rows, err := conn.Query(ctx, q, args)
+	if err != nil {
+		return fmt.Errorf("cannot query common third party by slug: %w", err)
+	}
+	defer rows.Close()
+
+	row, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[CommonThirdParty])
+	if err != nil {
+		if errors.Is(err, pgx.ErrNoRows) {
+			return ErrResourceNotFound
+		}
+		return fmt.Errorf("cannot collect common third party by slug: %w", err)
+	}
+
+	*t = row
+
+	return nil
+}
+
 func (t CommonThirdParty) Insert(
 	ctx context.Context,
 	conn pg.Tx,
@@ -171,6 +230,7 @@ func (t CommonThirdParty) Insert(
 INSERT INTO common_third_parties (
     id,
     name,
+    slug,
     category,
     headquarter_address,
     legal_name,
@@ -192,6 +252,7 @@ INSERT INTO common_third_parties (
 ) VALUES (
     @id,
     @name,
+    @slug,
     @category,
     @headquarter_address,
     @legal_name,
@@ -216,6 +277,7 @@ INSERT INTO common_third_parties (
 	args := pgx.StrictNamedArgs{
 		"id":                               t.ID,
 		"name":                             t.Name,
+		"slug":                             t.Slug,
 		"category":                         t.Category,
 		"headquarter_address":              t.HeadquarterAddress,
 		"legal_name":                       t.LegalName,
@@ -244,7 +306,7 @@ INSERT INTO common_third_parties (
 	return nil
 }
 
-// Upsert inserts a row, or on lower(name) conflict updates every column except
+// Upsert inserts a row, or on slug conflict updates every column except
 // id and created_at. Returns true if a new row was inserted, false if an
 // existing row was updated.
 func (t CommonThirdParty) Upsert(
@@ -255,6 +317,7 @@ func (t CommonThirdParty) Upsert(
 INSERT INTO common_third_parties (
     id,
     name,
+    slug,
     category,
     headquarter_address,
     legal_name,
@@ -276,6 +339,7 @@ INSERT INTO common_third_parties (
 ) VALUES (
     @id,
     @name,
+    @slug,
     @category,
     @headquarter_address,
     @legal_name,
@@ -295,7 +359,7 @@ INSERT INTO common_third_parties (
     @created_at,
     @updated_at
 )
-ON CONFLICT (lower(name)) DO UPDATE
+ON CONFLICT (slug) DO UPDATE
 SET
     name                             = EXCLUDED.name,
     category                         = EXCLUDED.category,
@@ -320,6 +384,7 @@ RETURNING (xmax = 0) AS inserted
 	args := pgx.StrictNamedArgs{
 		"id":                               t.ID,
 		"name":                             t.Name,
+		"slug":                             t.Slug,
 		"category":                         t.Category,
 		"headquarter_address":              t.HeadquarterAddress,
 		"legal_name":                       t.LegalName,
@@ -384,6 +449,7 @@ func (t *CommonThirdParties) LoadAll(
 SELECT
     id,
     name,
+    slug,
     category,
     headquarter_address,
     legal_name,

pkg/coredata/migrations/20260514T123727Z.sql

@@ -0,0 +1,32 @@
+-- Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
+--
+-- Permission to use, copy, modify, and/or distribute this software for any
+-- purpose with or without fee is hereby granted, provided that the above
+-- copyright notice and this permission notice appear in all copies.
+--
+-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+-- PERFORMANCE OF THIS SOFTWARE.
+
+ALTER TABLE common_third_parties ADD COLUMN slug TEXT;
+
+UPDATE common_third_parties
+SET slug = lower(
+    trim(BOTH '-' FROM
+        regexp_replace(
+            regexp_replace(
+                regexp_replace(name, '[^a-zA-Z0-9 _-]', '', 'g'),
+            '[ _]+', '-', 'g'),
+        '-+', '-', 'g')
+    )
+);
+
+ALTER TABLE common_third_parties ALTER COLUMN slug SET NOT NULL;
+
+DROP INDEX common_third_parties_name_key;
+
+CREATE UNIQUE INDEX common_third_parties_slug_key ON common_third_parties (slug);