rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Restore tracker mapping linking, drop only create

Browse Source 
The tracker-mapping worker had been reduced to catalog resolution only,
which removed not just the auto-creation of an org ThirdParty but also
the auto-linking of an existing one. Only the creation needed to go: it
raced the load-then-create check and produced duplicate vendors.

Restore the full org ThirdParty resolution (exact common-id link,
sibling direct-link, high-confidence heuristic, and the disambiguation
agent) and remove only the CreateFromCommon branch and its
categorisation gate. When nothing matches, the worker now leaves
third_party_id unset rather than creating a vendor; creation happens
exclusively through the explicit ImportFromCommon action. Drop the
now-dead CreateFromCommon helper and rename match.go to common_match.go.

Fix a latent test bug surfaced by actually running the DB-backed suite
(skipped in CI without Postgres): the heuristic-match candidate lacked
Level 1, so the level-filtered candidate loader excluded it and the old
fallback create masked the miss.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-06-11 10:55:29 +02:00
parent 9a7bc26d49
commit 145aeaf402
17 changed files with 1323 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

341
pkg/cookiebanner/tracker_mapping_worker.go
View File

@@ -39,17 +39,20 @@ import (
const defaultMappingStaleAfter = 10 * time.Minute
type trackerMappingHandler struct {
pg *pg.Client
logger *log.Logger
mappingAgent *agent.Agent
agentTimeout time.Duration
staleAfter time.Duration
pg *pg.Client
logger *log.Logger
mappingAgent *agent.Agent
disambiguationAgent *agent.Agent
agentTimeout time.Duration
disambiguationTimeout time.Duration
staleAfter time.Duration
}
func NewTrackerMappingWorker(
pgClient *pg.Client,
logger *log.Logger,
mappingCfg TrackerMappingAgentConfig,
disambiguationCfg thirdparty.DisambiguationAgentConfig,
staleAfter time.Duration,
opts ...worker.Option,
) *worker.Worker[coredata.TrackerPattern] {
@@ -63,16 +66,21 @@ func NewTrackerMappingWorker(
}
h := &trackerMappingHandler{
pg: pgClient,
logger: logger,
agentTimeout: agentTimeout,
staleAfter: staleAfter,
pg: pgClient,
logger: logger,
agentTimeout: agentTimeout,
disambiguationTimeout: disambiguationCfg.Timeout,
staleAfter: staleAfter,
}
if mappingCfg.LLMClient != nil {
h.mappingAgent = buildTrackerMappingAgent(mappingCfg, pgClient, logger)
}
if disambiguationCfg.LLMClient != nil {
h.disambiguationAgent = thirdparty.BuildDisambiguationAgent(disambiguationCfg, logger)
}
return worker.New(
"tracker-mapping-worker",
h,
@@ -124,11 +132,14 @@ func (h *trackerMappingHandler) RecoverStale(ctx context.Context) error {
// catalogMatch is the result of a single catalog signal. commonPatternID
// is the catalog row the signal resolved (or backfilled); commonThirdPartyID
// is the catalog third party the signal discovered, when any. A nil
// *catalogMatch means the signal produced nothing.
// is the catalog third party the signal discovered, when any; thirdPartyID
// is an existing org ThirdParty the signal knows directly (e.g. a sibling
// pattern already promoted in the same organization). A nil *catalogMatch
// means the signal produced nothing.
type catalogMatch struct {
commonPatternID *gid.GID
commonThirdPartyID *gid.GID
thirdPartyID *gid.GID
}
// Process resolves the catalog mapping for a tracker pattern and links it
@@ -145,11 +156,10 @@ type catalogMatch struct {
// common_tracker_pattern_id but its catalog row has no common third
// party yet.
//
// The worker no longer materializes per-org ThirdParty rows: it resolves
// the shared catalog link only. An org ThirdParty is created exclusively
// through the explicit per-vendor import action, which also backfills
// tracker_patterns.third_party_id; an already-set third_party_id is
// preserved here untouched.
// Org ThirdParty resolution only links to an existing party (even for
// uncategorised or extension-sourced patterns); it never creates a brand
// new org ThirdParty. Creating an org ThirdParty from a catalog vendor is
// done exclusively through the explicit ImportFromCommon action.
func (h *trackerMappingHandler) Process(ctx context.Context, tp coredata.TrackerPattern) error {
scope := coredata.NewScopeFromObjectID(tp.ID)
@@ -174,6 +184,7 @@ func (h *trackerMappingHandler) Process(ctx context.Context, tp coredata.Tracker
commonPatternID := det.commonPatternID
commonThirdPartyID := det.commonThirdPartyID
directThirdPartyID := det.directThirdPartyID
// Phase 2: tracker-mapping agent (no transaction). It runs only when
// the deterministic signals could not resolve a catalog third party.
@@ -205,6 +216,25 @@ func (h *trackerMappingHandler) Process(ctx context.Context, tp coredata.Tracker
}
}
// Phase 3: org ThirdParty resolution. The heuristic ranking and the
// disambiguation agent run without a transaction; only the final link
// touches the database (in a short transaction).
thirdPartyID := tp.ThirdPartyID
if thirdPartyID == nil {
switch {
case directThirdPartyID != nil:
thirdPartyID = directThirdPartyID
case commonThirdPartyID != nil:
resolved, err := h.resolveOrgThirdParty(ctx, tp, *commonThirdPartyID)
if err != nil {
return fmt.Errorf("cannot resolve org third party: %w", err)
}
thirdPartyID = resolved
}
}
// Phase 4: persist the pattern mapping in a short transaction. The
// unmatched fallback keeps catalog coverage complete even when no
// vendor was resolved.
@@ -221,6 +251,7 @@ func (h *trackerMappingHandler) Process(ctx context.Context, tp coredata.Tracker
}
tp.CommonTrackerPatternID = commonPatternID
tp.ThirdPartyID = thirdPartyID
tp.UpdatedAt = time.Now()
// Descriptions are owned by the common-pattern enrichment
@@ -290,6 +321,7 @@ type deterministicResult struct {
origin string
commonPatternID *gid.GID
commonThirdPartyID *gid.GID
directThirdPartyID *gid.GID
domains []string
commonThirdPartyPreexisted bool
}
@@ -371,6 +403,7 @@ func (h *trackerMappingHandler) resolveDeterministic(
if siblingMatch != nil {
res.commonPatternID = firstNonNil(res.commonPatternID, siblingMatch.commonPatternID)
res.commonThirdPartyID = siblingMatch.commonThirdPartyID
res.directThirdPartyID = siblingMatch.thirdPartyID
}
if res.commonThirdPartyID != nil {
@@ -805,8 +838,10 @@ func (h *trackerMappingHandler) persistAgentIdentification(
// matchBySiblingOrigin finds other tracker patterns on the same banner
// that share initiator domains with the current pattern. Sharing an
// origin across multiple detected patterns is a strong indicator of the
// same third party, so the common third party the siblings resolve to is
// upserted onto the catalog row.
// same third party. When the siblings resolve to a single existing org
// ThirdParty, that id is returned directly so promotion can link to it
// without re-running heuristics; otherwise the resolved common third
// party is upserted onto the catalog row.
func (h *trackerMappingHandler) matchBySiblingOrigin(
ctx context.Context,
tx pg.Tx,
@@ -837,14 +872,19 @@ func (h *trackerMappingHandler) matchBySiblingOrigin(
scope := coredata.NewScopeFromObjectID(tp.ID)
commonThirdPartyID, err := h.resolveThirdPartyFromSiblings(ctx, tx, scope, siblingIDs)
commonThirdPartyID, thirdPartyID, err := h.resolveThirdPartyFromSiblings(ctx, tx, scope, siblingIDs)
if err != nil {
return nil, fmt.Errorf("cannot resolve third party from siblings: %w", err)
}
// No catalog third party to record: leave catalog creation to a later
// signal or the unmatched fallback.
// No catalog third party to record: surface a directly-known org
// third party (if any) so promotion can still link to it, and leave
// catalog creation to a later signal or the unmatched fallback.
if commonThirdPartyID == nil {
if thirdPartyID != nil {
return &catalogMatch{thirdPartyID: thirdPartyID}, nil
}
return nil, nil
}
@@ -876,25 +916,37 @@ func (h *trackerMappingHandler) matchBySiblingOrigin(
return &catalogMatch{
commonPatternID: &commonPattern.ID,
commonThirdPartyID: commonPattern.CommonThirdPartyID,
thirdPartyID: thirdPartyID,
}, nil
}
// resolveThirdPartyFromSiblings inspects sibling patterns to resolve a
// single unambiguous catalog third party for backfill. It is resolved
// first from the siblings' org ThirdParties, then, when those carry none,
// from siblings' common_tracker_pattern rows. It returns nil when the
// siblings carry no catalog third party or disagree on one.
// third party. It returns two independent signals: a direct org
// ThirdParty (set only when the siblings share a single one — the
// strongest, same-org signal), and a single unambiguous catalog third
// party for backfill. The catalog third party is resolved first from the
// siblings' org ThirdParties, then, when those carry none, from siblings'
// common_tracker_pattern rows. Either signal may be nil; siblings that
// disagree on the catalog third party resolve it to nothing.
func (h *trackerMappingHandler) resolveThirdPartyFromSiblings(
ctx context.Context,
conn pg.Querier,
scope coredata.Scoper,
siblingIDs []gid.GID,
) (commonThirdPartyID *gid.GID, err error) {
) (commonThirdPartyID *gid.GID, thirdPartyID *gid.GID, err error) {
var patterns coredata.TrackerPatterns
thirdPartyIDs, err := patterns.LoadDistinctThirdPartyIDsByIDs(ctx, conn, scope, siblingIDs)
if err != nil {
return nil, fmt.Errorf("cannot load distinct third party ids from siblings: %w", err)
return nil, nil, fmt.Errorf("cannot load distinct third party ids from siblings: %w", err)
}
// A single org third party shared across the siblings is the
// strongest, same-org signal: link to it directly. This is resolved
// independently from the catalog third party used for backfill.
if len(thirdPartyIDs) == 1 {
directID := thirdPartyIDs[0]
thirdPartyID = &directID
}
if len(thirdPartyIDs) > 0 {
@@ -913,26 +965,29 @@ func (h *trackerMappingHandler) resolveThirdPartyFromSiblings(
if len(commonIDs) == 1 {
for id := range commonIDs {
return &id, nil
return &id, thirdPartyID, nil
}
}
// Siblings are linked to several different catalog third
// parties: do not guess one.
// Siblings are promoted to several different catalog third
// parties: do not guess one. A single shared org third party (if
// any) is still a safe direct link.
if len(commonIDs) > 1 {
return nil, nil
return nil, thirdPartyID, nil
}
}
// Fall back to siblings carrying only a common_tracker_pattern_id, or
// whose org ThirdParty is not itself linked to the catalog.
// whose org ThirdParty is not itself linked to the catalog. This is
// reached when the org-third-party scan above found no catalog third
// party, so it must not be short-circuited by a direct match.
commonPatternIDs, err := patterns.LoadDistinctCommonTrackerPatternIDsByIDs(ctx, conn, scope, siblingIDs)
if err != nil {
return nil, fmt.Errorf("cannot load distinct common tracker pattern ids from siblings: %w", err)
return nil, nil, fmt.Errorf("cannot load distinct common tracker pattern ids from siblings: %w", err)
}
if len(commonPatternIDs) == 0 {
return nil, nil
return nil, thirdPartyID, nil
}
commonIDs := make(map[gid.GID]struct{})
@@ -950,11 +1005,11 @@ func (h *trackerMappingHandler) resolveThirdPartyFromSiblings(
if len(commonIDs) == 1 {
for id := range commonIDs {
return &id, nil
return &id, thirdPartyID, nil
}
}
return nil, nil
return nil, thirdPartyID, nil
}
func (h *trackerMappingHandler) createUnmatchedPattern(
@@ -980,3 +1035,217 @@ func (h *trackerMappingHandler) createUnmatchedPattern(
return &commonPattern.ID, nil
}
// resolveOrgThirdParty resolves an org ThirdParty for the given pattern
// from a known catalog third party by linking to an existing party. The
// resolution order is:
//
// 1. Exact link by common_third_party_id (O(1)).
// 2. Heuristic match against the org's existing ThirdParty rows
// (lowercased name, suffix-stripped name, slug, website host,
// CommonThirdPartyDomain overlap).
// 3. Agent disambiguation when the heuristic is ambiguous.
//
// When none of these resolve an existing party, the function returns
// (nil, nil); it never creates a brand new org ThirdParty (that happens
// only through the explicit ImportFromCommon action). A confident
// heuristic/agent match is auto-tagged with common_third_party_id so
// subsequent resolutions hit the exact-link path in O(1).
func (h *trackerMappingHandler) resolveOrgThirdParty(
ctx context.Context,
tp coredata.TrackerPattern,
commonThirdPartyID gid.GID,
) (*gid.GID, error) {
scope := coredata.NewScopeFromObjectID(tp.ID)
// Read phase: exact link, candidate ranking, and eligibility. No
// write or LLM call happens here.
var prep orgThirdPartyPrep
if err := h.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
var err error
prep, err = h.prepareOrgThirdParty(ctx, conn, scope, tp, commonThirdPartyID)
return err
},
); err != nil {
return nil, err
}
if prep.existingID != nil {
return prep.existingID, nil
}
picked := prep.highConfidence
viaAgent := false
// Agent phase (no transaction): disambiguate among the heuristic
// candidates when none scored high enough on its own.
if picked == nil && prep.eligibleForAgent && h.disambiguationAgent != nil {
matchedID, err := thirdparty.Disambiguate(
ctx,
h.disambiguationAgent,
h.logger,
prep.commonParty,
prep.commonDomains,
prep.agentSet,
h.disambiguationTimeout,
)
if err != nil {
h.logger.WarnCtx(
ctx,
"third-party disambiguation agent failed",
log.Error(err),
log.String("tracker_pattern_id", tp.ID.String()),
)
}
if matchedID != nil {
for _, c := range prep.agentSet {
if c.ThirdParty.ID == *matchedID {
picked = c.ThirdParty
viaAgent = true
break
}
}
}
}
// Nothing to link: leave the pattern without an org third party. An
// org ThirdParty is created only through the explicit ImportFromCommon
// action, never here.
if picked == nil {
return nil, nil
}
// Write phase: link the picked candidate to the catalog entry in a
// short transaction.
if err := h.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
if err := thirdparty.LinkToCommon(ctx, tx, scope, picked, commonThirdPartyID); err != nil {
return fmt.Errorf("cannot link third party to common: %w", err)
}
if viaAgent {
h.logger.InfoCtx(
ctx,
"promoted tracker pattern via disambiguation agent",
log.String("tracker_pattern_id", tp.ID.String()),
log.String("third_party_id", picked.ID.String()),
)
} else {
h.logger.InfoCtx(
ctx,
"promoted tracker pattern via heuristic match",
log.String("tracker_pattern_id", tp.ID.String()),
log.String("third_party_id", picked.ID.String()),
log.Float64("score", prep.highScore),
)
}
return nil
},
); err != nil {
return nil, err
}
return &picked.ID, nil
}
// orgThirdPartyPrep is the read-phase outcome for org ThirdParty
// resolution. existingID is set when an exact common-id link already
// exists (the other fields are then unused). Otherwise highConfidence
// holds a heuristic match at or above HighConfidenceScore (with
// highScore), or agentSet/eligibleForAgent describe the disambiguation
// candidates.
type orgThirdPartyPrep struct {
existingID *gid.GID
commonParty coredata.CommonThirdParty
commonDomains coredata.CommonThirdPartyDomains
agentSet []thirdparty.ScoredCandidate
highConfidence *coredata.ThirdParty
highScore float64
eligibleForAgent bool
}
// prepareOrgThirdParty performs the read-only work for org ThirdParty
// resolution: it checks for an exact common-id link, loads the catalog
// entry and the org's existing third parties, and ranks the candidates.
// It makes no writes and no LLM call.
func (h *trackerMappingHandler) prepareOrgThirdParty(
ctx context.Context,
conn pg.Querier,
scope coredata.Scoper,
tp coredata.TrackerPattern,
commonThirdPartyID gid.GID,
) (orgThirdPartyPrep, error) {
var prep orgThirdPartyPrep
var existing coredata.ThirdParty
err := existing.LoadByOrganizationIDAndCommonThirdPartyID(
ctx,
conn,
scope,
tp.OrganizationID,
commonThirdPartyID,
)
if err == nil {
id := existing.ID
prep.existingID = &id
return prep, nil
}
if !errors.Is(err, coredata.ErrResourceNotFound) {
return prep, fmt.Errorf("cannot load org third party by common id: %w", err)
}
if err := prep.commonParty.LoadByID(ctx, conn, commonThirdPartyID); err != nil {
return prep, fmt.Errorf("cannot load common third party: %w", err)
}
if err := prep.commonDomains.LoadByCommonThirdPartyID(ctx, conn, commonThirdPartyID); err != nil {
return prep, fmt.Errorf("cannot load common third party domains: %w", err)
}
firstLevel := 1
var orgThirdParties coredata.ThirdParties
if err := orgThirdParties.LoadAllByOrganizationID(
ctx,
conn,
scope,
tp.OrganizationID,
coredata.NewThirdPartyFilter(nil, &firstLevel, nil),
); err != nil {
return prep, fmt.Errorf("cannot load org third parties: %w", err)
}
ranked := thirdparty.RankCandidates(prep.commonParty, prep.commonDomains, orgThirdParties)
if len(ranked) > 0 && ranked[0].Score >= thirdparty.HighConfidenceScore {
prep.highConfidence = ranked[0].ThirdParty
prep.highScore = ranked[0].Score
} else {
prep.agentSet = ranked
if len(prep.agentSet) > thirdparty.MaxAgentCandidates {
prep.agentSet = prep.agentSet[:thirdparty.MaxAgentCandidates]
}
for _, c := range prep.agentSet {
if c.Score >= thirdparty.MinAgentScore {
prep.eligibleForAgent = true
break
}
}
}
return prep, nil
}