Promote tracker patterns to org third parties via worker
Manual moves of a non-extension TrackerPattern lacking a ThirdPartyID now request mapping, which the tracker-mapping worker resolves with a four-stage pipeline: exact common_third_party_id link, heuristic ranking, agent disambiguation, and finally CreateFromCommon. Existing fuzzy-matched org rows are tagged with common_third_party_id so the next promotion takes the O(1) exact-link path. The matching primitives live in pkg/thirdparty (RankCandidates, LinkToCommon, CreateFromCommon, ScoredCandidate, threshold constants) so the disambiguation agent and the heuristic share one candidate type. Cookiebanner orchestrates them; cookie-banner-specific concerns (pattern -> common-pattern -> common-party navigation, the EXTENSION gate, and structured logs) stay in the worker. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -2628,6 +2628,20 @@ func (s *Service) MoveTrackerPatternToCategory(
|
||||
return fmt.Errorf("cannot update tracker pattern: %w", err)
|
||||
}
|
||||
|
||||
// A manual move is the user's signal that this is a
|
||||
// real tracker. Enqueue the tracker-mapping worker so
|
||||
// it can promote the pattern to an org ThirdParty (or
|
||||
// link an existing one) — never EXTENSION-sourced
|
||||
// patterns, and never patterns we already promoted.
|
||||
// SetMappingRequested is idempotent: it short-circuits
|
||||
// when mapping_requested_at is already non-NULL.
|
||||
if pattern.ThirdPartyID == nil &&
|
||||
(pattern.Source == nil || *pattern.Source != coredata.CookieSourceExtension) {
|
||||
if err := pattern.SetMappingRequested(ctx, tx); err != nil {
|
||||
return fmt.Errorf("cannot enqueue tracker mapping after move: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var banner coredata.CookieBanner
|
||||
if err := banner.LoadByID(ctx, tx, scope, pattern.CookieBannerID); err != nil {
|
||||
return fmt.Errorf("cannot load cookie banner: %w", err)
|
||||
|
||||
@@ -48,6 +48,9 @@ type TrackerMappingAgentResult struct {
|
||||
Confidence float64 `json:"confidence" jsonschema:"Confidence level from 0.0 to 1.0. Set below 0.5 if unsure."`
|
||||
}
|
||||
|
||||
// TrackerMappingConfig configures the tracker-mapping agent (catalog
|
||||
// identification). The agent uses DB-backed search tools and may also
|
||||
// use Firecrawl for web search when an API key is supplied.
|
||||
type TrackerMappingConfig struct {
|
||||
LLMClient *llm.Client
|
||||
Model string
|
||||
|
||||
@@ -28,18 +28,21 @@ import (
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/llm"
|
||||
"go.probo.inc/probo/pkg/slug"
|
||||
"go.probo.inc/probo/pkg/thirdparty"
|
||||
)
|
||||
|
||||
type trackerMappingHandler struct {
|
||||
pg *pg.Client
|
||||
logger *log.Logger
|
||||
agent *agent.Agent
|
||||
pg *pg.Client
|
||||
logger *log.Logger
|
||||
mappingAgent *agent.Agent
|
||||
disambiguationAgent *agent.Agent
|
||||
}
|
||||
|
||||
func NewTrackerMappingWorker(
|
||||
pgClient *pg.Client,
|
||||
logger *log.Logger,
|
||||
cfg TrackerMappingConfig,
|
||||
mappingCfg TrackerMappingConfig,
|
||||
disambiguationCfg thirdparty.DisambiguationConfig,
|
||||
opts ...worker.Option,
|
||||
) *worker.Worker[coredata.TrackerPattern] {
|
||||
h := &trackerMappingHandler{
|
||||
@@ -47,8 +50,12 @@ func NewTrackerMappingWorker(
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
if cfg.LLMClient != nil {
|
||||
h.agent = buildTrackerMappingAgent(cfg, pgClient, logger)
|
||||
if mappingCfg.LLMClient != nil {
|
||||
h.mappingAgent = buildTrackerMappingAgent(mappingCfg, pgClient, logger)
|
||||
}
|
||||
|
||||
if disambiguationCfg.LLMClient != nil {
|
||||
h.disambiguationAgent = thirdparty.BuildDisambiguationAgent(disambiguationCfg, logger)
|
||||
}
|
||||
|
||||
return worker.New(
|
||||
@@ -82,40 +89,62 @@ func (h *trackerMappingHandler) Claim(ctx context.Context) (coredata.TrackerPatt
|
||||
return tp, nil
|
||||
}
|
||||
|
||||
// Process resolves the catalog mapping (when missing) and then promotes
|
||||
// the pattern to an org ThirdParty (when eligible). When a pattern is
|
||||
// re-triggered by a manual move (it already carries a
|
||||
// common_tracker_pattern_id), we MUST NOT re-resolve the catalog: the
|
||||
// existing link is preserved and we jump straight to third-party
|
||||
// promotion.
|
||||
func (h *trackerMappingHandler) Process(ctx context.Context, tp coredata.TrackerPattern) error {
|
||||
return h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
var (
|
||||
commonPatternID *gid.GID
|
||||
thirdPartyID *gid.GID
|
||||
err error
|
||||
)
|
||||
|
||||
commonPatternID, thirdPartyID, err = h.matchByPattern(ctx, tx, tp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot match by pattern: %w", err)
|
||||
}
|
||||
|
||||
if commonPatternID == nil {
|
||||
commonPatternID, thirdPartyID, err = h.matchByDomain(ctx, tx, tp)
|
||||
if tp.CommonTrackerPatternID != nil {
|
||||
commonPatternID = tp.CommonTrackerPatternID
|
||||
} else {
|
||||
commonPatternID, err = h.matchByPattern(ctx, tx, tp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot match by domain: %w", err)
|
||||
return fmt.Errorf("cannot match by pattern: %w", err)
|
||||
}
|
||||
|
||||
if commonPatternID == nil {
|
||||
commonPatternID, err = h.matchByDomain(ctx, tx, tp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot match by domain: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if commonPatternID == nil && h.mappingAgent != nil {
|
||||
commonPatternID, err = h.identifyWithAgent(ctx, tx, tp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot identify with agent: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if commonPatternID == nil {
|
||||
commonPatternID, err = h.createUnmatchedPattern(ctx, tx, tp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create unmatched pattern: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if commonPatternID == nil && h.agent != nil {
|
||||
commonPatternID, thirdPartyID, err = h.identifyWithAgent(ctx, tx, tp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot identify with agent: %w", err)
|
||||
}
|
||||
}
|
||||
thirdPartyID := tp.ThirdPartyID
|
||||
|
||||
if commonPatternID == nil {
|
||||
commonPatternID, err = h.createUnmatchedPattern(ctx, tx, tp)
|
||||
if thirdPartyID == nil &&
|
||||
commonPatternID != nil &&
|
||||
(tp.Source == nil || *tp.Source != coredata.CookieSourceExtension) {
|
||||
promoted, err := h.promoteThirdParty(ctx, tx, tp, *commonPatternID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot create unmatched pattern: %w", err)
|
||||
return fmt.Errorf("cannot promote third party: %w", err)
|
||||
}
|
||||
|
||||
thirdPartyID = promoted
|
||||
}
|
||||
|
||||
if commonPatternID != nil || thirdPartyID != nil {
|
||||
@@ -136,59 +165,55 @@ func (h *trackerMappingHandler) Process(ctx context.Context, tp coredata.Tracker
|
||||
)
|
||||
}
|
||||
|
||||
// matchByPattern looks for a catalog row with the same pattern. It now
|
||||
// only returns the catalog ID; third-party resolution happens later in
|
||||
// promoteThirdParty.
|
||||
func (h *trackerMappingHandler) matchByPattern(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
tp coredata.TrackerPattern,
|
||||
) (*gid.GID, *gid.GID, error) {
|
||||
) (*gid.GID, error) {
|
||||
var commonPattern coredata.CommonTrackerPattern
|
||||
if err := commonPattern.LoadByPattern(ctx, conn, tp.TrackerType, tp.Pattern, tp.MaxAgeSeconds); err != nil {
|
||||
if errors.Is(err, coredata.ErrResourceNotFound) {
|
||||
return nil, nil, nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return nil, nil, fmt.Errorf("cannot load common tracker pattern: %w", err)
|
||||
return nil, fmt.Errorf("cannot load common tracker pattern: %w", err)
|
||||
}
|
||||
|
||||
var thirdPartyID *gid.GID
|
||||
|
||||
if commonPattern.CommonThirdPartyID != nil {
|
||||
var err error
|
||||
|
||||
thirdPartyID, err = h.resolveThirdParty(ctx, conn, tp, &commonPattern)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot resolve third party from pattern match: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &commonPattern.ID, thirdPartyID, nil
|
||||
return &commonPattern.ID, nil
|
||||
}
|
||||
|
||||
// matchByDomain finds a CommonThirdParty whose registered domains
|
||||
// overlap the pattern's observed initiator domains, and upserts a
|
||||
// CommonTrackerPattern linking the two. As with matchByPattern,
|
||||
// third-party resolution is deferred to promoteThirdParty.
|
||||
func (h *trackerMappingHandler) matchByDomain(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
tp coredata.TrackerPattern,
|
||||
) (*gid.GID, *gid.GID, error) {
|
||||
) (*gid.GID, error) {
|
||||
var trackers coredata.DetectedTrackers
|
||||
|
||||
domains, err := trackers.LoadInitiatorDomainsByTrackerPatternID(ctx, tx, tp.ID, 10)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot load initiator domains: %w", err)
|
||||
return nil, fmt.Errorf("cannot load initiator domains: %w", err)
|
||||
}
|
||||
|
||||
if len(domains) == 0 {
|
||||
return nil, nil, nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
filter := coredata.NewCommonThirdPartyDomainFilter(domains)
|
||||
|
||||
var matchedDomains coredata.CommonThirdPartyDomains
|
||||
if err := matchedDomains.Load(ctx, tx, 1, filter); err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot load common third party domain by domain match: %w", err)
|
||||
return nil, fmt.Errorf("cannot load common third party domain by domain match: %w", err)
|
||||
}
|
||||
|
||||
if len(matchedDomains) == 0 {
|
||||
return nil, nil, nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
commonThirdPartyID := matchedDomains[0].CommonThirdPartyID
|
||||
@@ -208,22 +233,17 @@ func (h *trackerMappingHandler) matchByDomain(
|
||||
}
|
||||
|
||||
if _, err := commonPattern.Upsert(ctx, tx); err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot upsert common tracker pattern from domain match: %w", err)
|
||||
return nil, fmt.Errorf("cannot upsert common tracker pattern from domain match: %w", err)
|
||||
}
|
||||
|
||||
thirdPartyID, err := h.resolveThirdParty(ctx, tx, tp, &commonPattern)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot resolve third party from domain match: %w", err)
|
||||
}
|
||||
|
||||
return &commonPattern.ID, thirdPartyID, nil
|
||||
return &commonPattern.ID, nil
|
||||
}
|
||||
|
||||
func (h *trackerMappingHandler) identifyWithAgent(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
tp coredata.TrackerPattern,
|
||||
) (*gid.GID, *gid.GID, error) {
|
||||
) (*gid.GID, error) {
|
||||
var trackers coredata.DetectedTrackers
|
||||
|
||||
domains, err := trackers.LoadInitiatorDomainsByTrackerPatternID(ctx, tx, tp.ID, 5)
|
||||
@@ -238,7 +258,7 @@ func (h *trackerMappingHandler) identifyWithAgent(
|
||||
|
||||
result, err := agent.RunTyped[TrackerMappingAgentResult](
|
||||
agentCtx,
|
||||
h.agent,
|
||||
h.mappingAgent,
|
||||
[]llm.Message{
|
||||
{
|
||||
Role: llm.RoleUser,
|
||||
@@ -254,7 +274,7 @@ func (h *trackerMappingHandler) identifyWithAgent(
|
||||
log.String("pattern", tp.Pattern),
|
||||
)
|
||||
|
||||
return nil, nil, nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
identification := result.Output
|
||||
@@ -267,7 +287,7 @@ func (h *trackerMappingHandler) identifyWithAgent(
|
||||
log.Float64("confidence", identification.Confidence),
|
||||
)
|
||||
|
||||
return nil, nil, nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
confidence := float32(identification.Confidence)
|
||||
@@ -284,7 +304,7 @@ func (h *trackerMappingHandler) identifyWithAgent(
|
||||
domains,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot resolve or create common third party: %w", err)
|
||||
return nil, fmt.Errorf("cannot resolve or create common third party: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -303,12 +323,7 @@ func (h *trackerMappingHandler) identifyWithAgent(
|
||||
}
|
||||
|
||||
if _, err := commonPattern.Upsert(ctx, tx); err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot upsert common tracker pattern from agent: %w", err)
|
||||
}
|
||||
|
||||
thirdPartyID, err := h.resolveThirdParty(ctx, tx, tp, &commonPattern)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("cannot resolve third party from agent match: %w", err)
|
||||
return nil, fmt.Errorf("cannot upsert common tracker pattern from agent: %w", err)
|
||||
}
|
||||
|
||||
h.logger.InfoCtx(
|
||||
@@ -319,7 +334,7 @@ func (h *trackerMappingHandler) identifyWithAgent(
|
||||
log.Float64("confidence", identification.Confidence),
|
||||
)
|
||||
|
||||
return &commonPattern.ID, thirdPartyID, nil
|
||||
return &commonPattern.ID, nil
|
||||
}
|
||||
|
||||
func (h *trackerMappingHandler) resolveOrCreateCommonThirdParty(
|
||||
@@ -408,32 +423,163 @@ func (h *trackerMappingHandler) createUnmatchedPattern(
|
||||
return &commonPattern.ID, nil
|
||||
}
|
||||
|
||||
func (h *trackerMappingHandler) resolveThirdParty(
|
||||
// promoteThirdParty resolves an org ThirdParty for the given pattern
|
||||
// once the catalog mapping is known. The resolution order is:
|
||||
//
|
||||
// 1. Exact link by common_third_party_id (O(1)).
|
||||
// 2. Heuristic match against the org's existing ThirdParty rows
|
||||
// (lowercased name, suffix-stripped name, slug, website host,
|
||||
// CommonThirdPartyDomain overlap).
|
||||
// 3. Agent disambiguation when the heuristic is ambiguous.
|
||||
// 4. Fallback create from CommonThirdParty.
|
||||
//
|
||||
// A confident heuristic/agent match is auto-tagged with
|
||||
// common_third_party_id so subsequent promotions hit the exact-link
|
||||
// path in O(1). Returns (nil, nil) when the catalog row has no
|
||||
// CommonThirdPartyID — there is nothing to promote to.
|
||||
func (h *trackerMappingHandler) promoteThirdParty(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
tx pg.Tx,
|
||||
tp coredata.TrackerPattern,
|
||||
commonPattern *coredata.CommonTrackerPattern,
|
||||
commonPatternID gid.GID,
|
||||
) (*gid.GID, error) {
|
||||
var commonPattern coredata.CommonTrackerPattern
|
||||
if err := commonPattern.LoadByID(ctx, tx, commonPatternID); err != nil {
|
||||
return nil, fmt.Errorf("cannot load common tracker pattern: %w", err)
|
||||
}
|
||||
|
||||
if commonPattern.CommonThirdPartyID == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
commonThirdPartyID := *commonPattern.CommonThirdPartyID
|
||||
scope := coredata.NewScopeFromObjectID(tp.ID)
|
||||
|
||||
var t coredata.ThirdParty
|
||||
if err := t.LoadByOrganizationIDAndCommonThirdPartyID(
|
||||
var existing coredata.ThirdParty
|
||||
|
||||
err := existing.LoadByOrganizationIDAndCommonThirdPartyID(
|
||||
ctx,
|
||||
conn,
|
||||
tx,
|
||||
scope,
|
||||
tp.OrganizationID,
|
||||
*commonPattern.CommonThirdPartyID,
|
||||
); err != nil {
|
||||
if errors.Is(err, coredata.ErrResourceNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("cannot resolve third party: %w", err)
|
||||
commonThirdPartyID,
|
||||
)
|
||||
if err == nil {
|
||||
return &existing.ID, nil
|
||||
}
|
||||
|
||||
return &t.ID, nil
|
||||
if !errors.Is(err, coredata.ErrResourceNotFound) {
|
||||
return nil, fmt.Errorf("cannot load org third party by common id: %w", err)
|
||||
}
|
||||
|
||||
var commonParty coredata.CommonThirdParty
|
||||
if err := commonParty.LoadByID(ctx, tx, commonThirdPartyID); err != nil {
|
||||
return nil, fmt.Errorf("cannot load common third party: %w", err)
|
||||
}
|
||||
|
||||
var commonDomains coredata.CommonThirdPartyDomains
|
||||
if err := commonDomains.LoadByCommonThirdPartyID(ctx, tx, commonThirdPartyID); err != nil {
|
||||
return nil, fmt.Errorf("cannot load common third party domains: %w", err)
|
||||
}
|
||||
|
||||
var orgThirdParties coredata.ThirdParties
|
||||
if err := orgThirdParties.LoadAllByOrganizationID(ctx, tx, scope, tp.OrganizationID); err != nil {
|
||||
return nil, fmt.Errorf("cannot load org third parties: %w", err)
|
||||
}
|
||||
|
||||
ranked := thirdparty.RankCandidates(commonParty, commonDomains, orgThirdParties)
|
||||
|
||||
if len(ranked) > 0 && ranked[0].Score >= thirdparty.HighConfidenceScore {
|
||||
picked := ranked[0].ThirdParty
|
||||
|
||||
if err := thirdparty.LinkToCommon(ctx, tx, scope, picked, commonThirdPartyID); err != nil {
|
||||
return nil, fmt.Errorf("cannot link fuzzy-matched third party to common: %w", err)
|
||||
}
|
||||
|
||||
h.logger.InfoCtx(
|
||||
ctx,
|
||||
"promoted tracker pattern via heuristic match",
|
||||
log.String("tracker_pattern_id", tp.ID.String()),
|
||||
log.String("third_party_id", picked.ID.String()),
|
||||
log.Float64("score", ranked[0].Score),
|
||||
)
|
||||
|
||||
return &picked.ID, nil
|
||||
}
|
||||
|
||||
agentSet := ranked
|
||||
if len(agentSet) > thirdparty.MaxAgentCandidates {
|
||||
agentSet = agentSet[:thirdparty.MaxAgentCandidates]
|
||||
}
|
||||
|
||||
eligibleForAgent := false
|
||||
|
||||
for _, c := range agentSet {
|
||||
if c.Score >= thirdparty.MinAgentScore {
|
||||
eligibleForAgent = true
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if eligibleForAgent && h.disambiguationAgent != nil {
|
||||
matchedID, err := thirdparty.Disambiguate(
|
||||
ctx,
|
||||
h.disambiguationAgent,
|
||||
h.logger,
|
||||
commonParty,
|
||||
commonDomains,
|
||||
agentSet,
|
||||
)
|
||||
if err != nil {
|
||||
h.logger.WarnCtx(
|
||||
ctx,
|
||||
"third-party disambiguation agent failed",
|
||||
log.Error(err),
|
||||
log.String("tracker_pattern_id", tp.ID.String()),
|
||||
)
|
||||
}
|
||||
|
||||
if matchedID != nil {
|
||||
var picked *coredata.ThirdParty
|
||||
|
||||
for _, c := range agentSet {
|
||||
if c.ThirdParty.ID == *matchedID {
|
||||
picked = c.ThirdParty
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if picked != nil {
|
||||
if err := thirdparty.LinkToCommon(ctx, tx, scope, picked, commonThirdPartyID); err != nil {
|
||||
return nil, fmt.Errorf("cannot link agent-matched third party to common: %w", err)
|
||||
}
|
||||
|
||||
h.logger.InfoCtx(
|
||||
ctx,
|
||||
"promoted tracker pattern via disambiguation agent",
|
||||
log.String("tracker_pattern_id", tp.ID.String()),
|
||||
log.String("third_party_id", picked.ID.String()),
|
||||
)
|
||||
|
||||
return &picked.ID, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
created, err := thirdparty.CreateFromCommon(ctx, tx, scope, tp.OrganizationID, commonParty)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create third party from common: %w", err)
|
||||
}
|
||||
|
||||
h.logger.InfoCtx(
|
||||
ctx,
|
||||
"promoted tracker pattern by creating org third party from catalog",
|
||||
log.String("tracker_pattern_id", tp.ID.String()),
|
||||
log.String("third_party_id", created.ID.String()),
|
||||
log.String("common_third_party_id", commonThirdPartyID.String()),
|
||||
)
|
||||
|
||||
return &created.ID, nil
|
||||
}
|
||||
|
||||
495
pkg/cookiebanner/tracker_mapping_worker_test.go
Normal file
495
pkg/cookiebanner/tracker_mapping_worker_test.go
Normal file
@@ -0,0 +1,495 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package cookiebanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.gearno.de/kit/log"
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
)
|
||||
|
||||
//go:fix inline
|
||||
func ptr[T any](v T) *T { return new(v) }
|
||||
|
||||
// promotionFixture extends workerFixture with a CommonThirdParty and a
|
||||
// CommonTrackerPattern linking the catalog to the test pattern. It is
|
||||
// the minimum scaffolding promoteThirdParty needs to run end-to-end.
|
||||
type promotionFixture struct {
|
||||
workerFixture
|
||||
commonThirdParty coredata.CommonThirdParty
|
||||
commonPatternID gid.GID
|
||||
trackerPattern coredata.TrackerPattern
|
||||
commonThirdPartyID gid.GID
|
||||
}
|
||||
|
||||
func seedPromotionFixture(t *testing.T, ctx context.Context, client *pg.Client) promotionFixture {
|
||||
t.Helper()
|
||||
|
||||
fx := seedWorkerFixture(t, ctx, client)
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
|
||||
commonThirdPartyID := gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType)
|
||||
commonThirdParty := coredata.CommonThirdParty{
|
||||
ID: commonThirdPartyID,
|
||||
Name: "Google",
|
||||
Slug: "google",
|
||||
Category: coredata.ThirdPartyCategoryAnalytics,
|
||||
WebsiteURL: new("https://google.com"),
|
||||
Certifications: []string{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
commonPattern := coredata.CommonTrackerPattern{
|
||||
ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType),
|
||||
CommonThirdPartyID: &commonThirdPartyID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "_ga",
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
Description: "",
|
||||
Confidence: 0.9,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
pattern := coredata.TrackerPattern{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CookieBannerID: fx.banner.ID,
|
||||
CookieCategoryID: fx.normalCategoryID,
|
||||
CommonTrackerPatternID: &commonPattern.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "_ga",
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
DisplayName: "_ga",
|
||||
Description: "",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
if err := commonThirdParty.Insert(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := commonPattern.Upsert(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return pattern.Insert(ctx, tx, fx.scope)
|
||||
}))
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = client.WithTx(context.Background(), func(ctx context.Context, tx pg.Tx) error {
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM common_third_party_domains WHERE common_third_party_id = $1`, commonThirdPartyID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM common_tracker_patterns WHERE id = $1`, commonPattern.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM common_third_parties WHERE id = $1`, commonThirdPartyID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM third_parties WHERE organization_id = $1`, fx.organizationID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
})
|
||||
|
||||
return promotionFixture{
|
||||
workerFixture: fx,
|
||||
commonThirdParty: commonThirdParty,
|
||||
commonPatternID: commonPattern.ID,
|
||||
commonThirdPartyID: commonThirdPartyID,
|
||||
trackerPattern: pattern,
|
||||
}
|
||||
}
|
||||
|
||||
func newMappingHandler(client *pg.Client) *trackerMappingHandler {
|
||||
return &trackerMappingHandler{
|
||||
pg: client,
|
||||
logger: log.NewLogger(log.WithOutput(io.Discard)),
|
||||
}
|
||||
}
|
||||
|
||||
// promote runs promoteThirdParty inside its own transaction so each
|
||||
// test case starts from a clean state.
|
||||
func promote(
|
||||
t *testing.T,
|
||||
ctx context.Context,
|
||||
h *trackerMappingHandler,
|
||||
client *pg.Client,
|
||||
tp coredata.TrackerPattern,
|
||||
commonPatternID gid.GID,
|
||||
) *gid.GID {
|
||||
t.Helper()
|
||||
|
||||
var got *gid.GID
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
var err error
|
||||
|
||||
got, err = h.promoteThirdParty(ctx, tx, tp, commonPatternID)
|
||||
|
||||
return err
|
||||
}))
|
||||
|
||||
return got
|
||||
}
|
||||
|
||||
func TestPromoteThirdParty_ExactCommonLink(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
fx := seedPromotionFixture(t, ctx, client)
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
existing := coredata.ThirdParty{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.ThirdPartyEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CommonThirdPartyID: &fx.commonThirdPartyID,
|
||||
Name: "Google LLC",
|
||||
Category: coredata.ThirdPartyCategoryAnalytics,
|
||||
Certifications: []string{},
|
||||
Countries: coredata.CountryCodes{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
return existing.Insert(ctx, tx, fx.scope)
|
||||
}))
|
||||
|
||||
got := promote(t, ctx, newMappingHandler(client), client, fx.trackerPattern, fx.commonPatternID)
|
||||
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, existing.ID, *got, "should return the existing org ThirdParty linked by common id")
|
||||
}
|
||||
|
||||
func TestPromoteThirdParty_HeuristicMatch(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
fx := seedPromotionFixture(t, ctx, client)
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
manualEntry := coredata.ThirdParty{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.ThirdPartyEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
Name: "Google LLC",
|
||||
Category: coredata.ThirdPartyCategoryAnalytics,
|
||||
Certifications: []string{},
|
||||
Countries: coredata.CountryCodes{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
return manualEntry.Insert(ctx, tx, fx.scope)
|
||||
}))
|
||||
|
||||
got := promote(t, ctx, newMappingHandler(client), client, fx.trackerPattern, fx.commonPatternID)
|
||||
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, manualEntry.ID, *got, "heuristic match should return the manually-entered ThirdParty")
|
||||
|
||||
var reloaded coredata.ThirdParty
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return reloaded.LoadByID(ctx, conn, fx.scope, manualEntry.ID)
|
||||
}))
|
||||
|
||||
require.NotNil(t, reloaded.CommonThirdPartyID, "matched row must be tagged with common_third_party_id")
|
||||
assert.Equal(t, fx.commonThirdPartyID, *reloaded.CommonThirdPartyID)
|
||||
}
|
||||
|
||||
func TestPromoteThirdParty_FallbackCreate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
fx := seedPromotionFixture(t, ctx, client)
|
||||
|
||||
got := promote(t, ctx, newMappingHandler(client), client, fx.trackerPattern, fx.commonPatternID)
|
||||
|
||||
require.NotNil(t, got, "fallback should create a new ThirdParty")
|
||||
|
||||
var reloaded coredata.ThirdParty
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return reloaded.LoadByID(ctx, conn, fx.scope, *got)
|
||||
}))
|
||||
|
||||
assert.Equal(t, fx.organizationID, reloaded.OrganizationID)
|
||||
assert.Equal(t, "Google", reloaded.Name)
|
||||
require.NotNil(t, reloaded.CommonThirdPartyID)
|
||||
assert.Equal(t, fx.commonThirdPartyID, *reloaded.CommonThirdPartyID)
|
||||
assert.Equal(t, coredata.ThirdPartyCategoryAnalytics, reloaded.Category)
|
||||
assert.False(t, reloaded.FirstLevel)
|
||||
assert.False(t, reloaded.ShowOnTrustCenter)
|
||||
}
|
||||
|
||||
func TestPromoteThirdParty_NoCommonThirdPartyOnPattern(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
fx := seedWorkerFixture(t, ctx, client)
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
commonPattern := coredata.CommonTrackerPattern{
|
||||
ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType),
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "unknown_xyz",
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
Description: "",
|
||||
Confidence: 0.5,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
pattern := coredata.TrackerPattern{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CookieBannerID: fx.banner.ID,
|
||||
CookieCategoryID: fx.normalCategoryID,
|
||||
CommonTrackerPatternID: &commonPattern.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "unknown_xyz",
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
DisplayName: "unknown_xyz",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
if _, err := commonPattern.Upsert(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return pattern.Insert(ctx, tx, fx.scope)
|
||||
}))
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = client.WithTx(context.Background(), func(ctx context.Context, tx pg.Tx) error {
|
||||
_, err := tx.Exec(ctx, `DELETE FROM common_tracker_patterns WHERE id = $1`, commonPattern.ID)
|
||||
|
||||
return err
|
||||
})
|
||||
})
|
||||
|
||||
got := promote(t, ctx, newMappingHandler(client), client, pattern, commonPattern.ID)
|
||||
|
||||
assert.Nil(t, got, "patterns whose catalog row has no CommonThirdPartyID should not be promoted")
|
||||
}
|
||||
|
||||
// TestProcess_PreservesCatalogMappingOnReTrigger asserts that when
|
||||
// Process is called for a pattern that already carries a
|
||||
// common_tracker_pattern_id, the catalog pipeline is skipped and the
|
||||
// existing catalog link is preserved verbatim.
|
||||
func TestProcess_PreservesCatalogMappingOnReTrigger(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
fx := seedPromotionFixture(t, ctx, client)
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
return fx.trackerPattern.SetMappingRequested(ctx, tx)
|
||||
}))
|
||||
|
||||
h := newMappingHandler(client)
|
||||
require.NoError(t, h.Process(ctx, fx.trackerPattern))
|
||||
|
||||
var reloaded coredata.TrackerPattern
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return reloaded.LoadByID(ctx, conn, fx.scope, fx.trackerPattern.ID)
|
||||
}))
|
||||
|
||||
require.NotNil(t, reloaded.CommonTrackerPatternID, "common tracker pattern link must be preserved")
|
||||
assert.Equal(t, fx.commonPatternID, *reloaded.CommonTrackerPatternID)
|
||||
require.NotNil(t, reloaded.ThirdPartyID, "the worker should have promoted to an org ThirdParty")
|
||||
}
|
||||
|
||||
// TestProcess_ExtensionPatternIsNotPromoted asserts that even when a
|
||||
// pattern has a catalog link, a Source=EXTENSION pattern stays
|
||||
// un-promoted.
|
||||
func TestProcess_ExtensionPatternIsNotPromoted(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
fx := seedPromotionFixture(t, ctx, client)
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
source := coredata.CookieSourceExtension
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
_, err := tx.Exec(
|
||||
ctx,
|
||||
`UPDATE tracker_patterns
|
||||
SET source = $1,
|
||||
mapping_requested_at = $2
|
||||
WHERE id = $3`,
|
||||
source,
|
||||
now,
|
||||
fx.trackerPattern.ID,
|
||||
)
|
||||
|
||||
return err
|
||||
}))
|
||||
|
||||
var reloadedBefore coredata.TrackerPattern
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return reloadedBefore.LoadByID(ctx, conn, fx.scope, fx.trackerPattern.ID)
|
||||
}))
|
||||
|
||||
h := newMappingHandler(client)
|
||||
require.NoError(t, h.Process(ctx, reloadedBefore))
|
||||
|
||||
var reloaded coredata.TrackerPattern
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return reloaded.LoadByID(ctx, conn, fx.scope, fx.trackerPattern.ID)
|
||||
}))
|
||||
|
||||
assert.Nil(t, reloaded.ThirdPartyID, "EXTENSION-sourced pattern must not be promoted")
|
||||
}
|
||||
|
||||
// TestProcess_NoOpWhenAlreadyPromoted asserts that re-running the
|
||||
// worker on a pattern that already has a third_party_id leaves the
|
||||
// row alone (the guard in Process).
|
||||
func TestProcess_NoOpWhenAlreadyPromoted(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
fx := seedPromotionFixture(t, ctx, client)
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
preExisting := coredata.ThirdParty{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.ThirdPartyEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CommonThirdPartyID: &fx.commonThirdPartyID,
|
||||
Name: "Google",
|
||||
Category: coredata.ThirdPartyCategoryAnalytics,
|
||||
Certifications: []string{},
|
||||
Countries: coredata.CountryCodes{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
if err := preExisting.Insert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fx.trackerPattern.ThirdPartyID = &preExisting.ID
|
||||
|
||||
_, err := tx.Exec(
|
||||
ctx,
|
||||
`UPDATE tracker_patterns
|
||||
SET third_party_id = $1,
|
||||
mapping_requested_at = $2
|
||||
WHERE id = $3`,
|
||||
preExisting.ID,
|
||||
now,
|
||||
fx.trackerPattern.ID,
|
||||
)
|
||||
|
||||
return err
|
||||
}))
|
||||
|
||||
var reloadedBefore coredata.TrackerPattern
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return reloadedBefore.LoadByID(ctx, conn, fx.scope, fx.trackerPattern.ID)
|
||||
}))
|
||||
|
||||
h := newMappingHandler(client)
|
||||
require.NoError(t, h.Process(ctx, reloadedBefore))
|
||||
|
||||
var reloaded coredata.TrackerPattern
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return reloaded.LoadByID(ctx, conn, fx.scope, fx.trackerPattern.ID)
|
||||
}))
|
||||
|
||||
require.NotNil(t, reloaded.ThirdPartyID)
|
||||
assert.Equal(t, preExisting.ID, *reloaded.ThirdPartyID, "third_party_id must not be overwritten")
|
||||
}
|
||||
|
||||
func TestPromoteThirdParty_ExactCommonLinkIgnoresSimilarUnlinked(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
fx := seedPromotionFixture(t, ctx, client)
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
|
||||
manualEntry := coredata.ThirdParty{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.ThirdPartyEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
Name: "Google LLC",
|
||||
Category: coredata.ThirdPartyCategoryAnalytics,
|
||||
Certifications: []string{},
|
||||
Countries: coredata.CountryCodes{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
linked := coredata.ThirdParty{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.ThirdPartyEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CommonThirdPartyID: &fx.commonThirdPartyID,
|
||||
Name: "Google",
|
||||
Category: coredata.ThirdPartyCategoryAnalytics,
|
||||
Certifications: []string{},
|
||||
Countries: coredata.CountryCodes{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
if err := manualEntry.Insert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return linked.Insert(ctx, tx, fx.scope)
|
||||
}))
|
||||
|
||||
got := promote(t, ctx, newMappingHandler(client), client, fx.trackerPattern, fx.commonPatternID)
|
||||
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, linked.ID, *got, "exact-link path must short-circuit before the heuristic fires")
|
||||
}
|
||||
Reference in New Issue
Block a user