diff --git a/.cursor/rules/coredata-migrations.mdc b/.cursor/rules/coredata-migrations.mdc new file mode 100644 index 000000000..9c8aea3ce --- /dev/null +++ b/.cursor/rules/coredata-migrations.mdc @@ -0,0 +1,36 @@ +--- +description: Coredata SQL migration naming and conventions +globs: pkg/coredata/migrations/*.sql +alwaysApply: false +--- + +# Coredata migrations + +See full guides: `contrib/claude/coredata.md` (Migrations) and +`contrib/claude/license.md` (SQL header). + +## Naming — never invent or infer the timestamp + +Migration files use UTC timestamp naming `YYYYMMDDTHHMMSSZ.sql`. The +timestamp is the **real current time**, obtained by running the command — +do NOT hand-write a round number (e.g. `...T100000Z`) and do NOT infer the +next name from existing files. + +```bash +# Run this and use its exact output as the filename: +date -u +"%Y%m%dT%H%M%SZ.sql" +``` + +## License header + +Every `.sql` file starts with the ISC header (use the current year). Copy +the exact block from `contrib/claude/license.md` (SQL section). + +## Content rules + +- One logical change per file. +- No indexes by default — add one only when justified by observed + production latency. Constraint-enforcing indexes (e.g. unique) are exempt. +- Avoid `DEFAULT` clauses. When adding a non-nullable column to an existing + table, set a `DEFAULT` to backfill existing rows, then `DROP DEFAULT` in + the same migration so inserts must supply the value explicitly. diff --git a/pkg/cookiebanner/prompts/tracker_identification.txt.tmpl b/pkg/cookiebanner/prompts/tracker_identification.txt.tmpl index d07a74260..942af99f2 100644 --- a/pkg/cookiebanner/prompts/tracker_identification.txt.tmpl +++ b/pkg/cookiebanner/prompts/tracker_identification.txt.tmpl @@ -9,10 +9,12 @@ Return a structured JSON response with: - third_party_name: the canonical company/service name (e.g. "Google Analytics", not "google" or "GA") - category: the business category of the third party - third_party_confidence: how confident you are about WHICH company or service set this tracker (0.0 to 1.0). This is the confidence in the attribution alone — not a judgment of whether the artifact is a "real" web tracker. A browser-extension artifact whose name names its vendor can still warrant high confidence here. +- evidence_source: the concrete evidence that backs the attribution — one of "database_match" (an exact pattern match in the internal database), "naming_convention" (the tracker's meaningful prefix or an embedded vendor name), "web_search" (a web result that names the setter), "browser_page" (a page you opened that names the setter), or "none" (no concrete evidence). It MUST be "none" whenever third_party_name is empty, and it MUST be a concrete kind (not "none") whenever you name a vendor — an attribution with evidence_source "none" is discarded. +- is_first_party: true when the artifact has no third party at all — it is the scanned site's own tracker, a generic library or log key (e.g. "loglevel", "debug"), a browser-extension key that embeds the scanned site's origin, or otherwise not attributable to any external vendor. Leave it false whenever a vendor is or might be responsible. -1. First use the search_tracker_patterns tool to look for similar known patterns in the database. Strip variable parts (IDs, UUIDs, timestamps) from the pattern name and search for the fixed prefix or root (e.g. for "_gat_UA-12345678-1", search for "_gat"). +1. First use the search_tracker_patterns tool to look for similar known patterns in the database. Strip variable parts (IDs, UUIDs, timestamps) from the pattern name and search for the fixed prefix or root (e.g. for "_gat_UA-12345678-1", search for "_gat"). Weigh the results by their confidence: a high-confidence match (a curated/seed entry) is authoritative, but a lower-confidence match is only a hint from an earlier automatic guess — corroborate it with the naming convention or a web/browser result before relying on it, and never treat it as proof on its own. Only count this as evidence_source "database_match" for an exact, high-confidence match on the same pattern. 2. If search_tracker_patterns returns results with a third party name, use the search_third_parties tool to confirm the exact name in the database and get its category. @@ -56,6 +58,10 @@ Return a structured JSON response with: 10. Only attribute a tracker to a company or service when you have concrete evidence: an exact (perfect) match on the pattern in the database, an unmistakable naming convention where the tracker's meaningful prefix belongs to that vendor (including a vendor name embedded in the key), or a clear web search result whose tracker name shares that meaningful prefix. Absent a shared meaningful prefix or a perfect pattern match, do NOT imagine a vendor — never guess or invent attributions based on vague similarity, a shared generic word, or general knowledge. If no evidence supports a match, return an empty third_party_name with third_party_confidence below 0.3. -11. For the category field, use one of: {{.Categories}}. +11. Always set evidence_source to the kind of evidence you actually used for the attribution, and only to a concrete kind ("database_match", "naming_convention", "web_search", or "browser_page") when that evidence genuinely exists. If you are tempted to name a vendor from general knowledge or a vague resemblance with no concrete evidence, set evidence_source to "none" and leave third_party_name empty instead — an attribution carrying evidence_source "none" is rejected. + +12. Set is_first_party to true when the artifact plainly has no external vendor behind it. This covers: the scanned site's own trackers; generic developer/library keys that any application can write (e.g. "loglevel", "debug", "redux", framework-internal keys); a browser-extension key that embeds the scanned site's own origin (e.g. "ethereum-https://"); and any key whose only plausible owner is the first party. A true value is a terminal verdict — the tracker will never be re-examined for a vendor — so only set it when you are confident no third party is responsible; when a vendor is or might be responsible, leave it false and attribute (or leave undetermined) instead. + +13. For the category field, use one of: {{.Categories}}. Most cookies fall under ANALYTICS or MARKETING. diff --git a/pkg/cookiebanner/tracker_mapping_agent.go b/pkg/cookiebanner/tracker_mapping_agent.go index 4028dc30b..6cc441118 100644 --- a/pkg/cookiebanner/tracker_mapping_agent.go +++ b/pkg/cookiebanner/tracker_mapping_agent.go @@ -62,17 +62,59 @@ const ( // rather than the pattern, so the stored row confidence is a // constant like the other heuristic signals (domain, sibling). agentSourceConfidence = 0.8 + + // trustedAttributionConfidence is the bar a catalog row must meet for + // its third party to be adopted deterministically by another pattern + // (the existing-link and matchByPattern paths). Only curated/seed rows + // and operator links (confidence 1.0) clear it; agent (0.8) and + // domain/sibling (0.7) attributions do not, so a single low-confidence + // guess never becomes an authoritative precedent that auto-propagates + // across organizations. Such rows are reused as hints only: the + // pattern falls through to the evidence-guarded agent, which can + // corroborate the guess (promoting the row to this tier) or override + // it. + trustedAttributionConfidence float32 = 0.9 ) //go:embed prompts/tracker_identification.txt.tmpl var trackerIdentificationPrompt string +// Tracker-mapping evidence kinds. The agent must report which concrete +// evidence backs a vendor attribution; an attribution without one of the +// substantive kinds (i.e. "none" or empty) is discarded so the agent +// never attributes a vendor from general knowledge or vague similarity. +const ( + evidenceSourceDatabaseMatch = "database_match" + evidenceSourceNamingConvention = "naming_convention" + evidenceSourceWebSearch = "web_search" + evidenceSourceBrowserPage = "browser_page" + evidenceSourceNone = "none" +) + // TrackerMappingAgentResult is the structured output the tracker-mapping // agent returns. type TrackerMappingAgentResult struct { ThirdPartyName string `json:"third_party_name" jsonschema:"Name of the company or service that sets this tracker (e.g. 'Google Analytics', 'Meta Pixel'). Empty string if truly unknown."` Category coredata.ThirdPartyCategory `json:"category" jsonschema:"Third party category"` ThirdPartyConfidence float64 `json:"third_party_confidence" jsonschema:"Confidence (0.0 to 1.0) in which company or service set this tracker, independent of whether the artifact is a classic web tracker. Set below 0.5 if unsure who set it."` + EvidenceSource string `json:"evidence_source" jsonschema:"The concrete evidence that backs the attribution: 'database_match' (exact pattern in the database), 'naming_convention' (the tracker's meaningful prefix or an embedded vendor name), 'web_search' (a web result naming the setter), 'browser_page' (a page you opened that names the setter), or 'none' when there is no concrete evidence. Must be 'none' whenever third_party_name is empty."` + IsFirstParty bool `json:"is_first_party" jsonschema:"True when the artifact has no third party at all: it is the scanned site's own tracker, a generic library or log key (e.g. 'loglevel'), a browser-extension key that embeds the scanned site's origin, or otherwise not attributable to any external vendor. Leave false when a vendor is or might be responsible."` +} + +// evidenceSupportsAttribution reports whether the agent supplied a +// concrete evidence kind for a vendor attribution. An empty value or +// "none" (or any unrecognized value) does not support an attribution. +func evidenceSupportsAttribution(evidenceSource string) bool { + switch evidenceSource { + case + evidenceSourceDatabaseMatch, + evidenceSourceNamingConvention, + evidenceSourceWebSearch, + evidenceSourceBrowserPage: + return true + } + + return false } // buildTrackerMappingAgent builds the tracker-mapping agent. extraTools diff --git a/pkg/cookiebanner/tracker_mapping_agent_test.go b/pkg/cookiebanner/tracker_mapping_agent_test.go index aea3ce8c0..4d1b146d5 100644 --- a/pkg/cookiebanner/tracker_mapping_agent_test.go +++ b/pkg/cookiebanner/tracker_mapping_agent_test.go @@ -15,11 +15,14 @@ package cookiebanner import ( + "context" "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/gid" ) func TestNameMatchesSiteDomain(t *testing.T) { @@ -130,6 +133,204 @@ func TestNameIsCookieDatabaseAggregator(t *testing.T) { } } +func TestEvidenceSupportsAttribution(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + evidence string + expected bool + }{ + {name: "database match supports", evidence: evidenceSourceDatabaseMatch, expected: true}, + {name: "naming convention supports", evidence: evidenceSourceNamingConvention, expected: true}, + {name: "web search supports", evidence: evidenceSourceWebSearch, expected: true}, + {name: "browser page supports", evidence: evidenceSourceBrowserPage, expected: true}, + {name: "none does not support", evidence: evidenceSourceNone, expected: false}, + {name: "empty does not support", evidence: "", expected: false}, + {name: "unknown value does not support", evidence: "vibes", expected: false}, + } + + for _, tt := range tests { + t.Run( + tt.name, + func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.expected, evidenceSupportsAttribution(tt.evidence)) + }, + ) + } +} + +func TestInterpretCatalogRow(t *testing.T) { + t.Parallel() + + vendorID := gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType) + + t.Run( + "first party verdict is terminal", + func(t *testing.T) { + t.Parallel() + + adopt, untrusted, firstParty := interpretCatalogRow(coredata.CommonTrackerPattern{ + CommonThirdPartyID: &vendorID, + Confidence: 1, + Attribution: coredata.CommonTrackerPatternAttributionFirstParty, + }) + + assert.Nil(t, adopt) + assert.Nil(t, untrusted) + assert.True(t, firstParty) + }, + ) + + t.Run( + "trusted vendor is adopted", + func(t *testing.T) { + t.Parallel() + + adopt, untrusted, firstParty := interpretCatalogRow(coredata.CommonTrackerPattern{ + CommonThirdPartyID: &vendorID, + Confidence: trustedAttributionConfidence, + Attribution: coredata.CommonTrackerPatternAttributionThirdParty, + }) + + require.NotNil(t, adopt) + assert.Equal(t, vendorID, *adopt) + assert.Nil(t, untrusted) + assert.False(t, firstParty) + }, + ) + + t.Run( + "low-confidence vendor is untrusted, not adopted", + func(t *testing.T) { + t.Parallel() + + adopt, untrusted, firstParty := interpretCatalogRow(coredata.CommonTrackerPattern{ + CommonThirdPartyID: &vendorID, + Confidence: agentSourceConfidence, + Attribution: coredata.CommonTrackerPatternAttributionThirdParty, + }) + + assert.Nil(t, adopt) + require.NotNil(t, untrusted) + assert.Equal(t, vendorID, *untrusted) + assert.False(t, firstParty) + }, + ) + + t.Run( + "unlinked row yields nothing", + func(t *testing.T) { + t.Parallel() + + adopt, untrusted, firstParty := interpretCatalogRow(coredata.CommonTrackerPattern{ + Confidence: 0.5, + Attribution: coredata.CommonTrackerPatternAttributionUndetermined, + }) + + assert.Nil(t, adopt) + assert.Nil(t, untrusted) + assert.False(t, firstParty) + }, + ) +} + +func TestIsPreExistingSource(t *testing.T) { + t.Parallel() + + preExisting := coredata.CookieSourcePreExisting + script := coredata.CookieSourceScript + + assert.True(t, isPreExistingSource(coredata.TrackerPattern{Source: &preExisting})) + assert.False(t, isPreExistingSource(coredata.TrackerPattern{Source: &script})) + assert.False(t, isPreExistingSource(coredata.TrackerPattern{Source: nil})) +} + +func TestVendorAttributionRejected(t *testing.T) { + t.Parallel() + + h := newMappingHandler(nil) + ctx := context.Background() + tp := coredata.TrackerPattern{Pattern: "_x", TrackerType: coredata.TrackerTypeCookie} + + confident := func(mut func(*TrackerMappingAgentResult)) TrackerMappingAgentResult { + r := TrackerMappingAgentResult{ + ThirdPartyName: "Acme Analytics", + Category: coredata.ThirdPartyCategoryAnalytics, + ThirdPartyConfidence: 0.9, + EvidenceSource: evidenceSourceNamingConvention, + } + if mut != nil { + mut(&r) + } + + return r + } + + t.Run( + "accepts a confident, evidence-backed attribution", + func(t *testing.T) { + t.Parallel() + assert.False(t, h.vendorAttributionRejected(ctx, tp, confident(nil), "https://example.com")) + }, + ) + + t.Run( + "rejects below confidence threshold", + func(t *testing.T) { + t.Parallel() + r := confident(func(r *TrackerMappingAgentResult) { r.ThirdPartyConfidence = 0.3 }) + assert.True(t, h.vendorAttributionRejected(ctx, tp, r, "https://example.com")) + }, + ) + + t.Run( + "rejects empty name", + func(t *testing.T) { + t.Parallel() + r := confident(func(r *TrackerMappingAgentResult) { r.ThirdPartyName = "" }) + assert.True(t, h.vendorAttributionRejected(ctx, tp, r, "https://example.com")) + }, + ) + + t.Run( + "rejects when evidence source is none", + func(t *testing.T) { + t.Parallel() + r := confident(func(r *TrackerMappingAgentResult) { r.EvidenceSource = evidenceSourceNone }) + assert.True(t, h.vendorAttributionRejected(ctx, tp, r, "https://example.com")) + }, + ) + + t.Run( + "rejects when evidence source is empty", + func(t *testing.T) { + t.Parallel() + r := confident(func(r *TrackerMappingAgentResult) { r.EvidenceSource = "" }) + assert.True(t, h.vendorAttributionRejected(ctx, tp, r, "https://example.com")) + }, + ) + + t.Run( + "rejects when name matches scanned site", + func(t *testing.T) { + t.Parallel() + r := confident(func(r *TrackerMappingAgentResult) { r.ThirdPartyName = "Example" }) + assert.True(t, h.vendorAttributionRejected(ctx, tp, r, "https://example.com")) + }, + ) + + t.Run( + "rejects cookie-database aggregator", + func(t *testing.T) { + t.Parallel() + r := confident(func(r *TrackerMappingAgentResult) { r.ThirdPartyName = "Cookiepedia" }) + assert.True(t, h.vendorAttributionRejected(ctx, tp, r, "https://example.com")) + }, + ) +} + func TestBuildAgentPrompt(t *testing.T) { t.Parallel() diff --git a/pkg/cookiebanner/tracker_mapping_worker.go b/pkg/cookiebanner/tracker_mapping_worker.go index e792fb36a..47245a82b 100644 --- a/pkg/cookiebanner/tracker_mapping_worker.go +++ b/pkg/cookiebanner/tracker_mapping_worker.go @@ -136,12 +136,40 @@ func (h *trackerMappingHandler) RecoverStale(ctx context.Context) error { // is the catalog row the signal resolved (or backfilled); commonThirdPartyID // is the catalog third party the signal discovered, when any; thirdPartyID // is an existing org ThirdParty the signal knows directly (e.g. a sibling -// pattern already promoted in the same organization). A nil *catalogMatch -// means the signal produced nothing. +// pattern already promoted in the same organization). firstParty is set +// when the resolved catalog row carries the terminal FIRST_PARTY verdict. +// untrustedThirdPartyID carries a vendor that was present on the resolved +// row but not adopted because its confidence fell below +// trustedAttributionConfidence; it lets the agent corroborate the prior +// guess. A nil *catalogMatch means the signal produced nothing. type catalogMatch struct { - commonPatternID *gid.GID - commonThirdPartyID *gid.GID - thirdPartyID *gid.GID + commonPatternID *gid.GID + commonThirdPartyID *gid.GID + thirdPartyID *gid.GID + untrustedThirdPartyID *gid.GID + firstParty bool +} + +// interpretCatalogRow maps a resolved catalog row onto the mapping +// pipeline's adoption rules. A FIRST_PARTY row is terminal. A vendor is +// adopted only when the row clears trustedAttributionConfidence; +// otherwise the vendor is surfaced as untrusted so the agent can +// corroborate it rather than the pipeline inheriting a low-confidence +// precedent. +func interpretCatalogRow(cp coredata.CommonTrackerPattern) (adopt *gid.GID, untrusted *gid.GID, firstParty bool) { + if cp.Attribution == coredata.CommonTrackerPatternAttributionFirstParty { + return nil, nil, true + } + + if cp.CommonThirdPartyID == nil { + return nil, nil, false + } + + if cp.Confidence >= trustedAttributionConfidence { + return cp.CommonThirdPartyID, nil, false + } + + return nil, cp.CommonThirdPartyID, false } // Process resolves the catalog mapping for a tracker pattern and links it @@ -191,8 +219,15 @@ func (h *trackerMappingHandler) Process(ctx context.Context, tp coredata.Tracker // Phase 2: tracker-mapping agent (no transaction). It runs only when // the deterministic signals could not resolve a catalog third party. // The LLM and web-search calls happen outside any transaction; the - // result is persisted in its own short transaction. - if commonThirdPartyID == nil && h.mappingEnabled { + // result is persisted in its own short transaction. Patterns whose + // source is PRE_EXISTING are skipped: that source is the low-signal + // catch-all (storage enumerated at SDK init, which bundles extension + // state and prior-session artifacts), so a speculative agent run on it + // is more likely to invent a vendor than to find a real one. The + // deterministic catalog match still applies above, so a known cookie + // still maps; and a later SCRIPT/EXTENSION detection upgrades the + // source and re-arms mapping, giving the agent a better-grounded run. + if commonThirdPartyID == nil && h.mappingEnabled && !det.firstParty && !isPreExistingSource(tp) { ident, err := h.identifyWithAgent(ctx, tp, det.origin) if err != nil { return fmt.Errorf("cannot identify with agent: %w", err) @@ -202,7 +237,13 @@ func (h *trackerMappingHandler) Process(ctx context.Context, tp coredata.Tracker if err := h.pg.WithTx( ctx, func(ctx context.Context, tx pg.Tx) error { - match, err := h.persistAgentIdentification(ctx, tx, tp, *ident) + var match *catalogMatch + + if ident.firstParty { + match, err = h.persistFirstPartyVerdict(ctx, tx, tp) + } else { + match, err = h.persistAgentIdentification(ctx, tx, tp, *ident, det.untrustedThirdPartyID) + } if err != nil { return err } @@ -324,8 +365,10 @@ type deterministicResult struct { commonPatternID *gid.GID commonThirdPartyID *gid.GID directThirdPartyID *gid.GID + untrustedThirdPartyID *gid.GID domains []string commonThirdPartyPreexisted bool + firstParty bool } // resolveDeterministic runs the catalog signals that need no network @@ -357,7 +400,7 @@ func (h *trackerMappingHandler) resolveDeterministic( return res, fmt.Errorf("cannot load linked common tracker pattern: %w", err) } - res.commonThirdPartyID = commonPattern.CommonThirdPartyID + res.commonThirdPartyID, res.untrustedThirdPartyID, res.firstParty = interpretCatalogRow(commonPattern) } else { match, err := h.matchByPattern(ctx, tx, tp) if err != nil { @@ -367,9 +410,18 @@ func (h *trackerMappingHandler) resolveDeterministic( if match != nil { res.commonPatternID = match.commonPatternID res.commonThirdPartyID = match.commonThirdPartyID + res.untrustedThirdPartyID = match.untrustedThirdPartyID + res.firstParty = match.firstParty } } + // A terminal FIRST_PARTY verdict short-circuits every remaining + // signal: the artifact has no third party, so neither the heuristic + // matches nor the agent should run, and no org party is linked. + if res.firstParty { + return res, nil + } + res.commonThirdPartyPreexisted = res.commonThirdPartyID != nil if res.commonThirdPartyID != nil { @@ -467,6 +519,15 @@ func (h *trackerMappingHandler) reenqueueUnmappedSiblings( return nil } +// isPreExistingSource reports whether the org tracker pattern's source is +// PRE_EXISTING. That source is the low-signal catch-all enumerated from +// storage at SDK init (it bundles browser-extension state and +// prior-session artifacts), so the speculative mapping agent is not run +// for it; the deterministic catalog signals still apply. +func isPreExistingSource(tp coredata.TrackerPattern) bool { + return tp.Source != nil && *tp.Source == coredata.CookieSourcePreExisting +} + // firstNonNil returns a when it is set, otherwise b. It keeps the first // catalog row id resolved by the pipeline stable: later signals upsert // the same row (same key) and return the same id, but the explicit guard @@ -518,9 +579,13 @@ func (h *trackerMappingHandler) matchByPattern( return nil, fmt.Errorf("cannot load common tracker pattern: %w", err) } + adopt, untrusted, firstParty := interpretCatalogRow(commonPattern) + return &catalogMatch{ - commonPatternID: &commonPattern.ID, - commonThirdPartyID: commonPattern.CommonThirdPartyID, + commonPatternID: &commonPattern.ID, + commonThirdPartyID: adopt, + untrustedThirdPartyID: untrusted, + firstParty: firstParty, }, nil } @@ -567,6 +632,7 @@ func (h *trackerMappingHandler) matchByDomain( MatchType: tp.MatchType, MaxAgeSeconds: tp.MaxAgeSeconds, Confidence: 0.7, + Attribution: coredata.CommonTrackerPatternAttributionThirdParty, CreatedAt: now, UpdatedAt: now, } @@ -581,10 +647,14 @@ func (h *trackerMappingHandler) matchByDomain( }, nil } -// agentIdentification carries a confident tracker-mapping agent result -// from the no-tx agent phase to the short transaction that persists it. +// agentIdentification carries a tracker-mapping agent verdict from the +// no-tx agent phase to the short transaction that persists it. Exactly +// one outcome is meaningful: firstParty set means the agent declared a +// terminal no-third-party verdict; otherwise result holds a defensible +// vendor attribution. type agentIdentification struct { - result TrackerMappingAgentResult + result TrackerMappingAgentResult + firstParty bool } // identifyWithAgent runs the tracker-mapping agent outside any @@ -664,10 +734,43 @@ func (h *trackerMappingHandler) identifyWithAgent( identification := result.Output + // A defensible vendor attribution wins: record it for the catalog. + if !h.vendorAttributionRejected(ctx, tp, identification, siteOrigin) { + return &agentIdentification{result: identification}, nil + } + + // No defensible vendor. An explicit first-party declaration is a + // terminal verdict: persist it so the pipeline stops retrying this + // artifact. Otherwise leave the pattern undetermined for a later, + // better-informed attempt (the unmatched fallback records it with no + // third party). + if identification.IsFirstParty { + h.logger.InfoCtx( + ctx, + "agent declared tracker first-party", + log.String("pattern", tp.Pattern), + ) + + return &agentIdentification{firstParty: true}, nil + } + + return nil, nil +} + +// vendorAttributionRejected reports whether the agent's vendor +// attribution must be discarded, logging the reason. It enforces, in +// order: a confident attribution, a concrete evidence source (no +// general-knowledge guesses), the scanned-site backstop, and the +// cookie-database-aggregator backstop. +func (h *trackerMappingHandler) vendorAttributionRejected( + ctx context.Context, + tp coredata.TrackerPattern, + identification TrackerMappingAgentResult, + siteOrigin string, +) bool { // The agent's confidence gauges the attribution (who set the // tracker), not whether the artifact is a meaningful tracker. Without - // a confident vendor there is nothing to catalog here; the unmatched - // fallback records the pattern with no third party instead. + // a confident vendor there is nothing to catalog here. if identification.ThirdPartyName == "" || identification.ThirdPartyConfidence < agentThirdPartyConfidenceThreshold { h.logger.InfoCtx( ctx, @@ -676,7 +779,23 @@ func (h *trackerMappingHandler) identifyWithAgent( log.Float64("third_party_confidence", identification.ThirdPartyConfidence), ) - return nil, nil + return true + } + + // Evidence guard: a vendor is attributed only on concrete evidence (a + // database match, a meaningful naming convention, or a web/browser + // result that names the setter). An attribution with no evidence + // source is a general-knowledge guess and is discarded, so a wrong + // precedent never enters the catalog. + if !evidenceSupportsAttribution(identification.EvidenceSource) { + h.logger.InfoCtx( + ctx, + "agent attribution lacks concrete evidence, discarding", + log.String("pattern", tp.Pattern), + log.String("evidence_source", identification.EvidenceSource), + ) + + return true } // Backstop for the prompt rule that the scanned site is never a third @@ -693,7 +812,7 @@ func (h *trackerMappingHandler) identifyWithAgent( log.String("pattern", tp.Pattern), ) - return nil, nil + return true } // Cookie-database and cookie-banner directory sites (Cookifi, @@ -712,12 +831,10 @@ func (h *trackerMappingHandler) identifyWithAgent( log.String("pattern", tp.Pattern), ) - return nil, nil + return true } - return &agentIdentification{ - result: identification, - }, nil + return false } // nameMatchesSiteDomain reports whether a candidate vendor name refers to @@ -788,11 +905,18 @@ func nameIsCookieDatabaseAggregator(name string) bool { // it resolves or creates the catalog third party and upserts the // catalog pattern row that links to it. It runs inside the caller's // short transaction. +// +// priorUntrustedThirdPartyID, when set, is the vendor an existing +// catalog row carried but that was too low-confidence to adopt +// deterministically. When the agent independently lands on the same +// vendor, that is corroboration: the row is promoted to the trusted tier +// so subsequent patterns adopt it without re-running the agent. func (h *trackerMappingHandler) persistAgentIdentification( ctx context.Context, tx pg.Tx, tp coredata.TrackerPattern, ident agentIdentification, + priorUntrustedThirdPartyID *gid.GID, ) (*catalogMatch, error) { commonThirdPartyID, err := thirdparty.ResolveOrCreateCommonThirdParty( ctx, @@ -805,6 +929,15 @@ func (h *trackerMappingHandler) persistAgentIdentification( return nil, fmt.Errorf("cannot resolve or create common third party: %w", err) } + confidence := float32(agentSourceConfidence) + + corroborated := priorUntrustedThirdPartyID != nil && + commonThirdPartyID != nil && + *priorUntrustedThirdPartyID == *commonThirdPartyID + if corroborated { + confidence = trustedAttributionConfidence + } + now := time.Now() commonPattern := coredata.CommonTrackerPattern{ ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType), @@ -813,7 +946,8 @@ func (h *trackerMappingHandler) persistAgentIdentification( Pattern: tp.Pattern, MatchType: tp.MatchType, MaxAgeSeconds: tp.MaxAgeSeconds, - Confidence: agentSourceConfidence, + Confidence: confidence, + Attribution: coredata.CommonTrackerPatternAttributionThirdParty, CreatedAt: now, UpdatedAt: now, } @@ -828,6 +962,7 @@ func (h *trackerMappingHandler) persistAgentIdentification( log.String("pattern", tp.Pattern), log.String("third_party", ident.result.ThirdPartyName), log.Float64("third_party_confidence", ident.result.ThirdPartyConfidence), + log.Bool("corroborated_prior_attribution", corroborated), ) return &catalogMatch{ @@ -836,6 +971,46 @@ func (h *trackerMappingHandler) persistAgentIdentification( }, nil } +// persistFirstPartyVerdict records the agent's terminal first-party +// verdict on the catalog: it upserts the row with no vendor and the +// FIRST_PARTY attribution, which the upsert preserves on later automated +// runs. Any stray low-confidence vendor a prior run left on the row is +// cleared. It runs inside the caller's short transaction. +func (h *trackerMappingHandler) persistFirstPartyVerdict( + ctx context.Context, + tx pg.Tx, + tp coredata.TrackerPattern, +) (*catalogMatch, error) { + now := time.Now() + commonPattern := coredata.CommonTrackerPattern{ + ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType), + TrackerType: tp.TrackerType, + Pattern: tp.Pattern, + MatchType: tp.MatchType, + MaxAgeSeconds: tp.MaxAgeSeconds, + Confidence: agentSourceConfidence, + Attribution: coredata.CommonTrackerPatternAttributionFirstParty, + CreatedAt: now, + UpdatedAt: now, + } + + if _, err := commonPattern.Upsert(ctx, tx); err != nil { + return nil, fmt.Errorf("cannot upsert first-party common tracker pattern: %w", err) + } + + h.logger.InfoCtx( + ctx, + "recorded first-party tracker verdict", + log.String("pattern", tp.Pattern), + log.String("tracker_pattern_id", tp.ID.String()), + ) + + return &catalogMatch{ + commonPatternID: &commonPattern.ID, + firstParty: true, + }, nil +} + // matchBySiblingOrigin finds other tracker patterns on the same banner // that share initiator domains with the current pattern. Sharing an // origin across multiple detected patterns is a strong indicator of the @@ -898,6 +1073,7 @@ func (h *trackerMappingHandler) matchBySiblingOrigin( MatchType: tp.MatchType, MaxAgeSeconds: tp.MaxAgeSeconds, Confidence: 0.7, + Attribution: coredata.CommonTrackerPatternAttributionThirdParty, CreatedAt: now, UpdatedAt: now, } @@ -1026,6 +1202,7 @@ func (h *trackerMappingHandler) createUnmatchedPattern( MatchType: tp.MatchType, MaxAgeSeconds: tp.MaxAgeSeconds, Confidence: 0.5, + Attribution: coredata.CommonTrackerPatternAttributionUndetermined, CreatedAt: now, UpdatedAt: now, } diff --git a/pkg/cookiebanner/tracker_mapping_worker_test.go b/pkg/cookiebanner/tracker_mapping_worker_test.go index d95af0e56..ab55ed637 100644 --- a/pkg/cookiebanner/tracker_mapping_worker_test.go +++ b/pkg/cookiebanner/tracker_mapping_worker_test.go @@ -1742,3 +1742,203 @@ func TestProcess_NoReenqueueWhenCommonThirdPartyPreexisted(t *testing.T) { assert.Nil(t, reloadedUnmapped.MappingRequestedAt, "re-trigger with a pre-existing common third party must not re-enqueue siblings") } + +// TestProcess_FirstPartyVerdictIsTerminal asserts that a pattern whose +// matching catalog row carries the FIRST_PARTY verdict is linked to that +// row but never attributed a third party: the heuristic signals and the +// agent are short-circuited, leaving third_party_id unset. +func TestProcess_FirstPartyVerdictIsTerminal(t *testing.T) { + t.Parallel() + + client := test.PGClient(t) + ctx := context.Background() + fx := seedWorkerFixture(t, ctx, client) + + now := time.Now().UTC().Truncate(time.Microsecond) + patternName := "loglevel_" + fx.scope.GetTenantID().String() + + firstPartyCommon := coredata.CommonTrackerPattern{ + ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType), + TrackerType: coredata.TrackerTypeLocalStorage, + Pattern: patternName, + MatchType: coredata.TrackerPatternMatchTypeExact, + Confidence: 0.8, + Attribution: coredata.CommonTrackerPatternAttributionFirstParty, + CreatedAt: now, + UpdatedAt: now, + } + + // An org pattern with no catalog link yet, so matchByPattern resolves + // the FIRST_PARTY row by (tracker_type, pattern, max_age). + target := coredata.TrackerPattern{ + ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType), + OrganizationID: fx.organizationID, + CookieBannerID: fx.banner.ID, + CookieCategoryID: fx.normalCategoryID, + TrackerType: coredata.TrackerTypeLocalStorage, + Pattern: patternName, + MatchType: coredata.TrackerPatternMatchTypeExact, + DisplayName: patternName, + CreatedAt: now, + UpdatedAt: now, + } + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + if _, err := firstPartyCommon.Upsert(ctx, tx); err != nil { + return err + } + + if err := target.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + return target.SetMappingRequested(ctx, tx) + })) + + t.Cleanup(func() { + _ = client.WithTx(context.Background(), func(ctx context.Context, tx pg.Tx) error { + _, _ = tx.Exec(ctx, `DELETE FROM common_tracker_patterns WHERE id = $1`, firstPartyCommon.ID) + return nil + }) + }) + + h := newMappingHandler(client) + require.NoError(t, h.Process(ctx, target)) + + var reloaded coredata.TrackerPattern + + require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { + return reloaded.LoadByID(ctx, conn, fx.scope, target.ID) + })) + + require.NotNil(t, reloaded.CommonTrackerPatternID, "the pattern must be linked to the first-party catalog row for coverage") + assert.Equal(t, firstPartyCommon.ID, *reloaded.CommonTrackerPatternID) + assert.Nil(t, reloaded.ThirdPartyID, "a first-party verdict must never attribute a third party") + + reloadedCommon := coredata.CommonTrackerPattern{} + require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { + return reloadedCommon.LoadByID(ctx, conn, firstPartyCommon.ID) + })) + + assert.Equal(t, coredata.CommonTrackerPatternAttributionFirstParty, reloadedCommon.Attribution, "verdict must remain first-party") + assert.Nil(t, reloadedCommon.CommonThirdPartyID, "first-party row must stay vendor-free") +} + +// TestProcess_LowConfidenceCatalogVendorNotAdopted asserts that a catalog +// row whose vendor was attributed below trustedAttributionConfidence is +// not adopted deterministically: the pattern links to the row but is not +// promoted to the vendor's org third party, so a single low-confidence +// guess never auto-propagates across organizations. +func TestProcess_LowConfidenceCatalogVendorNotAdopted(t *testing.T) { + t.Parallel() + + client := test.PGClient(t) + ctx := context.Background() + fx := seedWorkerFixture(t, ctx, client) + + now := time.Now().UTC().Truncate(time.Microsecond) + suffix := fx.scope.GetTenantID().String() + patternName := "lowconf_" + suffix + + commonThirdPartyID := gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType) + commonThirdParty := coredata.CommonThirdParty{ + ID: commonThirdPartyID, + Name: "Acme " + suffix, + Slug: "acme-lowconf-" + suffix, + Category: coredata.ThirdPartyCategoryAnalytics, + Certifications: []string{}, + CreatedAt: now, + UpdatedAt: now, + } + + // Agent-tier (0.8) attribution: below the 0.9 trusted bar. + lowConfCommon := coredata.CommonTrackerPattern{ + ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType), + CommonThirdPartyID: &commonThirdPartyID, + TrackerType: coredata.TrackerTypeCookie, + Pattern: patternName, + MatchType: coredata.TrackerPatternMatchTypeExact, + Confidence: agentSourceConfidence, + Attribution: coredata.CommonTrackerPatternAttributionThirdParty, + CreatedAt: now, + UpdatedAt: now, + } + + // An existing org party for the vendor, so promotion WOULD link if the + // vendor were adopted. + orgThirdParty := coredata.ThirdParty{ + ID: gid.New(fx.scope.GetTenantID(), coredata.ThirdPartyEntityType), + OrganizationID: fx.organizationID, + CommonThirdPartyID: &commonThirdPartyID, + Name: "Acme LLC", + Category: coredata.ThirdPartyCategoryAnalytics, + Certifications: []string{}, + Countries: coredata.CountryCodes{}, + CreatedAt: now, + UpdatedAt: now, + } + + target := coredata.TrackerPattern{ + ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType), + OrganizationID: fx.organizationID, + CookieBannerID: fx.banner.ID, + CookieCategoryID: fx.normalCategoryID, + TrackerType: coredata.TrackerTypeCookie, + Pattern: patternName, + MatchType: coredata.TrackerPatternMatchTypeExact, + DisplayName: patternName, + CreatedAt: now, + UpdatedAt: now, + } + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + if err := commonThirdParty.Insert(ctx, tx); err != nil { + return err + } + + if _, err := lowConfCommon.Upsert(ctx, tx); err != nil { + return err + } + + if err := orgThirdParty.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + if err := target.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + return target.SetMappingRequested(ctx, tx) + })) + + t.Cleanup(func() { + _ = client.WithTx(context.Background(), func(ctx context.Context, tx pg.Tx) error { + _, _ = tx.Exec(ctx, `DELETE FROM common_tracker_patterns WHERE id = $1`, lowConfCommon.ID) + _, _ = tx.Exec(ctx, `DELETE FROM common_third_parties WHERE id = $1`, commonThirdPartyID) + return nil + }) + }) + + h := newMappingHandler(client) + require.NoError(t, h.Process(ctx, target)) + + var reloaded coredata.TrackerPattern + + require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { + return reloaded.LoadByID(ctx, conn, fx.scope, target.ID) + })) + + require.NotNil(t, reloaded.CommonTrackerPatternID, "the pattern must still be linked to the catalog row") + assert.Equal(t, lowConfCommon.ID, *reloaded.CommonTrackerPatternID) + assert.Nil(t, reloaded.ThirdPartyID, "a below-trust catalog vendor must not be adopted/promoted") + + // The catalog row is untouched: its low-confidence vendor remains for + // a later evidence-backed corroboration. + reloadedCommon := coredata.CommonTrackerPattern{} + require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { + return reloadedCommon.LoadByID(ctx, conn, lowConfCommon.ID) + })) + + require.NotNil(t, reloadedCommon.CommonThirdPartyID, "the catalog vendor must be left in place") + assert.Equal(t, commonThirdPartyID, *reloadedCommon.CommonThirdPartyID) +} diff --git a/pkg/coredata/common_tracker_pattern.go b/pkg/coredata/common_tracker_pattern.go index 822ca6dc7..812f84075 100644 --- a/pkg/coredata/common_tracker_pattern.go +++ b/pkg/coredata/common_tracker_pattern.go @@ -30,20 +30,21 @@ import ( type ( CommonTrackerPattern struct { - ID gid.GID `db:"id"` - CommonThirdPartyID *gid.GID `db:"common_third_party_id"` - TrackerType TrackerType `db:"tracker_type"` - Pattern string `db:"pattern"` - MatchType TrackerPatternMatchType `db:"match_type"` - Description string `db:"description"` - MaxAgeSeconds *int `db:"max_age_seconds"` - Confidence float32 `db:"confidence"` - EnrichmentRequestedAt *time.Time `db:"enrichment_requested_at"` - Enrichment json.RawMessage `db:"enrichment"` - EnrichmentAttempts int `db:"enrichment_attempts"` - LastEnrichmentAttemptAt *time.Time `db:"last_enrichment_attempt_at"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ID gid.GID `db:"id"` + CommonThirdPartyID *gid.GID `db:"common_third_party_id"` + TrackerType TrackerType `db:"tracker_type"` + Pattern string `db:"pattern"` + MatchType TrackerPatternMatchType `db:"match_type"` + Description string `db:"description"` + MaxAgeSeconds *int `db:"max_age_seconds"` + Confidence float32 `db:"confidence"` + Attribution CommonTrackerPatternAttribution `db:"attribution"` + EnrichmentRequestedAt *time.Time `db:"enrichment_requested_at"` + Enrichment json.RawMessage `db:"enrichment"` + EnrichmentAttempts int `db:"enrichment_attempts"` + LastEnrichmentAttemptAt *time.Time `db:"last_enrichment_attempt_at"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` } CommonTrackerPatterns []*CommonTrackerPattern @@ -64,6 +65,7 @@ SELECT description, max_age_seconds, confidence, + attribution, enrichment_requested_at, enrichment, enrichment_attempts, @@ -115,6 +117,7 @@ SELECT description, max_age_seconds, confidence, + attribution, enrichment_requested_at, enrichment, enrichment_attempts, @@ -159,6 +162,10 @@ func (p CommonTrackerPattern) Insert( ctx context.Context, conn pg.Tx, ) error { + if p.Attribution == "" { + p.Attribution = CommonTrackerPatternAttributionUndetermined + } + q := ` INSERT INTO common_tracker_patterns ( id, @@ -169,6 +176,7 @@ INSERT INTO common_tracker_patterns ( description, max_age_seconds, confidence, + attribution, enrichment_requested_at, enrichment, enrichment_attempts, @@ -184,6 +192,7 @@ INSERT INTO common_tracker_patterns ( @description, @max_age_seconds, @confidence, + @attribution, @enrichment_requested_at, @enrichment, @enrichment_attempts, @@ -202,6 +211,7 @@ INSERT INTO common_tracker_patterns ( "description": p.Description, "max_age_seconds": p.MaxAgeSeconds, "confidence": p.Confidence, + "attribution": p.Attribution, "enrichment_requested_at": p.EnrichmentRequestedAt, "enrichment": p.Enrichment, "enrichment_attempts": p.EnrichmentAttempts, @@ -232,6 +242,10 @@ func (p *CommonTrackerPattern) Upsert( // enrichment, and re-arming resets the attempt counter and drops the // prior payload so the row reads as not-yet-completed again (see the // enrichment CASE below). + if p.Attribution == "" { + p.Attribution = CommonTrackerPatternAttributionUndetermined + } + q := ` INSERT INTO common_tracker_patterns ( id, @@ -242,6 +256,7 @@ INSERT INTO common_tracker_patterns ( description, max_age_seconds, confidence, + attribution, enrichment_requested_at, enrichment, enrichment_attempts, @@ -257,6 +272,7 @@ INSERT INTO common_tracker_patterns ( @description, @max_age_seconds, @confidence, + @attribution, CASE WHEN @description = '' THEN NOW() ELSE NULL END, NULL, 0, @@ -266,13 +282,29 @@ INSERT INTO common_tracker_patterns ( ) ON CONFLICT (tracker_type, pattern, COALESCE(max_age_seconds, -1)) DO UPDATE SET - common_third_party_id = EXCLUDED.common_third_party_id, + -- A terminal FIRST_PARTY row stays vendor-free: an automated upsert + -- must never attach a third party to an artifact an operator (or the + -- agent) ruled has none. Other rows take the incoming vendor. + common_third_party_id = CASE + WHEN common_tracker_patterns.attribution = 'FIRST_PARTY' THEN NULL + ELSE EXCLUDED.common_third_party_id + END, match_type = EXCLUDED.match_type, description = CASE WHEN EXCLUDED.description = '' THEN common_tracker_patterns.description ELSE EXCLUDED.description END, confidence = EXCLUDED.confidence, + -- A FIRST_PARTY verdict is terminal: it is only ever set by an + -- explicit operator action (proboctl mark-first-party). Automated + -- mapping upserts must never downgrade it back to a vendor or + -- UNDETERMINED, otherwise a stray domain/sibling match would + -- resurrect the very attribution the operator suppressed. + attribution = CASE + WHEN common_tracker_patterns.attribution = 'FIRST_PARTY' + THEN common_tracker_patterns.attribution + ELSE EXCLUDED.attribution + END, -- A blank, unlinked catalog row that now gains a third party is -- re-queued for enrichment: the enrichment agent leaves descriptions -- blank when it cannot substantiate a purpose, and knowing the vendor @@ -314,6 +346,7 @@ RETURNING description, max_age_seconds, confidence, + attribution, enrichment_requested_at, enrichment, enrichment_attempts, @@ -333,6 +366,7 @@ RETURNING "description": p.Description, "max_age_seconds": p.MaxAgeSeconds, "confidence": p.Confidence, + "attribution": p.Attribution, "created_at": p.CreatedAt, "updated_at": p.UpdatedAt, } @@ -386,6 +420,7 @@ SELECT description, max_age_seconds, confidence, + attribution, enrichment_requested_at, enrichment, enrichment_attempts, @@ -505,6 +540,7 @@ SELECT description, max_age_seconds, confidence, + attribution, enrichment_requested_at, enrichment, enrichment_attempts, @@ -554,6 +590,7 @@ SELECT description, max_age_seconds, confidence, + attribution, enrichment_requested_at, enrichment, enrichment_attempts, @@ -743,6 +780,7 @@ SELECT description, max_age_seconds, confidence, + attribution, enrichment_requested_at, enrichment, enrichment_attempts, @@ -812,6 +850,7 @@ SELECT description, max_age_seconds, confidence, + attribution, enrichment_requested_at, enrichment, enrichment_attempts, @@ -916,11 +955,12 @@ ORDER BY pattern ASC // RelinkCommonThirdPartyByIDs repoints the given common tracker patterns // at a different common third party (or unlinks them when thirdPartyID is // nil). Linking is a manual operator attribution - the highest-trust -// signal - so it bumps confidence to 1 to match the curated/seed tier; -// unlinking makes no attribution and leaves confidence untouched. It only -// touches the catalog rows; callers re-arm enrichment and remap the -// org-scoped tracker patterns separately. Returns the number of rows -// updated. +// signal - so it bumps confidence to 1 to match the curated/seed tier and +// sets the attribution verdict to THIRD_PARTY; unlinking makes no +// attribution, returns the verdict to UNDETERMINED so the pipeline can +// re-probe the row, and leaves confidence untouched. It only touches the +// catalog rows; callers re-arm enrichment and remap the org-scoped tracker +// patterns separately. Returns the number of rows updated. func (ps *CommonTrackerPatterns) RelinkCommonThirdPartyByIDs( ctx context.Context, tx pg.Tx, @@ -932,6 +972,10 @@ UPDATE common_tracker_patterns SET common_third_party_id = @third_party_id, confidence = CASE WHEN @third_party_id::text IS NOT NULL THEN 1 ELSE confidence END, + attribution = CASE + WHEN @third_party_id::text IS NOT NULL THEN 'THIRD_PARTY'::common_tracker_pattern_attribution + ELSE 'UNDETERMINED'::common_tracker_pattern_attribution + END, updated_at = NOW() WHERE id = ANY(@ids) @@ -950,6 +994,42 @@ WHERE return result.RowsAffected(), nil } +// SetAttributionByIDs records a terminal attribution verdict on the given +// catalog rows. It is an operator action: marking a row FIRST_PARTY (or +// UNDETERMINED) clears any vendor link, because a non-third-party verdict +// cannot keep a common_third_party_id. THIRD_PARTY is not a valid verdict +// here - that attribution carries a vendor and must go through +// RelinkCommonThirdPartyByIDs. Callers re-arm the org-scoped tracker +// patterns separately. Returns the number of rows updated. +func (ps *CommonTrackerPatterns) SetAttributionByIDs( + ctx context.Context, + tx pg.Tx, + ids []gid.GID, + attribution CommonTrackerPatternAttribution, +) (int64, error) { + q := ` +UPDATE common_tracker_patterns +SET + attribution = @attribution, + common_third_party_id = NULL, + updated_at = NOW() +WHERE + id = ANY(@ids) +` + + args := pgx.StrictNamedArgs{ + "ids": ids, + "attribution": attribution, + } + + result, err := tx.Exec(ctx, q, args) + if err != nil { + return 0, fmt.Errorf("cannot set common tracker pattern attribution: %w", err) + } + + return result.RowsAffected(), nil +} + // RequestEnrichmentByIDs arms enrichment on the given common tracker // patterns by stamping enrichment_requested_at, which is the only column // the enrichment worker claims on. It resets enrichment_attempts to 0 so diff --git a/pkg/coredata/common_tracker_pattern_attribution.go b/pkg/coredata/common_tracker_pattern_attribution.go new file mode 100644 index 000000000..66ecee4a1 --- /dev/null +++ b/pkg/coredata/common_tracker_pattern_attribution.go @@ -0,0 +1,88 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package coredata + +import ( + "encoding" + "fmt" +) + +// CommonTrackerPatternAttribution is the terminal verdict a catalog row +// carries about who, if anyone, sets the tracker. +// +// CommonTrackerPatternAttributionUndetermined: the pipeline has not +// resolved a vendor yet. The deterministic signals and the mapping agent +// keep probing it (this is the state of the unmatched fallback row). +// +// CommonTrackerPatternAttributionThirdParty: a third party has been +// resolved; the row carries a common_third_party_id. +// +// CommonTrackerPatternAttributionFirstParty: terminal verdict that the +// artifact has no third party — it is the scanned site's own, a generic +// library/log key, an extension key embedding the site origin, or +// otherwise not attributable to any vendor. The mapping pipeline never +// attributes such a row again. +type CommonTrackerPatternAttribution string + +const ( + CommonTrackerPatternAttributionUndetermined CommonTrackerPatternAttribution = "UNDETERMINED" + CommonTrackerPatternAttributionThirdParty CommonTrackerPatternAttribution = "THIRD_PARTY" + CommonTrackerPatternAttributionFirstParty CommonTrackerPatternAttribution = "FIRST_PARTY" +) + +var ( + _ fmt.Stringer = CommonTrackerPatternAttribution("") + _ encoding.TextMarshaler = CommonTrackerPatternAttribution("") + _ encoding.TextUnmarshaler = (*CommonTrackerPatternAttribution)(nil) +) + +func CommonTrackerPatternAttributions() []CommonTrackerPatternAttribution { + return []CommonTrackerPatternAttribution{ + CommonTrackerPatternAttributionUndetermined, + CommonTrackerPatternAttributionThirdParty, + CommonTrackerPatternAttributionFirstParty, + } +} + +func (v CommonTrackerPatternAttribution) IsValid() bool { + switch v { + case + CommonTrackerPatternAttributionUndetermined, + CommonTrackerPatternAttributionThirdParty, + CommonTrackerPatternAttributionFirstParty: + return true + } + + return false +} + +func (v CommonTrackerPatternAttribution) String() string { + return string(v) +} + +func (v CommonTrackerPatternAttribution) MarshalText() ([]byte, error) { + return []byte(v.String()), nil +} + +func (v *CommonTrackerPatternAttribution) UnmarshalText(text []byte) error { + val := CommonTrackerPatternAttribution(text) + if !val.IsValid() { + return fmt.Errorf("invalid CommonTrackerPatternAttribution value: %q", string(text)) + } + + *v = val + + return nil +} diff --git a/pkg/coredata/common_tracker_pattern_filter.go b/pkg/coredata/common_tracker_pattern_filter.go index 9e2e1ff13..812de9c46 100644 --- a/pkg/coredata/common_tracker_pattern_filter.go +++ b/pkg/coredata/common_tracker_pattern_filter.go @@ -79,6 +79,7 @@ type CommonTrackerPatternFilter struct { linked *bool described *bool state *CommonTrackerPatternEnrichmentState + attribution *CommonTrackerPatternAttribution } func NewCommonTrackerPatternFilter() *CommonTrackerPatternFilter { @@ -130,6 +131,11 @@ func (f *CommonTrackerPatternFilter) WithState(state *CommonTrackerPatternEnrich return f } +func (f *CommonTrackerPatternFilter) WithAttribution(attribution *CommonTrackerPatternAttribution) *CommonTrackerPatternFilter { + f.attribution = attribution + return f +} + func (f *CommonTrackerPatternFilter) SQLFragment() string { if f == nil { return "TRUE" @@ -188,6 +194,12 @@ func (f *CommonTrackerPatternFilter) SQLFragment() string { enrichment_requested_at IS NULL AND enrichment IS NULL ELSE TRUE END + AND + CASE + WHEN @filter_attribution::text IS NOT NULL THEN + attribution = @filter_attribution::common_tracker_pattern_attribution + ELSE TRUE + END )` } @@ -203,6 +215,7 @@ func (f *CommonTrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs { "filter_state_queued": false, "filter_state_enriched": false, "filter_state_unenriched": false, + "filter_attribution": nil, } if f == nil { @@ -248,5 +261,9 @@ func (f *CommonTrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs { } } + if f.attribution != nil { + args["filter_attribution"] = string(*f.attribution) + } + return args } diff --git a/pkg/coredata/common_tracker_pattern_test.go b/pkg/coredata/common_tracker_pattern_test.go index 2efdac172..78ffafbef 100644 --- a/pkg/coredata/common_tracker_pattern_test.go +++ b/pkg/coredata/common_tracker_pattern_test.go @@ -430,3 +430,179 @@ func TestCommonTrackerPattern_ResetStaleEnrichments_RespectsMaxAttempts(t *testi reloadedExhausted := loadCommonTrackerPattern(t, ctx, client, exhausted.ID) assert.Nil(t, reloadedExhausted.EnrichmentRequestedAt, "row at the max-attempts ceiling must not be re-queued") } + +// TestCommonTrackerPatternAttribution_IsValid pins the enum's accepted +// values. +func TestCommonTrackerPatternAttribution_IsValid(t *testing.T) { + t.Parallel() + + for _, v := range coredata.CommonTrackerPatternAttributions() { + assert.True(t, v.IsValid(), "%q must be valid", v) + } + + assert.False(t, coredata.CommonTrackerPatternAttribution("").IsValid()) + assert.False(t, coredata.CommonTrackerPatternAttribution("nonsense").IsValid()) +} + +// TestCommonTrackerPattern_Upsert_RoundTripsAttribution pins that the +// attribution verdict is persisted and read back, and that an empty +// verdict defaults to UNDETERMINED. +func TestCommonTrackerPattern_Upsert_RoundTripsAttribution(t *testing.T) { + t.Parallel() + + client := test.PGClient(t) + ctx := context.Background() + + now := time.Now().UTC().Truncate(time.Microsecond) + cp := coredata.CommonTrackerPattern{ + ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType), + TrackerType: coredata.TrackerTypeLocalStorage, + Pattern: "attr_default_" + gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType).String(), + MatchType: coredata.TrackerPatternMatchTypeExact, + Confidence: 0.5, + CreatedAt: now, + UpdatedAt: now, + } + insertCommonTrackerPattern(t, ctx, client, cp) + + reloaded := loadCommonTrackerPattern(t, ctx, client, cp.ID) + assert.Equal(t, coredata.CommonTrackerPatternAttributionUndetermined, reloaded.Attribution, "empty verdict must default to UNDETERMINED") +} + +// TestCommonTrackerPattern_Upsert_PreservesFirstPartyVerdict pins the +// terminal contract: once a row is FIRST_PARTY, an automated upsert that +// carries a vendor neither flips the verdict nor attaches the vendor. +func TestCommonTrackerPattern_Upsert_PreservesFirstPartyVerdict(t *testing.T) { + t.Parallel() + + client := test.PGClient(t) + ctx := context.Background() + + party := seedCommonThirdParty(t, ctx, client) + + now := time.Now().UTC().Truncate(time.Microsecond) + pattern := "first_party_terminal_" + gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType).String() + + firstParty := coredata.CommonTrackerPattern{ + ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType), + TrackerType: coredata.TrackerTypeLocalStorage, + Pattern: pattern, + MatchType: coredata.TrackerPatternMatchTypeExact, + Confidence: 0.8, + Attribution: coredata.CommonTrackerPatternAttributionFirstParty, + CreatedAt: now, + UpdatedAt: now, + } + insertCommonTrackerPattern(t, ctx, client, firstParty) + + // An automated upsert (same key) that tries to attach a vendor. + intruder := coredata.CommonTrackerPattern{ + ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType), + CommonThirdPartyID: &party.ID, + TrackerType: coredata.TrackerTypeLocalStorage, + Pattern: pattern, + MatchType: coredata.TrackerPatternMatchTypeExact, + Confidence: 0.7, + Attribution: coredata.CommonTrackerPatternAttributionThirdParty, + CreatedAt: now, + UpdatedAt: now.Add(time.Minute), + } + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + _, err := intruder.Upsert(ctx, tx) + return err + })) + + reloaded := loadCommonTrackerPattern(t, ctx, client, firstParty.ID) + assert.Equal(t, coredata.CommonTrackerPatternAttributionFirstParty, reloaded.Attribution, "FIRST_PARTY verdict must survive an automated upsert") + assert.Nil(t, reloaded.CommonThirdPartyID, "a terminal first-party row must stay vendor-free") +} + +// TestCommonTrackerPatterns_SetAttributionByIDs pins that the operator +// helper records the verdict and clears any vendor link. +func TestCommonTrackerPatterns_SetAttributionByIDs(t *testing.T) { + t.Parallel() + + client := test.PGClient(t) + ctx := context.Background() + + party := seedCommonThirdParty(t, ctx, client) + + now := time.Now().UTC().Truncate(time.Microsecond) + linked := coredata.CommonTrackerPattern{ + ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType), + CommonThirdPartyID: &party.ID, + TrackerType: coredata.TrackerTypeCookie, + Pattern: "to_first_party_" + gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType).String(), + MatchType: coredata.TrackerPatternMatchTypeExact, + Confidence: 0.8, + Attribution: coredata.CommonTrackerPatternAttributionThirdParty, + CreatedAt: now, + UpdatedAt: now, + } + insertCommonTrackerPattern(t, ctx, client, linked) + + var affected int64 + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + var ps coredata.CommonTrackerPatterns + var err error + + affected, err = ps.SetAttributionByIDs(ctx, tx, []gid.GID{linked.ID}, coredata.CommonTrackerPatternAttributionFirstParty) + + return err + })) + + assert.Equal(t, int64(1), affected) + + reloaded := loadCommonTrackerPattern(t, ctx, client, linked.ID) + assert.Equal(t, coredata.CommonTrackerPatternAttributionFirstParty, reloaded.Attribution) + assert.Nil(t, reloaded.CommonThirdPartyID, "marking first-party must clear the vendor link") +} + +// TestCommonTrackerPatterns_RelinkCommonThirdPartyByIDs_SetsAttribution +// pins that linking sets THIRD_PARTY and unlinking returns the row to +// UNDETERMINED. +func TestCommonTrackerPatterns_RelinkCommonThirdPartyByIDs_SetsAttribution(t *testing.T) { + t.Parallel() + + client := test.PGClient(t) + ctx := context.Background() + + party := seedCommonThirdParty(t, ctx, client) + + now := time.Now().UTC().Truncate(time.Microsecond) + row := coredata.CommonTrackerPattern{ + ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType), + TrackerType: coredata.TrackerTypeCookie, + Pattern: "relink_attr_" + gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType).String(), + MatchType: coredata.TrackerPatternMatchTypeExact, + Confidence: 0.5, + Attribution: coredata.CommonTrackerPatternAttributionUndetermined, + CreatedAt: now, + UpdatedAt: now, + } + insertCommonTrackerPattern(t, ctx, client, row) + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + var ps coredata.CommonTrackerPatterns + _, err := ps.RelinkCommonThirdPartyByIDs(ctx, tx, []gid.GID{row.ID}, &party.ID) + return err + })) + + linked := loadCommonTrackerPattern(t, ctx, client, row.ID) + assert.Equal(t, coredata.CommonTrackerPatternAttributionThirdParty, linked.Attribution, "linking must set THIRD_PARTY") + require.NotNil(t, linked.CommonThirdPartyID) + assert.Equal(t, party.ID, *linked.CommonThirdPartyID) + assert.Equal(t, float32(1), linked.Confidence, "linking must bump confidence to the curated tier") + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + var ps coredata.CommonTrackerPatterns + _, err := ps.RelinkCommonThirdPartyByIDs(ctx, tx, []gid.GID{row.ID}, nil) + return err + })) + + unlinked := loadCommonTrackerPattern(t, ctx, client, row.ID) + assert.Equal(t, coredata.CommonTrackerPatternAttributionUndetermined, unlinked.Attribution, "unlinking must return the verdict to UNDETERMINED") + assert.Nil(t, unlinked.CommonThirdPartyID) +} diff --git a/pkg/coredata/migrations/20260616T114832Z.sql b/pkg/coredata/migrations/20260616T114832Z.sql new file mode 100644 index 000000000..e0b2aa7f4 --- /dev/null +++ b/pkg/coredata/migrations/20260616T114832Z.sql @@ -0,0 +1,38 @@ +-- Copyright (c) 2026 Probo Inc . +-- +-- Permission to use, copy, modify, and/or distribute this software for any +-- purpose with or without fee is hereby granted, provided that the above +-- copyright notice and this permission notice appear in all copies. +-- +-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +-- PERFORMANCE OF THIS SOFTWARE. + +-- Give the global tracker-pattern catalog a terminal attribution verdict so a +-- row can record that it has no third party (a first-party or generic +-- artifact) rather than only "linked" vs "not yet linked". UNDETERMINED rows +-- are still probed by the mapping pipeline; THIRD_PARTY rows carry a vendor; +-- FIRST_PARTY rows are terminal and the pipeline never attributes them again. +CREATE TYPE common_tracker_pattern_attribution AS ENUM ( + 'UNDETERMINED', + 'THIRD_PARTY', + 'FIRST_PARTY' +); + +ALTER TABLE common_tracker_patterns + ADD COLUMN attribution common_tracker_pattern_attribution NOT NULL DEFAULT 'UNDETERMINED'; + +-- Backfill: any row already carrying a vendor is, by definition, attributed to +-- a third party. The DEFAULT covers the rest (UNDETERMINED). +UPDATE common_tracker_patterns +SET attribution = 'THIRD_PARTY' +WHERE common_third_party_id IS NOT NULL; + +-- The DEFAULT only backfills existing rows; drop it so inserts must supply the +-- value explicitly, matching the cookie_source convention. +ALTER TABLE common_tracker_patterns + ALTER COLUMN attribution DROP DEFAULT; diff --git a/pkg/proboctl/commontrackerpattern/commontrackerpattern.go b/pkg/proboctl/commontrackerpattern/commontrackerpattern.go index f5679272a..0dd1ec393 100644 --- a/pkg/proboctl/commontrackerpattern/commontrackerpattern.go +++ b/pkg/proboctl/commontrackerpattern/commontrackerpattern.go @@ -42,6 +42,7 @@ func NewCmdCommonTrackerPattern(f *cmdutil.Factory) *cobra.Command { cmd.AddCommand(newCmdStats(f)) cmd.AddCommand(newCmdLink(f)) cmd.AddCommand(newCmdUnlink(f)) + cmd.AddCommand(newCmdMarkFirstParty(f)) cmd.AddCommand(newCmdSetDescription(f)) return cmd diff --git a/pkg/proboctl/commontrackerpattern/list.go b/pkg/proboctl/commontrackerpattern/list.go index 116fcdff9..fdfeaad53 100644 --- a/pkg/proboctl/commontrackerpattern/list.go +++ b/pkg/proboctl/commontrackerpattern/list.go @@ -36,6 +36,7 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { flagLinkedOrg string flagKeyword string flagState string + flagAttribution string flagWithCommonThirdParty bool flagWithoutDescription bool flagSort string @@ -57,6 +58,7 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { cmd.Flags().StringVar(&flagLinkedOrg, "linked-org", "", "Filter to catalog rows linked to an organization's patterns (GID)") cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter by pattern/description substring") cmd.Flags().StringVar(&flagState, "state", "", "Filter by enrichment state (queued, enriched, unenriched)") + cmd.Flags().StringVar(&flagAttribution, "attribution", "", "Filter by attribution verdict (UNDETERMINED, THIRD_PARTY, FIRST_PARTY)") cmd.Flags().BoolVar(&flagWithCommonThirdParty, "with-common-third-party", false, "Filter by whether the pattern is linked to a common third party (true/false); ignored when not set") cmd.Flags().BoolVar(&flagWithoutDescription, "without-description", false, "Only patterns with a blank description") cmd.Flags().StringVar(&flagSort, "sort", "confidence", "Sort field: pattern, confidence, created, updated, attempted") @@ -93,7 +95,7 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { described = new(false) } - filter, err := buildListFilter(flagTrackerType, flagMatchType, flagKeyword, flagState, withCommonThirdParty, described) + filter, err := buildListFilter(flagTrackerType, flagMatchType, flagKeyword, flagState, flagAttribution, withCommonThirdParty, described) if err != nil { return err } @@ -229,7 +231,7 @@ func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredat return err } - table := clicmdutil.NewTable("ID", "TYPE", "MATCH", "PATTERN", "CONF", "STATE", "THIRD PARTY", "LAST ATTEMPT", "CREATED", "UPDATED") + table := clicmdutil.NewTable("ID", "TYPE", "MATCH", "PATTERN", "CONF", "VERDICT", "STATE", "THIRD PARTY", "LAST ATTEMPT", "CREATED", "UPDATED") for _, p := range patterns { thirdParty := "" @@ -248,6 +250,7 @@ func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredat string(p.MatchType), p.Pattern, fmt.Sprintf("%.2f", p.Confidence), + string(p.Attribution), enrichmentState(p), thirdParty, lastAttempt, @@ -309,7 +312,7 @@ func parseOrderBy(sort, order string) (page.OrderBy[coredata.CommonTrackerPatter } func buildListFilter( - trackerType, matchType, keyword, state string, + trackerType, matchType, keyword, state, attribution string, withCommonThirdParty, described *bool, ) (*coredata.CommonTrackerPatternFilter, error) { filter := coredata.NewCommonTrackerPatternFilter() @@ -345,6 +348,15 @@ func buildListFilter( filter.WithState(&st) } + if attribution != "" { + attr := coredata.CommonTrackerPatternAttribution(attribution) + if !attr.IsValid() { + return nil, fmt.Errorf("invalid --attribution value %q: valid values are UNDETERMINED, THIRD_PARTY, FIRST_PARTY", attribution) + } + + filter.WithAttribution(&attr) + } + if withCommonThirdParty != nil { filter.WithLinked(withCommonThirdParty) } diff --git a/pkg/proboctl/commontrackerpattern/mark_first_party.go b/pkg/proboctl/commontrackerpattern/mark_first_party.go new file mode 100644 index 000000000..7f0a46e34 --- /dev/null +++ b/pkg/proboctl/commontrackerpattern/mark_first_party.go @@ -0,0 +1,148 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package commontrackerpattern + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + "go.gearno.de/kit/pg" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/proboctl/cmdutil" +) + +func newCmdMarkFirstParty(f *cmdutil.Factory) *cobra.Command { + var ( + flagIDs []string + flagLinkedBanner string + flagLinkedOrg string + flagCommonThirdParty string + flagTrackerType string + flagKeyword string + flagState string + flagWithoutDescription bool + flagDryRun bool + flagYes bool + ) + + cmd := &cobra.Command{ + Use: "mark-first-party", + Short: "Mark common tracker patterns as first-party (no third party)", + Long: "Record the terminal FIRST_PARTY verdict on selected common tracker " + + "patterns: the artifact has no third party (it is the scanned site's own, " + + "a generic library/log key, or an extension key embedding the site origin). " + + "Any vendor link is cleared, and the uncategorised org tracker patterns " + + "linked to them are remapped (org third party cleared, mapping re-armed) so " + + "the pipeline drops the stale vendor; because the verdict is terminal the " + + "mapping worker leaves them unattributed. User-categorised and excluded org " + + "patterns are left untouched. Selection mirrors 'reenrich'. To re-attribute " + + "a row later, use 'link' (which returns it to THIRD_PARTY).", + Args: cobra.NoArgs, + } + + cmd.Flags().StringSliceVar(&flagIDs, "id", nil, "Common tracker pattern GID(s) to mark (repeatable)") + cmd.Flags().StringVar(&flagLinkedBanner, "linked-banner", "", "Select catalog rows linked to a cookie banner's patterns (GID)") + cmd.Flags().StringVar(&flagLinkedOrg, "linked-org", "", "Select catalog rows linked to an organization's patterns (GID)") + cmd.Flags().StringVar(&flagCommonThirdParty, "common-third-party", "", "Select patterns currently linked to a common third party (slug or GID)") + cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Filter selected patterns by tracker type") + cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter selected patterns by a pattern/description substring") + cmd.Flags().StringVar(&flagState, "state", "", "Filter selected patterns by enrichment state (queued, enriched, unenriched)") + cmd.Flags().BoolVar(&flagWithoutDescription, "without-description", false, "Only patterns with a blank description") + cmd.Flags().BoolVar(&flagDryRun, "dry-run", false, "Print the selected patterns without marking") + cmd.Flags().BoolVar(&flagYes, "yes", false, "Skip confirmation") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + pgClient, err := f.PgClient() + if err != nil { + return err + } + + ids, err := resolveReenrichIDs( + ctx, + pgClient, + flagIDs, + flagLinkedBanner, + flagLinkedOrg, + flagCommonThirdParty, + flagTrackerType, + flagKeyword, + flagState, + flagWithoutDescription, + ) + if err != nil { + return err + } + + out := f.IOStreams.Out + + if len(ids) == 0 { + _, _ = fmt.Fprintln(out, "No common tracker patterns matched the selection.") + return nil + } + + if flagDryRun { + _, _ = fmt.Fprintf(out, "Would mark %d common tracker pattern(s) as first-party.\n", len(ids)) + printSample(out, ids) + + return nil + } + + if !flagYes { + return fmt.Errorf("about to mark %d pattern(s) as first-party; pass --yes to proceed or --dry-run to preview", len(ids)) + } + + var ( + marked int64 + remapped int64 + ) + + if err := pgClient.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + var ps coredata.CommonTrackerPatterns + + marked, err = ps.SetAttributionByIDs(ctx, tx, ids, coredata.CommonTrackerPatternAttributionFirstParty) + if err != nil { + return err + } + + var tps coredata.TrackerPatterns + + remapped, err = tps.RequestMappingForUncategorisedByCommonTrackerPatternIDs(ctx, tx, ids) + if err != nil { + return err + } + + return nil + }, + ); err != nil { + return fmt.Errorf("cannot mark common tracker patterns first-party: %w", err) + } + + _, _ = fmt.Fprintf( + out, + "Marked %d pattern(s) first-party, remapped %d uncategorised org tracker pattern(s).\n", + marked, + remapped, + ) + + return nil + } + + return cmd +} diff --git a/pkg/proboctl/commontrackerpattern/show.go b/pkg/proboctl/commontrackerpattern/show.go index d2d189019..e8b018d2d 100644 --- a/pkg/proboctl/commontrackerpattern/show.go +++ b/pkg/proboctl/commontrackerpattern/show.go @@ -137,6 +137,7 @@ func renderPatternDetail(f *cmdutil.Factory, p coredata.CommonTrackerPattern, th row("Match type:", string(p.MatchType)) row("Pattern:", p.Pattern) row("Confidence:", fmt.Sprintf("%.2f", p.Confidence)) + row("Verdict:", string(p.Attribution)) row("State:", enrichmentState(&p)) if p.MaxAgeSeconds != nil { diff --git a/pkg/proboctl/seed/common-tracker-patterns/common_tracker_patterns.go b/pkg/proboctl/seed/common-tracker-patterns/common_tracker_patterns.go index 9059bd22b..411ca1fa9 100644 --- a/pkg/proboctl/seed/common-tracker-patterns/common_tracker_patterns.go +++ b/pkg/proboctl/seed/common-tracker-patterns/common_tracker_patterns.go @@ -136,6 +136,11 @@ func NewCmdCommonTrackerPatterns(f *cmdutil.Factory) *cobra.Command { continue } + attribution := coredata.CommonTrackerPatternAttributionUndetermined + if thirdPartyID != nil { + attribution = coredata.CommonTrackerPatternAttributionThirdParty + } + pattern := coredata.CommonTrackerPattern{ ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType), CommonThirdPartyID: thirdPartyID, @@ -145,6 +150,7 @@ func NewCmdCommonTrackerPatterns(f *cmdutil.Factory) *cobra.Command { Description: p.Description, MaxAgeSeconds: p.MaxAgeSeconds, Confidence: p.Confidence, + Attribution: attribution, CreatedAt: now, UpdatedAt: now, }