diff --git a/pkg/cookiebanner/prompts/tracker_identification.txt.tmpl b/pkg/cookiebanner/prompts/tracker_identification.txt.tmpl index 6b9df3453..3665dc6cf 100644 --- a/pkg/cookiebanner/prompts/tracker_identification.txt.tmpl +++ b/pkg/cookiebanner/prompts/tracker_identification.txt.tmpl @@ -8,7 +8,7 @@ Given a tracker pattern (cookie name, local storage key, etc.), its type, max-ag Return a structured JSON response with: - third_party_name: the canonical company/service name (e.g. "Google Analytics", not "google" or "GA") - category: the business category of the third party -- confidence: how confident you are in the identification (0.0 to 1.0) +- third_party_confidence: how confident you are about WHICH company or service set this tracker (0.0 to 1.0). This is the confidence in the attribution alone — not a judgment of whether the artifact is a "real" web tracker. A browser-extension artifact whose name names its vendor can still warrant high confidence here. @@ -37,15 +37,17 @@ Return a structured JSON response with: 5. The observed domains are useful signals but not conclusive on their own. Many sites load third-party tracker scripts through a first-party reverse proxy (e.g. t.example.com proxying PostHog). When a domain matches the scanned site, it reveals nothing about which third party set the tracker — rely on the naming convention or a database/web search instead. First-party proxy domains are filtered before they reach you, but if you still see the scanned site's own domain, ignore it as evidence. Well-known third-party tracking domains (e.g. doubleclick.net, facebook.com, analytics.google.com) remain strong evidence. -6. Be conservative with confidence: - - 0.9-1.0: exact pattern match found in database or unmistakable naming convention + matching domain +6. Be conservative with third_party_confidence. It measures certainty about the attribution (who set the tracker), nothing else: + - 0.9-1.0: exact pattern match found in database, or unmistakable naming convention + matching domain, or a name that embeds the vendor (e.g. "__darkreader__*" -> Dark Reader) - 0.7-0.8: strong signal from naming convention or domain, but not a database match - 0.5-0.6: reasonable guess based on partial naming patterns - Below 0.5: uncertain, speculative -7. If you truly cannot identify the tracker, set third_party_name to an empty string and confidence below 0.3. + Do not lower third_party_confidence just because the artifact is a browser-extension key, localStorage entry, or otherwise not a classic web tracker. The goal is to attribute the vendor, not to judge how "tracker-worthy" the artifact is — if the name unambiguously names its source, attribute it with high confidence. -8. Only attribute a tracker to a company or service when you have concrete evidence: a database match, an unmistakable naming convention, or a clear web search result. Never guess or invent attributions based on vague similarity or general knowledge. If no evidence supports a match, return an empty third_party_name with confidence below 0.3. +7. If you truly cannot identify who set the tracker, set third_party_name to an empty string and third_party_confidence below 0.3. + +8. Only attribute a tracker to a company or service when you have concrete evidence: a database match, an unmistakable naming convention (including a vendor name embedded in the key), or a clear web search result. Never guess or invent attributions based on vague similarity or general knowledge. If no evidence supports a match, return an empty third_party_name with third_party_confidence below 0.3. 9. For the category field, use one of: {{.Categories}}. Most cookies fall under ANALYTICS or MARKETING. diff --git a/pkg/cookiebanner/tracker_mapping_agent.go b/pkg/cookiebanner/tracker_mapping_agent.go index a0eea3e6a..18d9d0162 100644 --- a/pkg/cookiebanner/tracker_mapping_agent.go +++ b/pkg/cookiebanner/tracker_mapping_agent.go @@ -29,10 +29,15 @@ import ( ) const ( - agentTimeout = 60 * time.Second - agentMaxTurns = 5 - agentConfidenceThreshold = 0.6 - agentMaxPatternConfidence = 0.8 + agentTimeout = 60 * time.Second + agentMaxTurns = 5 + agentThirdPartyConfidenceThreshold = 0.6 + // agentSourceConfidence is the fixed confidence stored on catalog + // rows the agent attributes to a third party. The agent's own + // confidence now gauges the attribution (see ThirdPartyConfidence) + // rather than the pattern, so the stored row confidence is a + // constant like the other heuristic signals (domain, sibling). + agentSourceConfidence = 0.8 ) //go:embed prompts/tracker_identification.txt.tmpl @@ -41,9 +46,9 @@ var trackerIdentificationPrompt string // TrackerMappingAgentResult is the structured output the tracker-mapping // agent returns. type TrackerMappingAgentResult struct { - ThirdPartyName string `json:"third_party_name" jsonschema:"Name of the company or service that sets this tracker (e.g. 'Google Analytics', 'Meta Pixel'). Empty string if truly unknown."` - Category coredata.ThirdPartyCategory `json:"category" jsonschema:"Third party category"` - Confidence float64 `json:"confidence" jsonschema:"Confidence level from 0.0 to 1.0. Set below 0.5 if unsure."` + ThirdPartyName string `json:"third_party_name" jsonschema:"Name of the company or service that sets this tracker (e.g. 'Google Analytics', 'Meta Pixel'). Empty string if truly unknown."` + Category coredata.ThirdPartyCategory `json:"category" jsonschema:"Third party category"` + ThirdPartyConfidence float64 `json:"third_party_confidence" jsonschema:"Confidence (0.0 to 1.0) in which company or service set this tracker, independent of whether the artifact is a classic web tracker. Set below 0.5 if unsure who set it."` } func buildTrackerMappingAgent( diff --git a/pkg/cookiebanner/tracker_mapping_worker.go b/pkg/cookiebanner/tracker_mapping_worker.go index 02c0ffd1a..ecc7895c8 100644 --- a/pkg/cookiebanner/tracker_mapping_worker.go +++ b/pkg/cookiebanner/tracker_mapping_worker.go @@ -515,33 +515,29 @@ func (h *trackerMappingHandler) identifyWithAgent( identification := result.Output - if identification.Confidence < agentConfidenceThreshold { + // The agent's confidence gauges the attribution (who set the + // tracker), not whether the artifact is a meaningful tracker. Without + // a confident vendor there is nothing to catalog here; the unmatched + // fallback records the pattern with no third party instead. + if identification.ThirdPartyName == "" || identification.ThirdPartyConfidence < agentThirdPartyConfidenceThreshold { h.logger.InfoCtx( ctx, - "agent identification below confidence threshold", + "agent third-party attribution below confidence threshold", log.String("pattern", tp.Pattern), - log.Float64("confidence", identification.Confidence), + log.Float64("third_party_confidence", identification.ThirdPartyConfidence), ) return nil, nil } - confidence := float32(identification.Confidence) - if confidence > agentMaxPatternConfidence { - confidence = agentMaxPatternConfidence - } - - var commonThirdPartyID *gid.GID - if identification.ThirdPartyName != "" { - commonThirdPartyID, err = h.resolveOrCreateCommonThirdParty( - ctx, - tx, - identification, - domains, - ) - if err != nil { - return nil, fmt.Errorf("cannot resolve or create common third party: %w", err) - } + commonThirdPartyID, err := h.resolveOrCreateCommonThirdParty( + ctx, + tx, + identification, + domains, + ) + if err != nil { + return nil, fmt.Errorf("cannot resolve or create common third party: %w", err) } now := time.Now() @@ -552,7 +548,7 @@ func (h *trackerMappingHandler) identifyWithAgent( Pattern: tp.Pattern, MatchType: tp.MatchType, MaxAgeSeconds: tp.MaxAgeSeconds, - Confidence: confidence, + Confidence: agentSourceConfidence, CreatedAt: now, UpdatedAt: now, } @@ -566,7 +562,7 @@ func (h *trackerMappingHandler) identifyWithAgent( "agent identified tracker pattern", log.String("pattern", tp.Pattern), log.String("third_party", identification.ThirdPartyName), - log.Float64("confidence", identification.Confidence), + log.Float64("third_party_confidence", identification.ThirdPartyConfidence), ) return &catalogMatch{