The tracker-pattern source ranking collapsed HTTP into the PRE_EXISTING tier, so a cookie first enumerated as pre-existing and later re-observed only via a Set-Cookie response header stayed pre-existing. That left it on the agent-skipped tier (isPreExistingSource), even though an HTTP server-set cookie is real page evidence, not the extension-state catch-all the skip was built to suppress. Give HTTP its own rank between SCRIPT and EXTENSION (SCRIPT > HTTP > EXTENSION > PRE_EXISTING) in both sourceRank and the bestSource merge rollup, so an HTTP re-detection now promotes the pattern and re-arms mapping, unblocking the identification agent. Signed-off-by: Émile Ré <emile@probo.com>
958 lines
26 KiB
Go
958 lines
26 KiB
Go
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
|
|
//
|
|
// Permission to use, copy, modify, and/or distribute this software for any
|
|
// purpose with or without fee is hereby granted, provided that the above
|
|
// copyright notice and this permission notice appear in all copies.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
|
// PERFORMANCE OF THIS SOFTWARE.
|
|
|
|
package cookiebanner
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.gearno.de/kit/log"
|
|
"go.gearno.de/kit/pg"
|
|
"go.gearno.de/kit/worker"
|
|
"go.probo.inc/probo/pkg/coredata"
|
|
"go.probo.inc/probo/pkg/gid"
|
|
"go.probo.inc/probo/pkg/page"
|
|
)
|
|
|
|
const (
|
|
patternMergeThreshold = 3
|
|
|
|
// primarySeparators are the structural delimiters that always
|
|
// start a new token in splitTokens. `-` is intentionally excluded
|
|
// because it appears inside UUIDs, which splitTokens preserves as a
|
|
// single token; a value like "done:ecdd43d7-0193-4d24-b6ed-..."
|
|
// must split on ":" so the trailing UUID is isolated and collapsed
|
|
// to a wildcard rather than shredded into fixed hex anchors.
|
|
primarySeparators = "_:."
|
|
)
|
|
|
|
// durationUnits mirrors the snap table from cookie-utils.ts. The same
|
|
// tracker observed across different clients can have jitter in its
|
|
// max-age (e.g. an "Expires" header computed from Date.now() yields
|
|
// slightly different seconds each time). Snapping to the nearest
|
|
// human-meaningful unit absorbs that jitter so the patterns still
|
|
// merge. This is compliant because the resulting bucket matches the
|
|
// duration shown to end users in the cookie banner — two cookies that
|
|
// display the same human-readable lifetime will merge, two that
|
|
// display differently will not.
|
|
var durationUnits = [...]struct {
|
|
seconds int
|
|
snap int
|
|
}{
|
|
{365 * 24 * 3600, 21 * 24 * 3600}, // years, snap +-21 days
|
|
{30 * 24 * 3600, 2 * 24 * 3600}, // months, snap +-2 days
|
|
{7 * 24 * 3600, 12 * 3600}, // weeks, snap +-12 hours
|
|
{24 * 3600, 2 * 3600}, // days, snap +-2 hours
|
|
{3600, 5 * 60}, // hours, snap +-5 minutes
|
|
{60, 5}, // minutes, snap +-5 seconds
|
|
{1, 0}, // seconds, no snap
|
|
}
|
|
|
|
func durationBucket(maxAge *int) int {
|
|
if maxAge == nil || *maxAge <= 0 {
|
|
return -1
|
|
}
|
|
|
|
remaining := *maxAge
|
|
total := 0
|
|
|
|
for _, u := range durationUnits {
|
|
if remaining >= u.seconds-u.snap {
|
|
count := remaining / u.seconds
|
|
|
|
leftover := remaining - count*u.seconds
|
|
if leftover >= u.seconds-u.snap {
|
|
count++
|
|
remaining = 0
|
|
} else if leftover <= u.snap {
|
|
remaining = 0
|
|
} else {
|
|
remaining = leftover
|
|
}
|
|
|
|
total += count * u.seconds
|
|
}
|
|
}
|
|
|
|
return total
|
|
}
|
|
|
|
type patternAnalysisHandler struct {
|
|
svc *Service
|
|
pg *pg.Client
|
|
logger *log.Logger
|
|
}
|
|
|
|
func NewPatternAnalysisWorker(
|
|
svc *Service,
|
|
pgClient *pg.Client,
|
|
logger *log.Logger,
|
|
opts ...worker.Option,
|
|
) *worker.Worker[coredata.CookieBanner] {
|
|
h := &patternAnalysisHandler{
|
|
svc: svc,
|
|
pg: pgClient,
|
|
logger: logger,
|
|
}
|
|
|
|
return worker.New(
|
|
"tracker-pattern-analysis-worker",
|
|
h,
|
|
logger,
|
|
opts...,
|
|
)
|
|
}
|
|
|
|
func (h *patternAnalysisHandler) Claim(ctx context.Context) (coredata.CookieBanner, error) {
|
|
var banner coredata.CookieBanner
|
|
|
|
if err := h.pg.WithTx(
|
|
ctx,
|
|
func(ctx context.Context, tx pg.Tx) error {
|
|
if err := banner.LoadNextForPatternAnalysisForUpdateSkipLocked(ctx, tx); err != nil {
|
|
return err
|
|
}
|
|
|
|
return banner.ClearPatternAnalysisRequestedAt(ctx, tx)
|
|
},
|
|
); err != nil {
|
|
if errors.Is(err, coredata.ErrResourceNotFound) {
|
|
return coredata.CookieBanner{}, worker.ErrNoTask
|
|
}
|
|
|
|
return coredata.CookieBanner{}, fmt.Errorf("cannot claim pattern analysis task: %w", err)
|
|
}
|
|
|
|
return banner, nil
|
|
}
|
|
|
|
func (h *patternAnalysisHandler) Process(ctx context.Context, banner coredata.CookieBanner) error {
|
|
return h.pg.WithTx(
|
|
ctx,
|
|
func(ctx context.Context, tx pg.Tx) error {
|
|
scope := coredata.NewScopeFromObjectID(banner.ID)
|
|
|
|
exactPatterns, err := page.LoadAll(
|
|
ctx,
|
|
page.OrderBy[coredata.TrackerPatternOrderField]{
|
|
Field: coredata.TrackerPatternOrderFieldCreatedAt,
|
|
Direction: page.OrderDirectionAsc,
|
|
},
|
|
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
|
|
var batch coredata.TrackerPatterns
|
|
if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeExact), nil, new(false))); err != nil {
|
|
return nil, fmt.Errorf("cannot load exact patterns: %w", err)
|
|
}
|
|
|
|
return batch, nil
|
|
},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
mergeGroups := findMergeGroups(exactPatterns, patternMergeThreshold)
|
|
|
|
for key, group := range mergeGroups {
|
|
var maxAge *int
|
|
|
|
if key.durationBucket >= 0 {
|
|
v := key.durationBucket
|
|
maxAge = &v
|
|
}
|
|
|
|
source := bestSource(group)
|
|
|
|
// Carry over the resolved org ThirdParty (and its
|
|
// description) from the merged exacts when they agree,
|
|
// so the glob is seeded rather than re-mapped from
|
|
// scratch. Mapping is still re-armed below so the glob
|
|
// derives its own catalog row; the pre-set third party
|
|
// lets the mapping worker skip the expensive org
|
|
// resolution. Only the insert path consumes these: an
|
|
// existing glob is reloaded and keeps its own mapping.
|
|
inheritedThirdPartyID, inheritedDescription := inheritedMapping(group)
|
|
|
|
now := time.Now()
|
|
globPattern := &coredata.TrackerPattern{
|
|
ID: gid.New(banner.ID.TenantID(), coredata.TrackerPatternEntityType),
|
|
OrganizationID: group[0].OrganizationID,
|
|
CookieBannerID: banner.ID,
|
|
CookieCategoryID: key.categoryID,
|
|
TrackerType: key.trackerType,
|
|
Pattern: key.template,
|
|
MatchType: coredata.TrackerPatternMatchTypeGlob,
|
|
DisplayName: key.template,
|
|
MaxAgeSeconds: maxAge,
|
|
ThirdPartyID: inheritedThirdPartyID,
|
|
Description: inheritedDescription,
|
|
Source: source,
|
|
MappingRequestedAt: &now,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}
|
|
|
|
inserted, err := globPattern.InsertIfNotExists(ctx, tx, scope)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot insert glob pattern %q: %w", key.template, err)
|
|
}
|
|
|
|
if !inserted {
|
|
if err := globPattern.LoadByBannerIDTypeAndPattern(ctx, tx, scope, banner.ID, key.trackerType, key.template, maxAge); err != nil {
|
|
return fmt.Errorf("cannot load existing glob pattern %q: %w", key.template, err)
|
|
}
|
|
|
|
if globPattern.MatchType != coredata.TrackerPatternMatchTypeGlob || globPattern.CookieCategoryID != key.categoryID {
|
|
// The slot is occupied by an exact pattern or
|
|
// a user-recategorised glob. Skip the relink
|
|
// here so we don't overwrite the user's
|
|
// categorisation; the exact patterns in
|
|
// `group` will be picked up below by
|
|
// adoptUncategorisedPatterns if they live in
|
|
// the uncategorised category and globMatch
|
|
// the existing glob.
|
|
continue
|
|
}
|
|
|
|
if shouldPromoteSource(globPattern.Source, source) {
|
|
globPattern.Source = source
|
|
globPattern.UpdatedAt = now
|
|
|
|
if err := globPattern.Update(ctx, tx, scope); err != nil {
|
|
return fmt.Errorf("cannot promote source on glob pattern %q: %w", key.template, err)
|
|
}
|
|
|
|
// A stronger source can unblock mapping (a fresh
|
|
// initiator domain lets matchByDomain/
|
|
// matchBySiblingOrigin resolve a vendor), so
|
|
// re-arm mapping on the existing glob.
|
|
if err := globPattern.SetMappingRequested(ctx, tx); err != nil {
|
|
return fmt.Errorf("cannot request mapping after source promotion on glob pattern %q: %w", key.template, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, exactPattern := range group {
|
|
var trackers coredata.DetectedTrackers
|
|
if err := trackers.RelinkByTrackerPatternID(ctx, tx, scope, exactPattern.ID, globPattern.ID); err != nil {
|
|
return fmt.Errorf("cannot relink detected trackers from pattern %q: %w", exactPattern.Pattern, err)
|
|
}
|
|
|
|
if err := exactPattern.Delete(ctx, tx, scope); err != nil {
|
|
return fmt.Errorf("cannot delete orphaned exact pattern %q: %w", exactPattern.Pattern, err)
|
|
}
|
|
}
|
|
|
|
h.logger.InfoCtx(
|
|
ctx,
|
|
"merged exact patterns into glob pattern",
|
|
log.String("template", key.template),
|
|
log.Int("count", len(group)),
|
|
log.Bool("inserted", inserted),
|
|
log.String("banner_id", banner.ID.String()),
|
|
)
|
|
}
|
|
|
|
adopted, err := h.adoptUncategorisedPatterns(ctx, tx, scope, banner)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot adopt uncategorised patterns: %w", err)
|
|
}
|
|
|
|
var patterns coredata.TrackerPatterns
|
|
if err := patterns.RefreshLastMatchedAtByCookieBannerID(ctx, tx, scope, banner.ID); err != nil {
|
|
return fmt.Errorf("cannot refresh last_matched_at: %w", err)
|
|
}
|
|
|
|
// Merging exact patterns into a glob in the same category
|
|
// does not change visitor consent for those identifiers
|
|
// (findMergeGroups keys on category, so every member of a
|
|
// group is already under key.categoryID). Adoption is the
|
|
// only operation in this worker that moves trackers
|
|
// between categories and therefore changes consent.
|
|
if adopted {
|
|
if _, err := h.svc.ensureDraftVersionForBanner(ctx, tx, scope, banner.ID); err != nil {
|
|
return fmt.Errorf("cannot ensure draft version: %w", err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
},
|
|
)
|
|
}
|
|
|
|
type mergeGroupKey struct {
|
|
categoryID gid.GID
|
|
trackerType coredata.TrackerType
|
|
template string
|
|
durationBucket int
|
|
}
|
|
|
|
func findMergeGroups(
|
|
patterns coredata.TrackerPatterns,
|
|
threshold int,
|
|
) map[mergeGroupKey][]*coredata.TrackerPattern {
|
|
type memberKey struct {
|
|
groupKey mergeGroupKey
|
|
pattern *coredata.TrackerPattern
|
|
}
|
|
|
|
templateCounts := make(map[mergeGroupKey][]*coredata.TrackerPattern)
|
|
heuristicKeys := make(map[mergeGroupKey]bool)
|
|
seen := make(map[memberKey]bool)
|
|
|
|
for _, p := range patterns {
|
|
bucket := durationBucket(p.MaxAgeSeconds)
|
|
|
|
if tmpl, ok := heuristicTemplate(p.Pattern); ok {
|
|
key := mergeGroupKey{categoryID: p.CookieCategoryID, trackerType: p.TrackerType, template: tmpl, durationBucket: bucket}
|
|
|
|
mk := memberKey{key, p}
|
|
if !seen[mk] {
|
|
seen[mk] = true
|
|
|
|
templateCounts[key] = append(templateCounts[key], p)
|
|
}
|
|
|
|
heuristicKeys[key] = true
|
|
}
|
|
|
|
for _, tmpl := range templateCandidates(p.Pattern) {
|
|
key := mergeGroupKey{categoryID: p.CookieCategoryID, trackerType: p.TrackerType, template: tmpl, durationBucket: bucket}
|
|
|
|
mk := memberKey{key, p}
|
|
if !seen[mk] {
|
|
seen[mk] = true
|
|
|
|
templateCounts[key] = append(templateCounts[key], p)
|
|
}
|
|
}
|
|
}
|
|
|
|
type candidate struct {
|
|
key mergeGroupKey
|
|
fixedChars int
|
|
isHeuristic bool
|
|
patterns []*coredata.TrackerPattern
|
|
}
|
|
|
|
var candidates []candidate
|
|
|
|
for key, pats := range templateCounts {
|
|
isH := heuristicKeys[key]
|
|
|
|
effectiveThreshold := threshold
|
|
if isH {
|
|
effectiveThreshold = 1
|
|
}
|
|
|
|
if len(pats) >= effectiveThreshold {
|
|
candidates = append(candidates, candidate{key, len(strings.ReplaceAll(key.template, "*", "")), isH, pats})
|
|
}
|
|
}
|
|
|
|
// Sort: heuristic first, then descending specificity (more fixed
|
|
// characters), then descending coverage, then template name for a
|
|
// fully deterministic order.
|
|
sort.Slice(
|
|
candidates,
|
|
func(i, j int) bool {
|
|
if candidates[i].isHeuristic != candidates[j].isHeuristic {
|
|
return candidates[i].isHeuristic
|
|
}
|
|
|
|
if candidates[i].fixedChars != candidates[j].fixedChars {
|
|
return candidates[i].fixedChars > candidates[j].fixedChars
|
|
}
|
|
|
|
if len(candidates[i].patterns) != len(candidates[j].patterns) {
|
|
return len(candidates[i].patterns) > len(candidates[j].patterns)
|
|
}
|
|
|
|
return candidates[i].key.template < candidates[j].key.template
|
|
},
|
|
)
|
|
|
|
assigned := make(map[*coredata.TrackerPattern]bool)
|
|
groups := make(map[mergeGroupKey][]*coredata.TrackerPattern)
|
|
|
|
for _, c := range candidates {
|
|
effectiveThreshold := threshold
|
|
if c.isHeuristic {
|
|
effectiveThreshold = 1
|
|
}
|
|
|
|
var unassigned []*coredata.TrackerPattern
|
|
|
|
for _, p := range c.patterns {
|
|
if !assigned[p] {
|
|
unassigned = append(unassigned, p)
|
|
}
|
|
}
|
|
|
|
if len(unassigned) < effectiveThreshold {
|
|
continue
|
|
}
|
|
|
|
groups[c.key] = unassigned
|
|
for _, p := range unassigned {
|
|
assigned[p] = true
|
|
}
|
|
}
|
|
|
|
return groups
|
|
}
|
|
|
|
func heuristicTemplate(name string) (string, bool) {
|
|
tokens, seps := splitTokens(name)
|
|
if len(seps) == 0 {
|
|
return "", false
|
|
}
|
|
|
|
// Trim leading empty tokens (e.g. "__Secure-..." yields ["", "", ...]).
|
|
var prefix strings.Builder
|
|
for len(tokens) > 1 && tokens[0] == "" {
|
|
prefix.WriteString(string(seps[0]))
|
|
|
|
tokens = tokens[1:]
|
|
seps = seps[1:]
|
|
}
|
|
|
|
// Trim trailing empty tokens.
|
|
var suffix string
|
|
for len(tokens) > 1 && tokens[len(tokens)-1] == "" {
|
|
suffix = string(seps[len(seps)-1]) + suffix
|
|
tokens = tokens[:len(tokens)-1]
|
|
seps = seps[:len(seps)-1]
|
|
}
|
|
|
|
if len(seps) == 0 {
|
|
return "", false
|
|
}
|
|
|
|
changed := false
|
|
|
|
var (
|
|
resultTokens []string
|
|
resultSeps []byte
|
|
)
|
|
|
|
for i, t := range tokens {
|
|
if looksVariable(t) {
|
|
changed = true
|
|
|
|
if len(resultTokens) == 0 || resultTokens[len(resultTokens)-1] != "*" {
|
|
if i > 0 {
|
|
resultSeps = append(resultSeps, seps[i-1])
|
|
}
|
|
|
|
resultTokens = append(resultTokens, "*")
|
|
}
|
|
} else {
|
|
if i > 0 {
|
|
resultSeps = append(resultSeps, seps[i-1])
|
|
}
|
|
|
|
resultTokens = append(resultTokens, t)
|
|
}
|
|
}
|
|
|
|
if !changed {
|
|
return "", false
|
|
}
|
|
|
|
tmpl := prefix.String() + joinTokens(resultTokens, resultSeps) + suffix
|
|
if !templateHasFixedAnchor(tmpl) {
|
|
return "", false
|
|
}
|
|
|
|
return tmpl, true
|
|
}
|
|
|
|
func templateCandidates(name string) []string {
|
|
var candidates []string
|
|
|
|
for i, ch := range name {
|
|
if ch == '_' || ch == '-' {
|
|
tmpl := name[:i+1] + "*"
|
|
if templateHasFixedAnchor(tmpl) {
|
|
candidates = append(candidates, tmpl)
|
|
}
|
|
}
|
|
}
|
|
|
|
tokens, seps := splitTokens(name)
|
|
if len(tokens) >= 3 && len(seps) > 0 {
|
|
for pos := 1; pos < len(tokens)-1; pos++ {
|
|
left := joinTokens(tokens[:pos], seps[:pos-1])
|
|
right := joinTokens(tokens[pos+1:], seps[pos+1:])
|
|
|
|
tmpl := left + string(seps[pos-1]) + "*" + string(seps[pos]) + right
|
|
if templateHasFixedAnchor(tmpl) {
|
|
candidates = append(candidates, tmpl)
|
|
}
|
|
}
|
|
}
|
|
|
|
return candidates
|
|
}
|
|
|
|
func looksVariable(token string) bool {
|
|
if len(token) == 0 {
|
|
return false
|
|
}
|
|
|
|
hasDigit := false
|
|
hasLetter := false
|
|
allHex := true
|
|
allDigits := true
|
|
|
|
for _, ch := range token {
|
|
switch {
|
|
case ch >= '0' && ch <= '9':
|
|
hasDigit = true
|
|
case ch >= 'a' && ch <= 'f', ch >= 'A' && ch <= 'F':
|
|
hasLetter = true
|
|
allDigits = false
|
|
case ch >= 'g' && ch <= 'z', ch >= 'G' && ch <= 'Z':
|
|
hasLetter = true
|
|
allHex = false
|
|
allDigits = false
|
|
case ch == '-':
|
|
allHex = false
|
|
allDigits = false
|
|
default:
|
|
allHex = false
|
|
allDigits = false
|
|
}
|
|
}
|
|
|
|
if len(token) >= 8 && hasDigit && hasLetter {
|
|
return true
|
|
}
|
|
|
|
if len(token) >= 16 && allHex && hasDigit {
|
|
return true
|
|
}
|
|
|
|
if isUUIDShape(token) {
|
|
return true
|
|
}
|
|
|
|
if len(token) >= 8 && allDigits {
|
|
return true
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func isUUIDShape(s string) bool {
|
|
if len(s) != 36 {
|
|
return false
|
|
}
|
|
|
|
for i, ch := range s {
|
|
if i == 8 || i == 13 || i == 18 || i == 23 {
|
|
if ch != '-' {
|
|
return false
|
|
}
|
|
|
|
continue
|
|
}
|
|
|
|
if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') && (ch < 'A' || ch > 'F') {
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func splitTokens(name string) ([]string, []byte) {
|
|
primaryParts, primarySeps := splitOnAny(name, primarySeparators)
|
|
|
|
var (
|
|
tokens []string
|
|
seps []byte
|
|
)
|
|
|
|
for i, part := range primaryParts {
|
|
if i > 0 {
|
|
seps = append(seps, primarySeps[i-1])
|
|
}
|
|
|
|
if isUUIDShape(part) || !strings.Contains(part, "-") {
|
|
tokens = append(tokens, part)
|
|
} else {
|
|
for j, sub := range strings.Split(part, "-") {
|
|
if j > 0 {
|
|
seps = append(seps, '-')
|
|
}
|
|
|
|
tokens = append(tokens, sub)
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(seps) == 0 {
|
|
return []string{name}, nil
|
|
}
|
|
|
|
return tokens, seps
|
|
}
|
|
|
|
// splitOnAny splits s on every byte found in separators, returning the
|
|
// parts and the separator byte that preceded each part after the first.
|
|
// len(seps) == len(parts)-1.
|
|
func splitOnAny(s, separators string) ([]string, []byte) {
|
|
var (
|
|
parts []string
|
|
seps []byte
|
|
start int
|
|
)
|
|
|
|
for i := 0; i < len(s); i++ {
|
|
if strings.IndexByte(separators, s[i]) >= 0 {
|
|
parts = append(parts, s[start:i])
|
|
seps = append(seps, s[i])
|
|
start = i + 1
|
|
}
|
|
}
|
|
|
|
parts = append(parts, s[start:])
|
|
|
|
return parts, seps
|
|
}
|
|
|
|
func joinTokens(tokens []string, seps []byte) string {
|
|
var b strings.Builder
|
|
|
|
for i, t := range tokens {
|
|
if i > 0 {
|
|
b.WriteByte(seps[i-1])
|
|
}
|
|
|
|
b.WriteString(t)
|
|
}
|
|
|
|
return b.String()
|
|
}
|
|
|
|
// templateHasFixedAnchor reports whether tmpl contains at least one
|
|
// character beyond separators and wildcards. Templates like "_*",
|
|
// "__*", "-*", "--*", "__*__" would merge unrelated third parties
|
|
// (e.g. __support__, __darkreader__wasEnabledForHost,
|
|
// __EXT_APP_REFRESH_BLACK_SUB_DOMAINS__) under a single glob, so
|
|
// candidates without any fixed alphanumeric anchor are rejected. The
|
|
// separators recognised here mirror primarySeparators (plus the
|
|
// UUID-internal "-") so a separator-only template such as ":.*" is
|
|
// rejected too.
|
|
func templateHasFixedAnchor(tmpl string) bool {
|
|
for _, ch := range tmpl {
|
|
if ch != '*' && ch != '-' && !strings.ContainsRune(primarySeparators, ch) {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
func globMatch(pattern, name string) bool {
|
|
parts := strings.Split(pattern, "*")
|
|
if len(parts) == 1 {
|
|
return pattern == name
|
|
}
|
|
|
|
if !strings.HasPrefix(name, parts[0]) {
|
|
return false
|
|
}
|
|
|
|
name = name[len(parts[0]):]
|
|
|
|
last := parts[len(parts)-1]
|
|
if !strings.HasSuffix(name, last) {
|
|
return false
|
|
}
|
|
|
|
name = name[:len(name)-len(last)]
|
|
|
|
for _, part := range parts[1 : len(parts)-1] {
|
|
idx := strings.Index(name, part)
|
|
if idx == -1 {
|
|
return false
|
|
}
|
|
|
|
name = name[idx+len(part):]
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
// sourceRank converts a CookieSource into a comparable rank that
|
|
// reflects signal strength: SCRIPT > HTTP > EXTENSION > PRE_EXISTING.
|
|
// SCRIPT is a real page tracker observed being written by page JS;
|
|
// HTTP is a real server-set cookie (a Set-Cookie response header),
|
|
// which is page evidence just like SCRIPT and so must outrank both
|
|
// browser-extension state and the PRE_EXISTING catch-all — otherwise a
|
|
// cookie first enumerated as PRE_EXISTING and later re-observed only
|
|
// via Set-Cookie would stay PRE_EXISTING and remain agent-skipped.
|
|
// EXTENSION is high-confidence extension state; PRE_EXISTING (and nil)
|
|
// is the low-signal catch-all.
|
|
func sourceRank(s *coredata.CookieSource) int {
|
|
if s == nil {
|
|
return 0
|
|
}
|
|
|
|
switch *s {
|
|
case coredata.CookieSourceScript:
|
|
return 3
|
|
case coredata.CookieSourceHTTP:
|
|
return 2
|
|
case coredata.CookieSourceExtension:
|
|
return 1
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
|
|
// shouldPromoteSource reports whether candidate represents a stronger
|
|
// signal than existing under the SCRIPT > EXTENSION > PRE_EXISTING
|
|
// precedence used across the cookie-banner pipeline. Equal ranks do
|
|
// not promote so we avoid pointless writes.
|
|
func shouldPromoteSource(existing, candidate *coredata.CookieSource) bool {
|
|
return sourceRank(candidate) > sourceRank(existing)
|
|
}
|
|
|
|
// bestSource rolls up the source values of a group of exact patterns
|
|
// being merged into a single glob. Precedence is SCRIPT > HTTP >
|
|
// EXTENSION > PRE_EXISTING, mirroring the asymmetric signal strength of
|
|
// each bucket: SCRIPT is high-confidence page evidence (a real page
|
|
// tracker), HTTP is a real server-set cookie (a Set-Cookie response
|
|
// header) and is page evidence too, EXTENSION is high-confidence
|
|
// extension state, and PRE_EXISTING is the catch-all that may include
|
|
// extension state injected before SDK load. nil collapses into
|
|
// PRE_EXISTING.
|
|
func bestSource(patterns []*coredata.TrackerPattern) *coredata.CookieSource {
|
|
var (
|
|
hasHTTP bool
|
|
hasExtension bool
|
|
)
|
|
|
|
for _, p := range patterns {
|
|
if p.Source == nil {
|
|
continue
|
|
}
|
|
|
|
switch *p.Source {
|
|
case coredata.CookieSourceScript:
|
|
return p.Source
|
|
case coredata.CookieSourceHTTP:
|
|
hasHTTP = true
|
|
case coredata.CookieSourceExtension:
|
|
hasExtension = true
|
|
}
|
|
}
|
|
|
|
switch {
|
|
case hasHTTP:
|
|
src := coredata.CookieSourceHTTP
|
|
return &src
|
|
case hasExtension:
|
|
src := coredata.CookieSourceExtension
|
|
return &src
|
|
default:
|
|
src := coredata.CookieSourcePreExisting
|
|
return &src
|
|
}
|
|
}
|
|
|
|
// inheritedMapping rolls up the resolved org ThirdParty of a group of
|
|
// exact patterns being merged into a glob, so the glob can be seeded
|
|
// instead of re-mapped from scratch. It returns a third party only when
|
|
// every resolved member agrees on a single id: a conflicting group (or
|
|
// one with no resolved member) returns nil, leaving the glob blank for a
|
|
// fresh mapping pass. When a third party is chosen, the description of
|
|
// the first member carrying that same id with non-empty text is returned
|
|
// too; the catalog link (common_tracker_pattern_id) is deliberately not
|
|
// inherited, since it is keyed on the exact pattern string rather than
|
|
// the glob template and the mapping worker derives the right row itself.
|
|
func inheritedMapping(patterns []*coredata.TrackerPattern) (*gid.GID, string) {
|
|
var thirdPartyID *gid.GID
|
|
|
|
for _, p := range patterns {
|
|
if p.ThirdPartyID == nil {
|
|
continue
|
|
}
|
|
|
|
if thirdPartyID == nil {
|
|
thirdPartyID = p.ThirdPartyID
|
|
continue
|
|
}
|
|
|
|
if *thirdPartyID != *p.ThirdPartyID {
|
|
return nil, ""
|
|
}
|
|
}
|
|
|
|
if thirdPartyID == nil {
|
|
return nil, ""
|
|
}
|
|
|
|
for _, p := range patterns {
|
|
if p.ThirdPartyID != nil && *p.ThirdPartyID == *thirdPartyID && p.Description != "" {
|
|
return thirdPartyID, p.Description
|
|
}
|
|
}
|
|
|
|
return thirdPartyID, ""
|
|
}
|
|
|
|
func (h *patternAnalysisHandler) adoptUncategorisedPatterns(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
scope coredata.Scoper,
|
|
banner coredata.CookieBanner,
|
|
) (bool, error) {
|
|
var uncategorised coredata.CookieCategory
|
|
if err := uncategorised.LoadUncategorisedByCookieBannerID(ctx, tx, scope, banner.ID); err != nil {
|
|
if errors.Is(err, coredata.ErrResourceNotFound) {
|
|
return false, nil
|
|
}
|
|
|
|
return false, fmt.Errorf("cannot load uncategorised category: %w", err)
|
|
}
|
|
|
|
globPatterns, err := page.LoadAll(
|
|
ctx,
|
|
page.OrderBy[coredata.TrackerPatternOrderField]{
|
|
Field: coredata.TrackerPatternOrderFieldCreatedAt,
|
|
Direction: page.OrderDirectionAsc,
|
|
},
|
|
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
|
|
var batch coredata.TrackerPatterns
|
|
if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(new(coredata.TrackerPatternMatchTypeGlob), nil, new(false))); err != nil {
|
|
return nil, fmt.Errorf("cannot load glob patterns: %w", err)
|
|
}
|
|
|
|
return batch, nil
|
|
},
|
|
)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
if len(globPatterns) == 0 {
|
|
return false, nil
|
|
}
|
|
|
|
sort.Slice(
|
|
globPatterns,
|
|
func(i, j int) bool {
|
|
return len(globPatterns[i].Pattern) > len(globPatterns[j].Pattern)
|
|
},
|
|
)
|
|
|
|
exactMatchType := coredata.TrackerPatternMatchTypeExact
|
|
|
|
uncategorisedExact, err := page.LoadAll(
|
|
ctx,
|
|
page.OrderBy[coredata.TrackerPatternOrderField]{
|
|
Field: coredata.TrackerPatternOrderFieldCreatedAt,
|
|
Direction: page.OrderDirectionAsc,
|
|
},
|
|
func(ctx context.Context, cursor *page.Cursor[coredata.TrackerPatternOrderField]) ([]*coredata.TrackerPattern, error) {
|
|
var batch coredata.TrackerPatterns
|
|
if err := batch.LoadByCookieBannerID(ctx, tx, scope, banner.ID, cursor, coredata.NewTrackerPatternFilter(&exactMatchType, &uncategorised.ID, new(false))); err != nil {
|
|
return nil, fmt.Errorf("cannot load uncategorised exact patterns: %w", err)
|
|
}
|
|
|
|
return batch, nil
|
|
},
|
|
)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
adopted := false
|
|
|
|
for _, ep := range uncategorisedExact {
|
|
var match *coredata.TrackerPattern
|
|
|
|
epBucket := durationBucket(ep.MaxAgeSeconds)
|
|
for _, gp := range globPatterns {
|
|
if ep.TrackerType == gp.TrackerType && globMatch(gp.Pattern, ep.Pattern) && durationBucket(gp.MaxAgeSeconds) == epBucket {
|
|
match = gp
|
|
break
|
|
}
|
|
}
|
|
|
|
if match == nil {
|
|
continue
|
|
}
|
|
|
|
var trackers coredata.DetectedTrackers
|
|
if err := trackers.RelinkByTrackerPatternID(ctx, tx, scope, ep.ID, match.ID); err != nil {
|
|
return false, fmt.Errorf("cannot relink detected trackers from pattern %q: %w", ep.Pattern, err)
|
|
}
|
|
|
|
// Promote the glob's source if this exact carries a
|
|
// stronger signal. The merge loop already handles
|
|
// promotion when the glob shares the exact's category,
|
|
// but adoption is the only path that can lift a glob
|
|
// the user (or an earlier pass) has placed in a
|
|
// non-uncategorised category. Without this, a
|
|
// PRE_EXISTING glob never advances to SCRIPT/EXTENSION
|
|
// even though new SDK-observed exacts confirm the
|
|
// stronger signal. We mutate match.Source in place
|
|
// before calling Update so subsequent adoptions against
|
|
// the same glob ratchet correctly (PRE_EXISTING →
|
|
// EXTENSION → SCRIPT) without redundant writes.
|
|
if shouldPromoteSource(match.Source, ep.Source) {
|
|
match.Source = ep.Source
|
|
match.UpdatedAt = time.Now()
|
|
|
|
if err := match.Update(ctx, tx, scope); err != nil {
|
|
return false, fmt.Errorf("cannot promote source on glob pattern %q: %w", match.Pattern, err)
|
|
}
|
|
|
|
// A stronger source can unblock mapping (a fresh
|
|
// initiator domain lets matchByDomain/matchBySiblingOrigin
|
|
// resolve a vendor), so re-arm mapping on the adopted glob.
|
|
if err := match.SetMappingRequested(ctx, tx); err != nil {
|
|
return false, fmt.Errorf("cannot request mapping after source promotion on glob pattern %q: %w", match.Pattern, err)
|
|
}
|
|
}
|
|
|
|
if err := ep.Delete(ctx, tx, scope); err != nil {
|
|
return false, fmt.Errorf("cannot delete adopted exact pattern %q: %w", ep.Pattern, err)
|
|
}
|
|
|
|
adopted = true
|
|
|
|
h.logger.InfoCtx(
|
|
ctx,
|
|
"adopted uncategorised exact pattern into glob pattern",
|
|
log.String("exact_pattern", ep.Pattern),
|
|
log.String("glob_pattern", match.Pattern),
|
|
log.String("banner_id", banner.ID.String()),
|
|
)
|
|
}
|
|
|
|
return adopted, nil
|
|
}
|