From 93cf5a59862d4de3f8f681e94536fdc8ffeace5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89mile=20R=C3=A9?= Date: Wed, 6 May 2026 17:12:56 +0400 Subject: [PATCH] Add duration-aware tracker pattern merging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trackers sharing a prefix but with materially different lifetimes (e.g. session vs 1-year) were incorrectly merged into a single prefix pattern. Port the snap table from cookie-utils.ts into Go and use it to bucket durations so only trackers that display the same human-readable lifetime can merge. Update the unique index to include COALESCE(max_age_seconds, -1) so prefix patterns with different durations can coexist. Signed-off-by: Émile Ré --- pkg/cookiebanner/worker.go | 101 +++++--- pkg/cookiebanner/worker_test.go | 245 +++++++++++++++---- pkg/coredata/migrations/20260506T123706Z.sql | 18 ++ pkg/coredata/tracker_pattern.go | 5 +- 4 files changed, 286 insertions(+), 83 deletions(-) create mode 100644 pkg/coredata/migrations/20260506T123706Z.sql diff --git a/pkg/cookiebanner/worker.go b/pkg/cookiebanner/worker.go index b77ceb352..fe5d5e6b4 100644 --- a/pkg/cookiebanner/worker.go +++ b/pkg/cookiebanner/worker.go @@ -31,6 +31,53 @@ import ( const patternMergeThreshold = 3 +// durationUnits mirrors the snap table from cookie-utils.ts. The same +// tracker observed across different clients can have jitter in its +// max-age (e.g. an "Expires" header computed from Date.now() yields +// slightly different seconds each time). Snapping to the nearest +// human-meaningful unit absorbs that jitter so the patterns still +// merge. This is compliant because the resulting bucket matches the +// duration shown to end users in the cookie banner — two cookies that +// display the same human-readable lifetime will merge, two that +// display differently will not. +var durationUnits = [...]struct { + seconds int + snap int +}{ + {365 * 24 * 3600, 21 * 24 * 3600}, // years, snap +-21 days + {30 * 24 * 3600, 2 * 24 * 3600}, // months, snap +-2 days + {7 * 24 * 3600, 12 * 3600}, // weeks, snap +-12 hours + {24 * 3600, 2 * 3600}, // days, snap +-2 hours + {3600, 5 * 60}, // hours, snap +-5 minutes + {60, 5}, // minutes, snap +-5 seconds + {1, 0}, // seconds, no snap +} + +func durationBucket(maxAge *int) int { + if maxAge == nil || *maxAge <= 0 { + return -1 + } + + remaining := *maxAge + total := 0 + for _, u := range durationUnits { + if remaining >= u.seconds-u.snap { + count := remaining / u.seconds + leftover := remaining - count*u.seconds + if leftover >= u.seconds-u.snap { + count++ + remaining = 0 + } else if leftover <= u.snap { + remaining = 0 + } else { + remaining = leftover + } + total += count * u.seconds + } + } + return total +} + type patternAnalysisHandler struct { svc *Service pg *pg.Client @@ -110,8 +157,12 @@ func (h *patternAnalysisHandler) Process(ctx context.Context, banner coredata.Co consentChanged := false for key, group := range mergeGroups { + var maxAge *int + if key.durationBucket >= 0 { + v := key.durationBucket + maxAge = &v + } - maxAge := mostCommonMaxAge(group) source := bestSource(group) prefixPattern := &coredata.TrackerPattern{ @@ -135,7 +186,7 @@ func (h *patternAnalysisHandler) Process(ctx context.Context, banner coredata.Co return fmt.Errorf("cannot insert prefix pattern %q: %w", key.prefix, err) } if !inserted { - if err := prefixPattern.LoadByBannerIDTypeAndPattern(ctx, tx, scope, banner.ID, key.trackerType, key.prefix); err != nil { + if err := prefixPattern.LoadByBannerIDTypeAndPattern(ctx, tx, scope, banner.ID, key.trackerType, key.prefix, maxAge); err != nil { return fmt.Errorf("cannot load existing prefix pattern %q: %w", key.prefix, err) } @@ -189,9 +240,10 @@ func (h *patternAnalysisHandler) Process(ctx context.Context, banner coredata.Co } type mergeGroupKey struct { - categoryID gid.GID - trackerType coredata.TrackerType - prefix string + categoryID gid.GID + trackerType coredata.TrackerType + prefix string + durationBucket int } func findMergeGroups( @@ -200,8 +252,9 @@ func findMergeGroups( ) map[mergeGroupKey][]*coredata.TrackerPattern { prefixCounts := make(map[mergeGroupKey][]*coredata.TrackerPattern) for _, p := range patterns { + bucket := durationBucket(p.MaxAgeSeconds) for _, pfx := range separatorPrefixes(p.Pattern) { - key := mergeGroupKey{categoryID: p.CookieCategoryID, trackerType: p.TrackerType, prefix: pfx} + key := mergeGroupKey{categoryID: p.CookieCategoryID, trackerType: p.TrackerType, prefix: pfx, durationBucket: bucket} prefixCounts[key] = append(prefixCounts[key], p) } } @@ -266,39 +319,6 @@ func bestSource(patterns []*coredata.TrackerPattern) *coredata.CookieSource { return &src } -func mostCommonMaxAge(patterns []*coredata.TrackerPattern) *int { - type key struct { - valid bool - val int - } - counts := make(map[key]int) - for _, p := range patterns { - k := key{} - if p.MaxAgeSeconds != nil { - k = key{valid: true, val: *p.MaxAgeSeconds} - } - counts[k]++ - } - - type entry struct { - k key - count int - } - entries := make([]entry, 0, len(counts)) - for k, c := range counts { - entries = append(entries, entry{k, c}) - } - sort.Slice(entries, func(i, j int) bool { - return entries[i].count > entries[j].count - }) - - if !entries[0].k.valid { - return nil - } - v := entries[0].k.val - return &v -} - func (h *patternAnalysisHandler) adoptUncategorisedPatterns( ctx context.Context, tx pg.Tx, @@ -349,8 +369,9 @@ func (h *patternAnalysisHandler) adoptUncategorisedPatterns( adopted := false for _, ep := range uncategorisedExact { var match *coredata.TrackerPattern + epBucket := durationBucket(ep.MaxAgeSeconds) for _, pp := range prefixPatterns { - if ep.TrackerType == pp.TrackerType && strings.HasPrefix(ep.Pattern, pp.Pattern) { + if ep.TrackerType == pp.TrackerType && strings.HasPrefix(ep.Pattern, pp.Pattern) && durationBucket(pp.MaxAgeSeconds) == epBucket { match = pp break } diff --git a/pkg/cookiebanner/worker_test.go b/pkg/cookiebanner/worker_test.go index ba82256bb..353911bd2 100644 --- a/pkg/cookiebanner/worker_test.go +++ b/pkg/cookiebanner/worker_test.go @@ -88,11 +88,14 @@ func TestSeparatorPrefixes(t *testing.T) { func TestFindMergeGroups(t *testing.T) { t.Parallel() - makePattern := func(name string) *coredata.TrackerPattern { + oneYear := 365 * 24 * 3600 + + makePattern := func(name string, maxAge *int) *coredata.TrackerPattern { return &coredata.TrackerPattern{ - Pattern: name, - TrackerType: coredata.TrackerTypeCookie, - MatchType: coredata.TrackerPatternMatchTypeExact, + Pattern: name, + TrackerType: coredata.TrackerTypeCookie, + MatchType: coredata.TrackerPatternMatchTypeExact, + MaxAgeSeconds: maxAge, } } @@ -102,15 +105,15 @@ func TestFindMergeGroups(t *testing.T) { t.Parallel() patterns := coredata.TrackerPatterns{ - makePattern("ph_phc_abc123"), - makePattern("ph_phc_def456"), - makePattern("ph_phc_ghi789"), + makePattern("ph_phc_abc123", &oneYear), + makePattern("ph_phc_def456", &oneYear), + makePattern("ph_phc_ghi789", &oneYear), } groups := findMergeGroups(patterns, 3) require.Len(t, groups, 1) - group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_phc_"}] + group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_phc_", durationBucket: durationBucket(&oneYear)}] require.True(t, ok) assert.Len(t, group, 3) }, @@ -122,15 +125,15 @@ func TestFindMergeGroups(t *testing.T) { t.Parallel() patterns := coredata.TrackerPatterns{ - makePattern("_ga_ABC123"), - makePattern("_ga_DEF456"), - makePattern("_ga_GHI789"), + makePattern("_ga_ABC123", &oneYear), + makePattern("_ga_DEF456", &oneYear), + makePattern("_ga_GHI789", &oneYear), } groups := findMergeGroups(patterns, 3) require.Len(t, groups, 1) - group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "_ga_"}] + group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "_ga_", durationBucket: durationBucket(&oneYear)}] require.True(t, ok) assert.Len(t, group, 3) }, @@ -142,15 +145,15 @@ func TestFindMergeGroups(t *testing.T) { t.Parallel() patterns := coredata.TrackerPatterns{ - makePattern("auth0_session_abc123"), - makePattern("auth0_session_def456"), - makePattern("auth0_session_ghi789"), + makePattern("auth0_session_abc123", &oneYear), + makePattern("auth0_session_def456", &oneYear), + makePattern("auth0_session_ghi789", &oneYear), } groups := findMergeGroups(patterns, 3) require.Len(t, groups, 1) - group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "auth0_session_"}] + group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "auth0_session_", durationBucket: durationBucket(&oneYear)}] require.True(t, ok) assert.Len(t, group, 3) }, @@ -162,8 +165,8 @@ func TestFindMergeGroups(t *testing.T) { t.Parallel() patterns := coredata.TrackerPatterns{ - makePattern("deadbeef_setting"), - makePattern("something_else"), + makePattern("deadbeef_setting", &oneYear), + makePattern("something_else", &oneYear), } groups := findMergeGroups(patterns, 3) @@ -177,22 +180,22 @@ func TestFindMergeGroups(t *testing.T) { t.Parallel() patterns := coredata.TrackerPatterns{ - makePattern("foo_bar_aaa"), - makePattern("foo_bar_bbb"), - makePattern("foo_bar_ccc"), - makePattern("foo_baz_xxx"), - makePattern("foo_baz_yyy"), - makePattern("foo_baz_zzz"), + makePattern("foo_bar_aaa", &oneYear), + makePattern("foo_bar_bbb", &oneYear), + makePattern("foo_bar_ccc", &oneYear), + makePattern("foo_baz_xxx", &oneYear), + makePattern("foo_baz_yyy", &oneYear), + makePattern("foo_baz_zzz", &oneYear), } groups := findMergeGroups(patterns, 3) require.Len(t, groups, 2) - barGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "foo_bar_"}] + barGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "foo_bar_", durationBucket: durationBucket(&oneYear)}] require.True(t, ok) assert.Len(t, barGroup, 3) - bazGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "foo_baz_"}] + bazGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "foo_baz_", durationBucket: durationBucket(&oneYear)}] require.True(t, ok) assert.Len(t, bazGroup, 3) }, @@ -204,16 +207,16 @@ func TestFindMergeGroups(t *testing.T) { t.Parallel() patterns := coredata.TrackerPatterns{ - makePattern("ph_phc_abc123"), - makePattern("ph_phc_def456"), - makePattern("ph_phc_ghi789"), - makePattern("ph_session_xyz"), + makePattern("ph_phc_abc123", &oneYear), + makePattern("ph_phc_def456", &oneYear), + makePattern("ph_phc_ghi789", &oneYear), + makePattern("ph_session_xyz", &oneYear), } groups := findMergeGroups(patterns, 3) require.Len(t, groups, 1) - group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_phc_"}] + group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_phc_", durationBucket: durationBucket(&oneYear)}] require.True(t, ok) assert.Len(t, group, 3) }, @@ -225,22 +228,22 @@ func TestFindMergeGroups(t *testing.T) { t.Parallel() patterns := coredata.TrackerPatterns{ - makePattern("ph_phc_abc123"), - makePattern("ph_phc_def456"), - makePattern("ph_phc_ghi789"), - makePattern("ph_session_aaa"), - makePattern("ph_session_bbb"), - makePattern("ph_session_ccc"), + makePattern("ph_phc_abc123", &oneYear), + makePattern("ph_phc_def456", &oneYear), + makePattern("ph_phc_ghi789", &oneYear), + makePattern("ph_session_aaa", &oneYear), + makePattern("ph_session_bbb", &oneYear), + makePattern("ph_session_ccc", &oneYear), } groups := findMergeGroups(patterns, 3) require.Len(t, groups, 2) - phcGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_phc_"}] + phcGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_phc_", durationBucket: durationBucket(&oneYear)}] require.True(t, ok) assert.Len(t, phcGroup, 3) - sessionGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_session_"}] + sessionGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_session_", durationBucket: durationBucket(&oneYear)}] require.True(t, ok) assert.Len(t, sessionGroup, 3) }, @@ -252,13 +255,171 @@ func TestFindMergeGroups(t *testing.T) { t.Parallel() patterns := coredata.TrackerPatterns{ - makePattern("PHPSESSID"), - makePattern("JSESSIONID"), - makePattern("ASPSESSIONID"), + makePattern("PHPSESSID", nil), + makePattern("JSESSIONID", nil), + makePattern("ASPSESSIONID", nil), } groups := findMergeGroups(patterns, 3) assert.Empty(t, groups) }, ) + + t.Run( + "session and persistent cookies do not merge", + func(t *testing.T) { + t.Parallel() + + patterns := coredata.TrackerPatterns{ + makePattern("_ga_ABC123", nil), + makePattern("_ga_DEF456", nil), + makePattern("_ga_GHI789", nil), + makePattern("_ga_JKL012", &oneYear), + makePattern("_ga_MNO345", &oneYear), + makePattern("_ga_PQR678", &oneYear), + } + + groups := findMergeGroups(patterns, 3) + require.Len(t, groups, 2) + + sessionGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "_ga_", durationBucket: -1}] + require.True(t, ok) + assert.Len(t, sessionGroup, 3) + + persistentGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "_ga_", durationBucket: durationBucket(&oneYear)}] + require.True(t, ok) + assert.Len(t, persistentGroup, 3) + }, + ) + + t.Run( + "close durations snap to same bucket and merge", + func(t *testing.T) { + t.Parallel() + + exactYear := 365 * 24 * 3600 + almostYear := 364 * 24 * 3600 + + patterns := coredata.TrackerPatterns{ + makePattern("_ga_ABC123", &exactYear), + makePattern("_ga_DEF456", &almostYear), + makePattern("_ga_GHI789", &exactYear), + } + + groups := findMergeGroups(patterns, 3) + require.Len(t, groups, 1) + }, + ) + + t.Run( + "different durations do not merge", + func(t *testing.T) { + t.Parallel() + + oneDay := 24 * 3600 + thirtyDays := 30 * 24 * 3600 + + patterns := coredata.TrackerPatterns{ + makePattern("_ga_ABC123", &oneDay), + makePattern("_ga_DEF456", &oneDay), + makePattern("_ga_GHI789", &oneDay), + makePattern("_ga_JKL012", &thirtyDays), + makePattern("_ga_MNO345", &thirtyDays), + makePattern("_ga_PQR678", &thirtyDays), + } + + groups := findMergeGroups(patterns, 3) + require.Len(t, groups, 2) + + dayGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "_ga_", durationBucket: durationBucket(&oneDay)}] + require.True(t, ok) + assert.Len(t, dayGroup, 3) + + monthGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "_ga_", durationBucket: durationBucket(&thirtyDays)}] + require.True(t, ok) + assert.Len(t, monthGroup, 3) + }, + ) +} + +func TestDurationBucket(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + maxAge *int + expected int + }{ + { + name: "nil is session", + maxAge: nil, + expected: -1, + }, + { + name: "zero is session", + maxAge: new(0), + expected: -1, + }, + { + name: "negative is session", + maxAge: new(-1), + expected: -1, + }, + { + name: "exact 1 year", + maxAge: new(365 * 24 * 3600), + expected: 365 * 24 * 3600, + }, + { + name: "364 days snaps to 1 year", + maxAge: new(364 * 24 * 3600), + expected: 365 * 24 * 3600, + }, + { + name: "exact 30 days", + maxAge: new(30 * 24 * 3600), + expected: 30 * 24 * 3600, + }, + { + name: "exact 1 day", + maxAge: new(24 * 3600), + expected: 24 * 3600, + }, + { + name: "23h snaps to 1 day", + maxAge: new(23 * 3600), + expected: 24 * 3600, + }, + { + name: "exact 1 hour", + maxAge: new(3600), + expected: 3600, + }, + { + name: "58 minutes snaps to 1 hour", + maxAge: new(58 * 60), + expected: 3600, + }, + { + name: "exact 5 minutes", + maxAge: new(5 * 60), + expected: 5 * 60, + }, + { + name: "1 day and 30 days are different buckets", + maxAge: new(24 * 3600), + expected: 24 * 3600, + }, + } + + for _, tt := range tests { + t.Run( + tt.name, + func(t *testing.T) { + t.Parallel() + result := durationBucket(tt.maxAge) + assert.Equal(t, tt.expected, result) + }, + ) + } } diff --git a/pkg/coredata/migrations/20260506T123706Z.sql b/pkg/coredata/migrations/20260506T123706Z.sql new file mode 100644 index 000000000..2b7f91abe --- /dev/null +++ b/pkg/coredata/migrations/20260506T123706Z.sql @@ -0,0 +1,18 @@ +-- Copyright (c) 2026 Probo Inc . +-- +-- Permission to use, copy, modify, and/or distribute this software for any +-- purpose with or without fee is hereby granted, provided that the above +-- copyright notice and this permission notice appear in all copies. +-- +-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +-- PERFORMANCE OF THIS SOFTWARE. + +DROP INDEX idx_tracker_patterns_unique_pattern_per_banner; + +CREATE UNIQUE INDEX idx_tracker_patterns_unique_pattern_per_banner + ON tracker_patterns (cookie_banner_id, tracker_type, pattern, COALESCE(max_age_seconds, -1)); diff --git a/pkg/coredata/tracker_pattern.go b/pkg/coredata/tracker_pattern.go index ae214f5e5..6422d1e91 100644 --- a/pkg/coredata/tracker_pattern.go +++ b/pkg/coredata/tracker_pattern.go @@ -149,6 +149,7 @@ func (tp *TrackerPattern) LoadByBannerIDTypeAndPattern( cookieBannerID gid.GID, trackerType TrackerType, pattern string, + maxAgeSeconds *int, ) error { q := ` SELECT @@ -174,6 +175,7 @@ WHERE AND cookie_banner_id = @cookie_banner_id AND tracker_type = @tracker_type AND pattern = @pattern + AND COALESCE(max_age_seconds, -1) = COALESCE(@max_age_seconds, -1) LIMIT 1; ` @@ -183,6 +185,7 @@ LIMIT 1; "cookie_banner_id": cookieBannerID, "tracker_type": trackerType, "pattern": pattern, + "max_age_seconds": maxAgeSeconds, } maps.Copy(args, scope.SQLArguments()) @@ -393,7 +396,7 @@ INSERT INTO tracker_patterns ( @created_at, @updated_at ) -ON CONFLICT (cookie_banner_id, tracker_type, pattern) DO NOTHING +ON CONFLICT (cookie_banner_id, tracker_type, pattern, COALESCE(max_age_seconds, -1)) DO NOTHING ` args := pgx.StrictNamedArgs{