rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add duration-aware tracker pattern merging

Browse Source 
Trackers sharing a prefix but with materially different lifetimes
(e.g. session vs 1-year) were incorrectly merged into a single
prefix pattern. Port the snap table from cookie-utils.ts into Go
and use it to bucket durations so only trackers that display the
same human-readable lifetime can merge. Update the unique index
to include COALESCE(max_age_seconds, -1) so prefix patterns with
different durations can coexist.

Signed-off-by: Émile Ré <emile@getprobo.com>
This commit is contained in:
Émile Ré
2026-05-06 17:12:56 +04:00
parent a911fcc220
commit 93cf5a5986
4 changed files with 286 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
pkg/cookiebanner/worker.go
View File

@@ -31,6 +31,53 @@ import (
const patternMergeThreshold = 3
// durationUnits mirrors the snap table from cookie-utils.ts. The same
// tracker observed across different clients can have jitter in its
// max-age (e.g. an "Expires" header computed from Date.now() yields
// slightly different seconds each time). Snapping to the nearest
// human-meaningful unit absorbs that jitter so the patterns still
// merge. This is compliant because the resulting bucket matches the
// duration shown to end users in the cookie banner — two cookies that
// display the same human-readable lifetime will merge, two that
// display differently will not.
var durationUnits = [...]struct {
seconds int
snap int
}{
{365 * 24 * 3600, 21 * 24 * 3600}, // years, snap +-21 days
{30 * 24 * 3600, 2 * 24 * 3600}, // months, snap +-2 days
{7 * 24 * 3600, 12 * 3600}, // weeks, snap +-12 hours
{24 * 3600, 2 * 3600}, // days, snap +-2 hours
{3600, 5 * 60}, // hours, snap +-5 minutes
{60, 5}, // minutes, snap +-5 seconds
{1, 0}, // seconds, no snap
}
func durationBucket(maxAge *int) int {
if maxAge == nil || *maxAge <= 0 {
return -1
}
remaining := *maxAge
total := 0
for _, u := range durationUnits {
if remaining >= u.seconds-u.snap {
count := remaining / u.seconds
leftover := remaining - count*u.seconds
if leftover >= u.seconds-u.snap {
count++
remaining = 0
} else if leftover <= u.snap {
remaining = 0
} else {
remaining = leftover
}
total += count * u.seconds
}
}
return total
}
type patternAnalysisHandler struct {
svc *Service
pg *pg.Client
@@ -110,8 +157,12 @@ func (h *patternAnalysisHandler) Process(ctx context.Context, banner coredata.Co
consentChanged := false
for key, group := range mergeGroups {
var maxAge *int
if key.durationBucket >= 0 {
v := key.durationBucket
maxAge = &v
}
maxAge := mostCommonMaxAge(group)
source := bestSource(group)
prefixPattern := &coredata.TrackerPattern{
@@ -135,7 +186,7 @@ func (h *patternAnalysisHandler) Process(ctx context.Context, banner coredata.Co
return fmt.Errorf("cannot insert prefix pattern %q: %w", key.prefix, err)
}
if !inserted {
if err := prefixPattern.LoadByBannerIDTypeAndPattern(ctx, tx, scope, banner.ID, key.trackerType, key.prefix); err != nil {
if err := prefixPattern.LoadByBannerIDTypeAndPattern(ctx, tx, scope, banner.ID, key.trackerType, key.prefix, maxAge); err != nil {
return fmt.Errorf("cannot load existing prefix pattern %q: %w", key.prefix, err)
}
@@ -189,9 +240,10 @@ func (h *patternAnalysisHandler) Process(ctx context.Context, banner coredata.Co
}
type mergeGroupKey struct {
categoryID gid.GID
trackerType coredata.TrackerType
prefix string
categoryID gid.GID
trackerType coredata.TrackerType
prefix string
durationBucket int
}
func findMergeGroups(
@@ -200,8 +252,9 @@ func findMergeGroups(
) map[mergeGroupKey][]*coredata.TrackerPattern {
prefixCounts := make(map[mergeGroupKey][]*coredata.TrackerPattern)
for _, p := range patterns {
bucket := durationBucket(p.MaxAgeSeconds)
for _, pfx := range separatorPrefixes(p.Pattern) {
key := mergeGroupKey{categoryID: p.CookieCategoryID, trackerType: p.TrackerType, prefix: pfx}
key := mergeGroupKey{categoryID: p.CookieCategoryID, trackerType: p.TrackerType, prefix: pfx, durationBucket: bucket}
prefixCounts[key] = append(prefixCounts[key], p)
}
}
@@ -266,39 +319,6 @@ func bestSource(patterns []*coredata.TrackerPattern) *coredata.CookieSource {
return &src
}
func mostCommonMaxAge(patterns []*coredata.TrackerPattern) *int {
type key struct {
valid bool
val int
}
counts := make(map[key]int)
for _, p := range patterns {
k := key{}
if p.MaxAgeSeconds != nil {
k = key{valid: true, val: *p.MaxAgeSeconds}
}
counts[k]++
}
type entry struct {
k key
count int
}
entries := make([]entry, 0, len(counts))
for k, c := range counts {
entries = append(entries, entry{k, c})
}
sort.Slice(entries, func(i, j int) bool {
return entries[i].count > entries[j].count
})
if !entries[0].k.valid {
return nil
}
v := entries[0].k.val
return &v
}
func (h *patternAnalysisHandler) adoptUncategorisedPatterns(
ctx context.Context,
tx pg.Tx,
@@ -349,8 +369,9 @@ func (h *patternAnalysisHandler) adoptUncategorisedPatterns(
adopted := false
for _, ep := range uncategorisedExact {
var match *coredata.TrackerPattern
epBucket := durationBucket(ep.MaxAgeSeconds)
for _, pp := range prefixPatterns {
if ep.TrackerType == pp.TrackerType && strings.HasPrefix(ep.Pattern, pp.Pattern) {
if ep.TrackerType == pp.TrackerType && strings.HasPrefix(ep.Pattern, pp.Pattern) && durationBucket(pp.MaxAgeSeconds) == epBucket {
match = pp
break
}

245
pkg/cookiebanner/worker_test.go
View File

@@ -88,11 +88,14 @@ func TestSeparatorPrefixes(t *testing.T) {
func TestFindMergeGroups(t *testing.T) {
t.Parallel()
makePattern := func(name string) *coredata.TrackerPattern {
oneYear := 365 * 24 * 3600
makePattern := func(name string, maxAge *int) *coredata.TrackerPattern {
return &coredata.TrackerPattern{
Pattern: name,
TrackerType: coredata.TrackerTypeCookie,
MatchType: coredata.TrackerPatternMatchTypeExact,
Pattern: name,
TrackerType: coredata.TrackerTypeCookie,
MatchType: coredata.TrackerPatternMatchTypeExact,
MaxAgeSeconds: maxAge,
}
}
@@ -102,15 +105,15 @@ func TestFindMergeGroups(t *testing.T) {
t.Parallel()
patterns := coredata.TrackerPatterns{
makePattern("ph_phc_abc123"),
makePattern("ph_phc_def456"),
makePattern("ph_phc_ghi789"),
makePattern("ph_phc_abc123", &oneYear),
makePattern("ph_phc_def456", &oneYear),
makePattern("ph_phc_ghi789", &oneYear),
}
groups := findMergeGroups(patterns, 3)
require.Len(t, groups, 1)
group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_phc_"}]
group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_phc_", durationBucket: durationBucket(&oneYear)}]
require.True(t, ok)
assert.Len(t, group, 3)
},
@@ -122,15 +125,15 @@ func TestFindMergeGroups(t *testing.T) {
t.Parallel()
patterns := coredata.TrackerPatterns{
makePattern("_ga_ABC123"),
makePattern("_ga_DEF456"),
makePattern("_ga_GHI789"),
makePattern("_ga_ABC123", &oneYear),
makePattern("_ga_DEF456", &oneYear),
makePattern("_ga_GHI789", &oneYear),
}
groups := findMergeGroups(patterns, 3)
require.Len(t, groups, 1)
group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "_ga_"}]
group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "_ga_", durationBucket: durationBucket(&oneYear)}]
require.True(t, ok)
assert.Len(t, group, 3)
},
@@ -142,15 +145,15 @@ func TestFindMergeGroups(t *testing.T) {
t.Parallel()
patterns := coredata.TrackerPatterns{
makePattern("auth0_session_abc123"),
makePattern("auth0_session_def456"),
makePattern("auth0_session_ghi789"),
makePattern("auth0_session_abc123", &oneYear),
makePattern("auth0_session_def456", &oneYear),
makePattern("auth0_session_ghi789", &oneYear),
}
groups := findMergeGroups(patterns, 3)
require.Len(t, groups, 1)
group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "auth0_session_"}]
group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "auth0_session_", durationBucket: durationBucket(&oneYear)}]
require.True(t, ok)
assert.Len(t, group, 3)
},
@@ -162,8 +165,8 @@ func TestFindMergeGroups(t *testing.T) {
t.Parallel()
patterns := coredata.TrackerPatterns{
makePattern("deadbeef_setting"),
makePattern("something_else"),
makePattern("deadbeef_setting", &oneYear),
makePattern("something_else", &oneYear),
}
groups := findMergeGroups(patterns, 3)
@@ -177,22 +180,22 @@ func TestFindMergeGroups(t *testing.T) {
t.Parallel()
patterns := coredata.TrackerPatterns{
makePattern("foo_bar_aaa"),
makePattern("foo_bar_bbb"),
makePattern("foo_bar_ccc"),
makePattern("foo_baz_xxx"),
makePattern("foo_baz_yyy"),
makePattern("foo_baz_zzz"),
makePattern("foo_bar_aaa", &oneYear),
makePattern("foo_bar_bbb", &oneYear),
makePattern("foo_bar_ccc", &oneYear),
makePattern("foo_baz_xxx", &oneYear),
makePattern("foo_baz_yyy", &oneYear),
makePattern("foo_baz_zzz", &oneYear),
}
groups := findMergeGroups(patterns, 3)
require.Len(t, groups, 2)
barGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "foo_bar_"}]
barGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "foo_bar_", durationBucket: durationBucket(&oneYear)}]
require.True(t, ok)
assert.Len(t, barGroup, 3)
bazGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "foo_baz_"}]
bazGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "foo_baz_", durationBucket: durationBucket(&oneYear)}]
require.True(t, ok)
assert.Len(t, bazGroup, 3)
},
@@ -204,16 +207,16 @@ func TestFindMergeGroups(t *testing.T) {
t.Parallel()
patterns := coredata.TrackerPatterns{
makePattern("ph_phc_abc123"),
makePattern("ph_phc_def456"),
makePattern("ph_phc_ghi789"),
makePattern("ph_session_xyz"),
makePattern("ph_phc_abc123", &oneYear),
makePattern("ph_phc_def456", &oneYear),
makePattern("ph_phc_ghi789", &oneYear),
makePattern("ph_session_xyz", &oneYear),
}
groups := findMergeGroups(patterns, 3)
require.Len(t, groups, 1)
group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_phc_"}]
group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_phc_", durationBucket: durationBucket(&oneYear)}]
require.True(t, ok)
assert.Len(t, group, 3)
},
@@ -225,22 +228,22 @@ func TestFindMergeGroups(t *testing.T) {
t.Parallel()
patterns := coredata.TrackerPatterns{
makePattern("ph_phc_abc123"),
makePattern("ph_phc_def456"),
makePattern("ph_phc_ghi789"),
makePattern("ph_session_aaa"),
makePattern("ph_session_bbb"),
makePattern("ph_session_ccc"),
makePattern("ph_phc_abc123", &oneYear),
makePattern("ph_phc_def456", &oneYear),
makePattern("ph_phc_ghi789", &oneYear),
makePattern("ph_session_aaa", &oneYear),
makePattern("ph_session_bbb", &oneYear),
makePattern("ph_session_ccc", &oneYear),
}
groups := findMergeGroups(patterns, 3)
require.Len(t, groups, 2)
phcGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_phc_"}]
phcGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_phc_", durationBucket: durationBucket(&oneYear)}]
require.True(t, ok)
assert.Len(t, phcGroup, 3)
sessionGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_session_"}]
sessionGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "ph_session_", durationBucket: durationBucket(&oneYear)}]
require.True(t, ok)
assert.Len(t, sessionGroup, 3)
},
@@ -252,13 +255,171 @@ func TestFindMergeGroups(t *testing.T) {
t.Parallel()
patterns := coredata.TrackerPatterns{
makePattern("PHPSESSID"),
makePattern("JSESSIONID"),
makePattern("ASPSESSIONID"),
makePattern("PHPSESSID", nil),
makePattern("JSESSIONID", nil),
makePattern("ASPSESSIONID", nil),
}
groups := findMergeGroups(patterns, 3)
assert.Empty(t, groups)
},
)
t.Run(
"session and persistent cookies do not merge",
func(t *testing.T) {
t.Parallel()
patterns := coredata.TrackerPatterns{
makePattern("_ga_ABC123", nil),
makePattern("_ga_DEF456", nil),
makePattern("_ga_GHI789", nil),
makePattern("_ga_JKL012", &oneYear),
makePattern("_ga_MNO345", &oneYear),
makePattern("_ga_PQR678", &oneYear),
}
groups := findMergeGroups(patterns, 3)
require.Len(t, groups, 2)
sessionGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "_ga_", durationBucket: -1}]
require.True(t, ok)
assert.Len(t, sessionGroup, 3)
persistentGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "_ga_", durationBucket: durationBucket(&oneYear)}]
require.True(t, ok)
assert.Len(t, persistentGroup, 3)
},
)
t.Run(
"close durations snap to same bucket and merge",
func(t *testing.T) {
t.Parallel()
exactYear := 365 * 24 * 3600
almostYear := 364 * 24 * 3600
patterns := coredata.TrackerPatterns{
makePattern("_ga_ABC123", &exactYear),
makePattern("_ga_DEF456", &almostYear),
makePattern("_ga_GHI789", &exactYear),
}
groups := findMergeGroups(patterns, 3)
require.Len(t, groups, 1)
},
)
t.Run(
"different durations do not merge",
func(t *testing.T) {
t.Parallel()
oneDay := 24 * 3600
thirtyDays := 30 * 24 * 3600
patterns := coredata.TrackerPatterns{
makePattern("_ga_ABC123", &oneDay),
makePattern("_ga_DEF456", &oneDay),
makePattern("_ga_GHI789", &oneDay),
makePattern("_ga_JKL012", &thirtyDays),
makePattern("_ga_MNO345", &thirtyDays),
makePattern("_ga_PQR678", &thirtyDays),
}
groups := findMergeGroups(patterns, 3)
require.Len(t, groups, 2)
dayGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "_ga_", durationBucket: durationBucket(&oneDay)}]
require.True(t, ok)
assert.Len(t, dayGroup, 3)
monthGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, prefix: "_ga_", durationBucket: durationBucket(&thirtyDays)}]
require.True(t, ok)
assert.Len(t, monthGroup, 3)
},
)
}
func TestDurationBucket(t *testing.T) {
t.Parallel()
tests := []struct {
name string
maxAge *int
expected int
}{
{
name: "nil is session",
maxAge: nil,
expected: -1,
},
{
name: "zero is session",
maxAge: new(0),
expected: -1,
},
{
name: "negative is session",
maxAge: new(-1),
expected: -1,
},
{
name: "exact 1 year",
maxAge: new(365 * 24 * 3600),
expected: 365 * 24 * 3600,
},
{
name: "364 days snaps to 1 year",
maxAge: new(364 * 24 * 3600),
expected: 365 * 24 * 3600,
},
{
name: "exact 30 days",
maxAge: new(30 * 24 * 3600),
expected: 30 * 24 * 3600,
},
{
name: "exact 1 day",
maxAge: new(24 * 3600),
expected: 24 * 3600,
},
{
name: "23h snaps to 1 day",
maxAge: new(23 * 3600),
expected: 24 * 3600,
},
{
name: "exact 1 hour",
maxAge: new(3600),
expected: 3600,
},
{
name: "58 minutes snaps to 1 hour",
maxAge: new(58 * 60),
expected: 3600,
},
{
name: "exact 5 minutes",
maxAge: new(5 * 60),
expected: 5 * 60,
},
{
name: "1 day and 30 days are different buckets",
maxAge: new(24 * 3600),
expected: 24 * 3600,
},
}
for _, tt := range tests {
t.Run(
tt.name,
func(t *testing.T) {
t.Parallel()
result := durationBucket(tt.maxAge)
assert.Equal(t, tt.expected, result)
},
)
}
}

18
pkg/coredata/migrations/20260506T123706Z.sql Normal file
View File

@@ -0,0 +1,18 @@
-- Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
--
-- Permission to use, copy, modify, and/or distribute this software for any
-- purpose with or without fee is hereby granted, provided that the above
-- copyright notice and this permission notice appear in all copies.
--
-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-- PERFORMANCE OF THIS SOFTWARE.
DROP INDEX idx_tracker_patterns_unique_pattern_per_banner;
CREATE UNIQUE INDEX idx_tracker_patterns_unique_pattern_per_banner
ON tracker_patterns (cookie_banner_id, tracker_type, pattern, COALESCE(max_age_seconds, -1));

5
pkg/coredata/tracker_pattern.go
View File

@@ -149,6 +149,7 @@ func (tp *TrackerPattern) LoadByBannerIDTypeAndPattern(
cookieBannerID gid.GID,
trackerType TrackerType,
pattern string,
maxAgeSeconds *int,
) error {
q := `
SELECT
@@ -174,6 +175,7 @@ WHERE
AND cookie_banner_id = @cookie_banner_id
AND tracker_type = @tracker_type
AND pattern = @pattern
AND COALESCE(max_age_seconds, -1) = COALESCE(@max_age_seconds, -1)
LIMIT 1;
`
@@ -183,6 +185,7 @@ LIMIT 1;
"cookie_banner_id": cookieBannerID,
"tracker_type": trackerType,
"pattern": pattern,
"max_age_seconds": maxAgeSeconds,
}
maps.Copy(args, scope.SQLArguments())
@@ -393,7 +396,7 @@ INSERT INTO tracker_patterns (
@created_at,
@updated_at
)
ON CONFLICT (cookie_banner_id, tracker_type, pattern) DO NOTHING
ON CONFLICT (cookie_banner_id, tracker_type, pattern, COALESCE(max_age_seconds, -1)) DO NOTHING
`
args := pgx.StrictNamedArgs{