diff --git a/pkg/cookiebanner/worker.go b/pkg/cookiebanner/worker.go index 3841c7276..908cd8cf1 100644 --- a/pkg/cookiebanner/worker.go +++ b/pkg/cookiebanner/worker.go @@ -250,35 +250,64 @@ func findMergeGroups( patterns coredata.TrackerPatterns, threshold int, ) map[mergeGroupKey][]*coredata.TrackerPattern { + type memberKey struct { + groupKey mergeGroupKey + pattern *coredata.TrackerPattern + } + templateCounts := make(map[mergeGroupKey][]*coredata.TrackerPattern) + heuristicKeys := make(map[mergeGroupKey]bool) + seen := make(map[memberKey]bool) + for _, p := range patterns { bucket := durationBucket(p.MaxAgeSeconds) + + if tmpl, ok := heuristicTemplate(p.Pattern); ok { + key := mergeGroupKey{categoryID: p.CookieCategoryID, trackerType: p.TrackerType, template: tmpl, durationBucket: bucket} + mk := memberKey{key, p} + if !seen[mk] { + seen[mk] = true + templateCounts[key] = append(templateCounts[key], p) + } + heuristicKeys[key] = true + } + for _, tmpl := range templateCandidates(p.Pattern) { key := mergeGroupKey{categoryID: p.CookieCategoryID, trackerType: p.TrackerType, template: tmpl, durationBucket: bucket} - templateCounts[key] = append(templateCounts[key], p) + mk := memberKey{key, p} + if !seen[mk] { + seen[mk] = true + templateCounts[key] = append(templateCounts[key], p) + } } } type candidate struct { - key mergeGroupKey - fixedChars int - patterns []*coredata.TrackerPattern + key mergeGroupKey + fixedChars int + isHeuristic bool + patterns []*coredata.TrackerPattern } var candidates []candidate for key, pats := range templateCounts { - if len(pats) >= threshold { - candidates = append(candidates, candidate{key, len(key.template) - 1, pats}) + isH := heuristicKeys[key] + effectiveThreshold := threshold + if isH { + effectiveThreshold = 1 + } + if len(pats) >= effectiveThreshold { + candidates = append(candidates, candidate{key, len(strings.ReplaceAll(key.template, "*", "")), isH, pats}) } } - // Sort by descending specificity (more fixed characters first), then - // descending coverage (more patterns matched first), then template - // name for a fully deterministic order. Without these tie-breakers - // the greedy assignment below depends on Go's randomised map - // iteration and the same input can produce different merge groups - // across runs. + // Sort: heuristic first, then descending specificity (more fixed + // characters), then descending coverage, then template name for a + // fully deterministic order. sort.Slice(candidates, func(i, j int) bool { + if candidates[i].isHeuristic != candidates[j].isHeuristic { + return candidates[i].isHeuristic + } if candidates[i].fixedChars != candidates[j].fixedChars { return candidates[i].fixedChars > candidates[j].fixedChars } @@ -292,6 +321,11 @@ func findMergeGroups( groups := make(map[mergeGroupKey][]*coredata.TrackerPattern) for _, c := range candidates { + effectiveThreshold := threshold + if c.isHeuristic { + effectiveThreshold = 1 + } + var unassigned []*coredata.TrackerPattern for _, p := range c.patterns { if !assigned[p] { @@ -299,7 +333,7 @@ func findMergeGroups( } } - if len(unassigned) < threshold { + if len(unassigned) < effectiveThreshold { continue } @@ -312,6 +346,36 @@ func findMergeGroups( return groups } +func heuristicTemplate(name string) (string, bool) { + tokens, sep := splitTokens(name) + if sep == 0 { + if looksVariable(name) { + return "*", true + } + return "", false + } + + s := string(sep) + changed := false + var result []string + for _, t := range tokens { + if looksVariable(t) { + changed = true + if len(result) == 0 || result[len(result)-1] != "*" { + result = append(result, "*") + } + } else { + result = append(result, t) + } + } + + if !changed { + return "", false + } + + return strings.Join(result, s), true +} + func templateCandidates(name string) []string { var candidates []string @@ -333,6 +397,72 @@ func templateCandidates(name string) []string { return candidates } +func looksVariable(token string) bool { + if len(token) == 0 { + return false + } + + hasDigit := false + hasLetter := false + allHex := true + allDigits := true + for _, ch := range token { + switch { + case ch >= '0' && ch <= '9': + hasDigit = true + case ch >= 'a' && ch <= 'f', ch >= 'A' && ch <= 'F': + hasLetter = true + allDigits = false + case ch >= 'g' && ch <= 'z', ch >= 'G' && ch <= 'Z': + hasLetter = true + allHex = false + allDigits = false + case ch == '-': + allHex = false + allDigits = false + default: + allHex = false + allDigits = false + } + } + + if len(token) >= 8 && hasDigit && hasLetter { + return true + } + + if len(token) >= 16 && allHex && hasDigit { + return true + } + + if isUUIDShape(token) { + return true + } + + if len(token) >= 8 && allDigits { + return true + } + + return false +} + +func isUUIDShape(s string) bool { + if len(s) != 36 { + return false + } + for i, ch := range s { + if i == 8 || i == 13 || i == 18 || i == 23 { + if ch != '-' { + return false + } + continue + } + if !((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F')) { + return false + } + } + return true +} + func splitTokens(name string) ([]string, byte) { if found := strings.Contains(name, "_"); found { return strings.Split(name, "_"), '_' @@ -344,15 +474,31 @@ func splitTokens(name string) ([]string, byte) { } func globMatch(pattern, name string) bool { - before, after, ok := strings.Cut(pattern, "*") - if !ok { + parts := strings.Split(pattern, "*") + if len(parts) == 1 { return pattern == name } - prefix := before - suffix := after - return strings.HasPrefix(name, prefix) && - strings.HasSuffix(name, suffix) && - len(name) >= len(prefix)+len(suffix) + + if !strings.HasPrefix(name, parts[0]) { + return false + } + name = name[len(parts[0]):] + + last := parts[len(parts)-1] + if !strings.HasSuffix(name, last) { + return false + } + name = name[:len(name)-len(last)] + + for _, part := range parts[1 : len(parts)-1] { + idx := strings.Index(name, part) + if idx == -1 { + return false + } + name = name[idx+len(part):] + } + + return true } func bestSource(patterns []*coredata.TrackerPattern) *coredata.CookieSource { diff --git a/pkg/cookiebanner/worker_test.go b/pkg/cookiebanner/worker_test.go index 00296c39e..e79419f7e 100644 --- a/pkg/cookiebanner/worker_test.go +++ b/pkg/cookiebanner/worker_test.go @@ -23,6 +23,233 @@ import ( "go.probo.inc/probo/pkg/gid" ) +func TestLooksVariable(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + token string + expected bool + }{ + { + name: "long mixed alphanumeric", + token: "XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN", + expected: true, + }, + { + name: "short mixed alphanumeric below threshold", + token: "abc123", + expected: false, + }, + { + name: "exactly 8 chars mixed", + token: "a1b2c3d4", + expected: true, + }, + { + name: "long hex string 16 chars", + token: "a1b2c3d4e5f60718", + expected: true, + }, + { + name: "short hex string below threshold", + token: "abcdef12", + expected: true, + }, + { + name: "pure letters not variable", + token: "posthog", + expected: false, + }, + { + name: "long pure letters not variable", + token: "authentication", + expected: false, + }, + { + name: "brand name with digit", + token: "auth0", + expected: false, + }, + { + name: "UUID shape", + token: "550e8400-e29b-41d4-a716-446655440000", + expected: true, + }, + { + name: "8 digit number", + token: "12345678", + expected: true, + }, + { + name: "short digit number", + token: "12345", + expected: false, + }, + { + name: "empty string", + token: "", + expected: false, + }, + { + name: "single char", + token: "x", + expected: false, + }, + { + name: "all uppercase letters", + token: "MEASUREMENT", + expected: false, + }, + { + name: "short brand c15t", + token: "c15t", + expected: false, + }, + { + name: "GA measurement ID style", + token: "G-1234ABCDEF", + expected: true, + }, + } + + for _, tt := range tests { + t.Run( + tt.name, + func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.expected, looksVariable(tt.token)) + }, + ) + } +} + +func TestIsUUIDShape(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + expected bool + }{ + { + name: "valid UUID lowercase", + input: "550e8400-e29b-41d4-a716-446655440000", + expected: true, + }, + { + name: "valid UUID uppercase", + input: "550E8400-E29B-41D4-A716-446655440000", + expected: true, + }, + { + name: "wrong length", + input: "550e8400-e29b-41d4-a716", + expected: false, + }, + { + name: "no dashes", + input: "550e8400e29b41d4a716446655440000xxxx", + expected: false, + }, + { + name: "dashes in wrong positions", + input: "550e840-0e29b-41d4-a716-446655440000", + expected: false, + }, + } + + for _, tt := range tests { + t.Run( + tt.name, + func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.expected, isUUIDShape(tt.input)) + }, + ) + } +} + +func TestHeuristicTemplate(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + template string + changed bool + }{ + { + name: "posthog hash in middle", + input: "ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_window_id", + template: "ph_phc_*_window_id", + changed: true, + }, + { + name: "no variable tokens", + input: "probo_consent_given", + template: "", + changed: false, + }, + { + name: "trailing hash", + input: "auth0_session_a1b2c3d4e5f6", + template: "auth0_session_*", + changed: true, + }, + { + name: "no separator with variable token", + input: "a1b2c3d4e5f6g7h8", + template: "*", + changed: true, + }, + { + name: "no separator without variable token", + input: "PHPSESSID", + template: "", + changed: false, + }, + { + name: "consecutive variable tokens collapse to single star", + input: "ph_a1b2c3d4_e5f6g7h8_window", + template: "ph_*_window", + changed: true, + }, + { + name: "UUID token replaced", + input: "session_550e8400-e29b-41d4-a716-446655440000_data", + template: "session_*_data", + changed: true, + }, + { + name: "leading underscore with hash", + input: "_ga_G1234ABCDEF", + template: "_ga_*", + changed: true, + }, + { + name: "dash separator with hash", + input: "c15t-consent-a1b2c3d4e5f6", + template: "c15t-consent-*", + changed: true, + }, + } + + for _, tt := range tests { + t.Run( + tt.name, + func(t *testing.T) { + t.Parallel() + tmpl, changed := heuristicTemplate(tt.input) + assert.Equal(t, tt.changed, changed) + if changed { + assert.Equal(t, tt.template, tmpl) + } + }, + ) + } +} + func TestTemplateCandidates(t *testing.T) { t.Parallel() @@ -185,6 +412,30 @@ func TestGlobMatch(t *testing.T) { input: "ph_phc_posthog", match: false, }, + { + name: "multi-star matches", + pattern: "ph_*_something_*_end", + input: "ph_hash1_something_hash2_end", + match: true, + }, + { + name: "multi-star wrong middle segment", + pattern: "ph_*_something_*_end", + input: "ph_hash1_other_hash2_end", + match: false, + }, + { + name: "multi-star wrong suffix", + pattern: "ph_*_something_*_end", + input: "ph_hash1_something_hash2_nope", + match: false, + }, + { + name: "multi-star with underscore-heavy middle", + pattern: "a_*_b_*_c", + input: "a_x_y_z_b_q_r_c", + match: true, + }, } for _, tt := range tests { @@ -542,6 +793,92 @@ func TestFindMergeGroups(t *testing.T) { assert.Len(t, group, 3) }, ) + + t.Run( + "single pattern with hash normalized via heuristic", + func(t *testing.T) { + t.Parallel() + + patterns := coredata.TrackerPatterns{ + makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_window_id", nil), + } + + groups := findMergeGroups(patterns, 3) + require.Len(t, groups, 1) + + group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_window_id", durationBucket: -1}] + require.True(t, ok) + assert.Len(t, group, 1) + }, + ) + + t.Run( + "same hash different suffixes produce separate heuristic globs", + func(t *testing.T) { + t.Parallel() + + patterns := coredata.TrackerPatterns{ + makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_window_id", nil), + makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_posthog", nil), + makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_primary_window_exists", nil), + } + + groups := findMergeGroups(patterns, 3) + require.Len(t, groups, 3) + + windowGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_window_id", durationBucket: -1}] + require.True(t, ok) + assert.Len(t, windowGroup, 1) + + posthogGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_posthog", durationBucket: -1}] + require.True(t, ok) + assert.Len(t, posthogGroup, 1) + + primaryGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_primary_window_exists", durationBucket: -1}] + require.True(t, ok) + assert.Len(t, primaryGroup, 1) + }, + ) + + t.Run( + "patterns without variable tokens still require statistical threshold", + func(t *testing.T) { + t.Parallel() + + patterns := coredata.TrackerPatterns{ + makePattern("foo_bar_aaa", &oneYear), + makePattern("foo_bar_bbb", &oneYear), + } + + groups := findMergeGroups(patterns, 3) + assert.Empty(t, groups) + }, + ) + + t.Run( + "heuristic and statistical patterns coexist", + func(t *testing.T) { + t.Parallel() + + patterns := coredata.TrackerPatterns{ + makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_window_id", &oneYear), + makePattern("foo_bar_aaa", &oneYear), + makePattern("foo_bar_bbb", &oneYear), + makePattern("foo_bar_ccc", &oneYear), + } + + groups := findMergeGroups(patterns, 3) + require.Len(t, groups, 2) + + heuristicGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_window_id", durationBucket: durationBucket(&oneYear)}] + require.True(t, ok) + assert.Len(t, heuristicGroup, 1) + + statGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "foo_bar_*", durationBucket: durationBucket(&oneYear)}] + require.True(t, ok) + assert.Len(t, statGroup, 3) + }, + ) } func TestDurationBucket(t *testing.T) { diff --git a/pkg/coredata/tracker_pattern.go b/pkg/coredata/tracker_pattern.go index d6f618383..f39f4b940 100644 --- a/pkg/coredata/tracker_pattern.go +++ b/pkg/coredata/tracker_pattern.go @@ -240,16 +240,17 @@ WHERE AND tracker_type = @tracker_type AND ( (match_type = @match_type_glob - AND starts_with(@identifier, split_part(pattern, '*', 1)) - AND right(@identifier, length(split_part(pattern, '*', 2))) = split_part(pattern, '*', 2) - AND length(@identifier) >= length(pattern) - 1) + AND @identifier LIKE + replace(replace(replace(replace( + pattern, E'\\', E'\\\\'), '%', E'\\%'), '_', E'\\_'), '*', '%') + ESCAPE E'\\') OR (match_type = @match_type_exact AND pattern = @identifier) ) ORDER BY CASE WHEN match_type = @match_type_exact AND pattern = @identifier THEN 0 ELSE 1 END, - length(pattern) - 1 DESC + length(replace(pattern, '*', '')) DESC LIMIT 1; `