Detect variable tokens in tracker pattern names

The pattern analysis worker now recognises UUID-like, hash-like,
and long numeric tokens as variable parts and replaces them with
wildcards heuristically, even from a single observation. This
prevents site-specific identifiers from being treated as static
text while meaningful suffixes (window_id, posthog, …) get
incorrectly wildcarded.

Also upgrades globMatch and the FindMatchingPattern SQL query
to support multiple wildcards in a single pattern.

Signed-off-by: Émile Ré <emile@getprobo.com>

pkg/cookiebanner/worker.go

@@ -250,35 +250,64 @@ func findMergeGroups(
 	patterns coredata.TrackerPatterns,
 	threshold int,
 ) map[mergeGroupKey][]*coredata.TrackerPattern {
+	type memberKey struct {
+		groupKey mergeGroupKey
+		pattern  *coredata.TrackerPattern
+	}
+
 	templateCounts := make(map[mergeGroupKey][]*coredata.TrackerPattern)
+	heuristicKeys := make(map[mergeGroupKey]bool)
+	seen := make(map[memberKey]bool)
+
 	for _, p := range patterns {
 		bucket := durationBucket(p.MaxAgeSeconds)
+
+		if tmpl, ok := heuristicTemplate(p.Pattern); ok {
+			key := mergeGroupKey{categoryID: p.CookieCategoryID, trackerType: p.TrackerType, template: tmpl, durationBucket: bucket}
+			mk := memberKey{key, p}
+			if !seen[mk] {
+				seen[mk] = true
+				templateCounts[key] = append(templateCounts[key], p)
+			}
+			heuristicKeys[key] = true
+		}
+
 		for _, tmpl := range templateCandidates(p.Pattern) {
 			key := mergeGroupKey{categoryID: p.CookieCategoryID, trackerType: p.TrackerType, template: tmpl, durationBucket: bucket}
-			templateCounts[key] = append(templateCounts[key], p)
+			mk := memberKey{key, p}
+			if !seen[mk] {
+				seen[mk] = true
+				templateCounts[key] = append(templateCounts[key], p)
+			}
 		}
 	}
 
 	type candidate struct {
-		key        mergeGroupKey
+		key         mergeGroupKey
-		fixedChars int
+		fixedChars  int
-		patterns   []*coredata.TrackerPattern
+		isHeuristic bool
+		patterns    []*coredata.TrackerPattern
 	}
 
 	var candidates []candidate
 	for key, pats := range templateCounts {
-		if len(pats) >= threshold {
+		isH := heuristicKeys[key]
-			candidates = append(candidates, candidate{key, len(key.template) - 1, pats})
+		effectiveThreshold := threshold
+		if isH {
+			effectiveThreshold = 1
+		}
+		if len(pats) >= effectiveThreshold {
+			candidates = append(candidates, candidate{key, len(strings.ReplaceAll(key.template, "*", "")), isH, pats})
 		}
 	}
 
-	// Sort by descending specificity (more fixed characters first), then
+	// Sort: heuristic first, then descending specificity (more fixed
-	// descending coverage (more patterns matched first), then template
+	// characters), then descending coverage, then template name for a
-	// name for a fully deterministic order. Without these tie-breakers
+	// fully deterministic order.
-	// the greedy assignment below depends on Go's randomised map
-	// iteration and the same input can produce different merge groups
-	// across runs.
 	sort.Slice(candidates, func(i, j int) bool {
+		if candidates[i].isHeuristic != candidates[j].isHeuristic {
+			return candidates[i].isHeuristic
+		}
 		if candidates[i].fixedChars != candidates[j].fixedChars {
 			return candidates[i].fixedChars > candidates[j].fixedChars
 		}
@@ -292,6 +321,11 @@ func findMergeGroups(
 	groups := make(map[mergeGroupKey][]*coredata.TrackerPattern)
 
 	for _, c := range candidates {
+		effectiveThreshold := threshold
+		if c.isHeuristic {
+			effectiveThreshold = 1
+		}
+
 		var unassigned []*coredata.TrackerPattern
 		for _, p := range c.patterns {
 			if !assigned[p] {
@@ -299,7 +333,7 @@ func findMergeGroups(
 			}
 		}
 
-		if len(unassigned) < threshold {
+		if len(unassigned) < effectiveThreshold {
 			continue
 		}
 
@@ -312,6 +346,36 @@ func findMergeGroups(
 	return groups
 }
 
+func heuristicTemplate(name string) (string, bool) {
+	tokens, sep := splitTokens(name)
+	if sep == 0 {
+		if looksVariable(name) {
+			return "*", true
+		}
+		return "", false
+	}
+
+	s := string(sep)
+	changed := false
+	var result []string
+	for _, t := range tokens {
+		if looksVariable(t) {
+			changed = true
+			if len(result) == 0 || result[len(result)-1] != "*" {
+				result = append(result, "*")
+			}
+		} else {
+			result = append(result, t)
+		}
+	}
+
+	if !changed {
+		return "", false
+	}
+
+	return strings.Join(result, s), true
+}
+
 func templateCandidates(name string) []string {
 	var candidates []string
 
@@ -333,6 +397,72 @@ func templateCandidates(name string) []string {
 	return candidates
 }
 
+func looksVariable(token string) bool {
+	if len(token) == 0 {
+		return false
+	}
+
+	hasDigit := false
+	hasLetter := false
+	allHex := true
+	allDigits := true
+	for _, ch := range token {
+		switch {
+		case ch >= '0' && ch <= '9':
+			hasDigit = true
+		case ch >= 'a' && ch <= 'f', ch >= 'A' && ch <= 'F':
+			hasLetter = true
+			allDigits = false
+		case ch >= 'g' && ch <= 'z', ch >= 'G' && ch <= 'Z':
+			hasLetter = true
+			allHex = false
+			allDigits = false
+		case ch == '-':
+			allHex = false
+			allDigits = false
+		default:
+			allHex = false
+			allDigits = false
+		}
+	}
+
+	if len(token) >= 8 && hasDigit && hasLetter {
+		return true
+	}
+
+	if len(token) >= 16 && allHex && hasDigit {
+		return true
+	}
+
+	if isUUIDShape(token) {
+		return true
+	}
+
+	if len(token) >= 8 && allDigits {
+		return true
+	}
+
+	return false
+}
+
+func isUUIDShape(s string) bool {
+	if len(s) != 36 {
+		return false
+	}
+	for i, ch := range s {
+		if i == 8 || i == 13 || i == 18 || i == 23 {
+			if ch != '-' {
+				return false
+			}
+			continue
+		}
+		if !((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F')) {
+			return false
+		}
+	}
+	return true
+}
+
 func splitTokens(name string) ([]string, byte) {
 	if found := strings.Contains(name, "_"); found {
 		return strings.Split(name, "_"), '_'
@@ -344,15 +474,31 @@ func splitTokens(name string) ([]string, byte) {
 }
 
 func globMatch(pattern, name string) bool {
-	before, after, ok := strings.Cut(pattern, "*")
+	parts := strings.Split(pattern, "*")
-	if !ok {
+	if len(parts) == 1 {
 		return pattern == name
 	}
-	prefix := before
+
-	suffix := after
+	if !strings.HasPrefix(name, parts[0]) {
-	return strings.HasPrefix(name, prefix) &&
+		return false
-		strings.HasSuffix(name, suffix) &&
+	}
-		len(name) >= len(prefix)+len(suffix)
+	name = name[len(parts[0]):]
+
+	last := parts[len(parts)-1]
+	if !strings.HasSuffix(name, last) {
+		return false
+	}
+	name = name[:len(name)-len(last)]
+
+	for _, part := range parts[1 : len(parts)-1] {
+		idx := strings.Index(name, part)
+		if idx == -1 {
+			return false
+		}
+		name = name[idx+len(part):]
+	}
+
+	return true
 }
 
 func bestSource(patterns []*coredata.TrackerPattern) *coredata.CookieSource {

pkg/cookiebanner/worker_test.go

@@ -23,6 +23,233 @@ import (
 	"go.probo.inc/probo/pkg/gid"
 )
 
+func TestLooksVariable(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		token    string
+		expected bool
+	}{
+		{
+			name:     "long mixed alphanumeric",
+			token:    "XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN",
+			expected: true,
+		},
+		{
+			name:     "short mixed alphanumeric below threshold",
+			token:    "abc123",
+			expected: false,
+		},
+		{
+			name:     "exactly 8 chars mixed",
+			token:    "a1b2c3d4",
+			expected: true,
+		},
+		{
+			name:     "long hex string 16 chars",
+			token:    "a1b2c3d4e5f60718",
+			expected: true,
+		},
+		{
+			name:     "short hex string below threshold",
+			token:    "abcdef12",
+			expected: true,
+		},
+		{
+			name:     "pure letters not variable",
+			token:    "posthog",
+			expected: false,
+		},
+		{
+			name:     "long pure letters not variable",
+			token:    "authentication",
+			expected: false,
+		},
+		{
+			name:     "brand name with digit",
+			token:    "auth0",
+			expected: false,
+		},
+		{
+			name:     "UUID shape",
+			token:    "550e8400-e29b-41d4-a716-446655440000",
+			expected: true,
+		},
+		{
+			name:     "8 digit number",
+			token:    "12345678",
+			expected: true,
+		},
+		{
+			name:     "short digit number",
+			token:    "12345",
+			expected: false,
+		},
+		{
+			name:     "empty string",
+			token:    "",
+			expected: false,
+		},
+		{
+			name:     "single char",
+			token:    "x",
+			expected: false,
+		},
+		{
+			name:     "all uppercase letters",
+			token:    "MEASUREMENT",
+			expected: false,
+		},
+		{
+			name:     "short brand c15t",
+			token:    "c15t",
+			expected: false,
+		},
+		{
+			name:     "GA measurement ID style",
+			token:    "G-1234ABCDEF",
+			expected: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(
+			tt.name,
+			func(t *testing.T) {
+				t.Parallel()
+				assert.Equal(t, tt.expected, looksVariable(tt.token))
+			},
+		)
+	}
+}
+
+func TestIsUUIDShape(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		input    string
+		expected bool
+	}{
+		{
+			name:     "valid UUID lowercase",
+			input:    "550e8400-e29b-41d4-a716-446655440000",
+			expected: true,
+		},
+		{
+			name:     "valid UUID uppercase",
+			input:    "550E8400-E29B-41D4-A716-446655440000",
+			expected: true,
+		},
+		{
+			name:     "wrong length",
+			input:    "550e8400-e29b-41d4-a716",
+			expected: false,
+		},
+		{
+			name:     "no dashes",
+			input:    "550e8400e29b41d4a716446655440000xxxx",
+			expected: false,
+		},
+		{
+			name:     "dashes in wrong positions",
+			input:    "550e840-0e29b-41d4-a716-446655440000",
+			expected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(
+			tt.name,
+			func(t *testing.T) {
+				t.Parallel()
+				assert.Equal(t, tt.expected, isUUIDShape(tt.input))
+			},
+		)
+	}
+}
+
+func TestHeuristicTemplate(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		input    string
+		template string
+		changed  bool
+	}{
+		{
+			name:     "posthog hash in middle",
+			input:    "ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_window_id",
+			template: "ph_phc_*_window_id",
+			changed:  true,
+		},
+		{
+			name:     "no variable tokens",
+			input:    "probo_consent_given",
+			template: "",
+			changed:  false,
+		},
+		{
+			name:     "trailing hash",
+			input:    "auth0_session_a1b2c3d4e5f6",
+			template: "auth0_session_*",
+			changed:  true,
+		},
+		{
+			name:     "no separator with variable token",
+			input:    "a1b2c3d4e5f6g7h8",
+			template: "*",
+			changed:  true,
+		},
+		{
+			name:     "no separator without variable token",
+			input:    "PHPSESSID",
+			template: "",
+			changed:  false,
+		},
+		{
+			name:     "consecutive variable tokens collapse to single star",
+			input:    "ph_a1b2c3d4_e5f6g7h8_window",
+			template: "ph_*_window",
+			changed:  true,
+		},
+		{
+			name:     "UUID token replaced",
+			input:    "session_550e8400-e29b-41d4-a716-446655440000_data",
+			template: "session_*_data",
+			changed:  true,
+		},
+		{
+			name:     "leading underscore with hash",
+			input:    "_ga_G1234ABCDEF",
+			template: "_ga_*",
+			changed:  true,
+		},
+		{
+			name:     "dash separator with hash",
+			input:    "c15t-consent-a1b2c3d4e5f6",
+			template: "c15t-consent-*",
+			changed:  true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(
+			tt.name,
+			func(t *testing.T) {
+				t.Parallel()
+				tmpl, changed := heuristicTemplate(tt.input)
+				assert.Equal(t, tt.changed, changed)
+				if changed {
+					assert.Equal(t, tt.template, tmpl)
+				}
+			},
+		)
+	}
+}
+
 func TestTemplateCandidates(t *testing.T) {
 	t.Parallel()
 
@@ -185,6 +412,30 @@ func TestGlobMatch(t *testing.T) {
 			input:   "ph_phc_posthog",
 			match:   false,
 		},
+		{
+			name:    "multi-star matches",
+			pattern: "ph_*_something_*_end",
+			input:   "ph_hash1_something_hash2_end",
+			match:   true,
+		},
+		{
+			name:    "multi-star wrong middle segment",
+			pattern: "ph_*_something_*_end",
+			input:   "ph_hash1_other_hash2_end",
+			match:   false,
+		},
+		{
+			name:    "multi-star wrong suffix",
+			pattern: "ph_*_something_*_end",
+			input:   "ph_hash1_something_hash2_nope",
+			match:   false,
+		},
+		{
+			name:    "multi-star with underscore-heavy middle",
+			pattern: "a_*_b_*_c",
+			input:   "a_x_y_z_b_q_r_c",
+			match:   true,
+		},
 	}
 
 	for _, tt := range tests {
@@ -542,6 +793,92 @@ func TestFindMergeGroups(t *testing.T) {
 			assert.Len(t, group, 3)
 		},
 	)
+
+	t.Run(
+		"single pattern with hash normalized via heuristic",
+		func(t *testing.T) {
+			t.Parallel()
+
+			patterns := coredata.TrackerPatterns{
+				makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_window_id", nil),
+			}
+
+			groups := findMergeGroups(patterns, 3)
+			require.Len(t, groups, 1)
+
+			group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_window_id", durationBucket: -1}]
+			require.True(t, ok)
+			assert.Len(t, group, 1)
+		},
+	)
+
+	t.Run(
+		"same hash different suffixes produce separate heuristic globs",
+		func(t *testing.T) {
+			t.Parallel()
+
+			patterns := coredata.TrackerPatterns{
+				makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_window_id", nil),
+				makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_posthog", nil),
+				makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_primary_window_exists", nil),
+			}
+
+			groups := findMergeGroups(patterns, 3)
+			require.Len(t, groups, 3)
+
+			windowGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_window_id", durationBucket: -1}]
+			require.True(t, ok)
+			assert.Len(t, windowGroup, 1)
+
+			posthogGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_posthog", durationBucket: -1}]
+			require.True(t, ok)
+			assert.Len(t, posthogGroup, 1)
+
+			primaryGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_primary_window_exists", durationBucket: -1}]
+			require.True(t, ok)
+			assert.Len(t, primaryGroup, 1)
+		},
+	)
+
+	t.Run(
+		"patterns without variable tokens still require statistical threshold",
+		func(t *testing.T) {
+			t.Parallel()
+
+			patterns := coredata.TrackerPatterns{
+				makePattern("foo_bar_aaa", &oneYear),
+				makePattern("foo_bar_bbb", &oneYear),
+			}
+
+			groups := findMergeGroups(patterns, 3)
+			assert.Empty(t, groups)
+		},
+	)
+
+	t.Run(
+		"heuristic and statistical patterns coexist",
+		func(t *testing.T) {
+			t.Parallel()
+
+			patterns := coredata.TrackerPatterns{
+				makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_window_id", &oneYear),
+				makePattern("foo_bar_aaa", &oneYear),
+				makePattern("foo_bar_bbb", &oneYear),
+				makePattern("foo_bar_ccc", &oneYear),
+			}
+
+			groups := findMergeGroups(patterns, 3)
+			require.Len(t, groups, 2)
+
+			heuristicGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_window_id", durationBucket: durationBucket(&oneYear)}]
+			require.True(t, ok)
+			assert.Len(t, heuristicGroup, 1)
+
+			statGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "foo_bar_*", durationBucket: durationBucket(&oneYear)}]
+			require.True(t, ok)
+			assert.Len(t, statGroup, 3)
+		},
+	)
 }
 
 func TestDurationBucket(t *testing.T) {

pkg/coredata/tracker_pattern.go

@@ -240,16 +240,17 @@ WHERE
 	AND tracker_type = @tracker_type
 	AND (
 		(match_type = @match_type_glob
-		 AND starts_with(@identifier, split_part(pattern, '*', 1))
+		 AND @identifier LIKE
-		 AND right(@identifier, length(split_part(pattern, '*', 2))) = split_part(pattern, '*', 2)
+		     replace(replace(replace(replace(
-		 AND length(@identifier) >= length(pattern) - 1)
+		         pattern, E'\\', E'\\\\'), '%', E'\\%'), '_', E'\\_'), '*', '%')
+		     ESCAPE E'\\')
 		OR (match_type = @match_type_exact AND pattern = @identifier)
 	)
 ORDER BY
 	CASE WHEN match_type = @match_type_exact AND pattern = @identifier THEN 0
 	     ELSE 1
 	END,
-	length(pattern) - 1 DESC
+	length(replace(pattern, '*', '')) DESC
 LIMIT 1;
 `