Detect variable tokens in tracker pattern names
The pattern analysis worker now recognises UUID-like, hash-like, and long numeric tokens as variable parts and replaces them with wildcards heuristically, even from a single observation. This prevents site-specific identifiers from being treated as static text while meaningful suffixes (window_id, posthog, …) get incorrectly wildcarded. Also upgrades globMatch and the FindMatchingPattern SQL query to support multiple wildcards in a single pattern. Signed-off-by: Émile Ré <emile@getprobo.com>
This commit is contained in:
Émile Ré
@@ -250,35 +250,64 @@ func findMergeGroups(
|
||||
patterns coredata.TrackerPatterns,
|
||||
threshold int,
|
||||
) map[mergeGroupKey][]*coredata.TrackerPattern {
|
||||
type memberKey struct {
|
||||
groupKey mergeGroupKey
|
||||
pattern *coredata.TrackerPattern
|
||||
}
|
||||
|
||||
templateCounts := make(map[mergeGroupKey][]*coredata.TrackerPattern)
|
||||
heuristicKeys := make(map[mergeGroupKey]bool)
|
||||
seen := make(map[memberKey]bool)
|
||||
|
||||
for _, p := range patterns {
|
||||
bucket := durationBucket(p.MaxAgeSeconds)
|
||||
|
||||
if tmpl, ok := heuristicTemplate(p.Pattern); ok {
|
||||
key := mergeGroupKey{categoryID: p.CookieCategoryID, trackerType: p.TrackerType, template: tmpl, durationBucket: bucket}
|
||||
mk := memberKey{key, p}
|
||||
if !seen[mk] {
|
||||
seen[mk] = true
|
||||
templateCounts[key] = append(templateCounts[key], p)
|
||||
}
|
||||
heuristicKeys[key] = true
|
||||
}
|
||||
|
||||
for _, tmpl := range templateCandidates(p.Pattern) {
|
||||
key := mergeGroupKey{categoryID: p.CookieCategoryID, trackerType: p.TrackerType, template: tmpl, durationBucket: bucket}
|
||||
templateCounts[key] = append(templateCounts[key], p)
|
||||
mk := memberKey{key, p}
|
||||
if !seen[mk] {
|
||||
seen[mk] = true
|
||||
templateCounts[key] = append(templateCounts[key], p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type candidate struct {
|
||||
key mergeGroupKey
|
||||
fixedChars int
|
||||
patterns []*coredata.TrackerPattern
|
||||
key mergeGroupKey
|
||||
fixedChars int
|
||||
isHeuristic bool
|
||||
patterns []*coredata.TrackerPattern
|
||||
}
|
||||
|
||||
var candidates []candidate
|
||||
for key, pats := range templateCounts {
|
||||
if len(pats) >= threshold {
|
||||
candidates = append(candidates, candidate{key, len(key.template) - 1, pats})
|
||||
isH := heuristicKeys[key]
|
||||
effectiveThreshold := threshold
|
||||
if isH {
|
||||
effectiveThreshold = 1
|
||||
}
|
||||
if len(pats) >= effectiveThreshold {
|
||||
candidates = append(candidates, candidate{key, len(strings.ReplaceAll(key.template, "*", "")), isH, pats})
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by descending specificity (more fixed characters first), then
|
||||
// descending coverage (more patterns matched first), then template
|
||||
// name for a fully deterministic order. Without these tie-breakers
|
||||
// the greedy assignment below depends on Go's randomised map
|
||||
// iteration and the same input can produce different merge groups
|
||||
// across runs.
|
||||
// Sort: heuristic first, then descending specificity (more fixed
|
||||
// characters), then descending coverage, then template name for a
|
||||
// fully deterministic order.
|
||||
sort.Slice(candidates, func(i, j int) bool {
|
||||
if candidates[i].isHeuristic != candidates[j].isHeuristic {
|
||||
return candidates[i].isHeuristic
|
||||
}
|
||||
if candidates[i].fixedChars != candidates[j].fixedChars {
|
||||
return candidates[i].fixedChars > candidates[j].fixedChars
|
||||
}
|
||||
@@ -292,6 +321,11 @@ func findMergeGroups(
|
||||
groups := make(map[mergeGroupKey][]*coredata.TrackerPattern)
|
||||
|
||||
for _, c := range candidates {
|
||||
effectiveThreshold := threshold
|
||||
if c.isHeuristic {
|
||||
effectiveThreshold = 1
|
||||
}
|
||||
|
||||
var unassigned []*coredata.TrackerPattern
|
||||
for _, p := range c.patterns {
|
||||
if !assigned[p] {
|
||||
@@ -299,7 +333,7 @@ func findMergeGroups(
|
||||
}
|
||||
}
|
||||
|
||||
if len(unassigned) < threshold {
|
||||
if len(unassigned) < effectiveThreshold {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -312,6 +346,36 @@ func findMergeGroups(
|
||||
return groups
|
||||
}
|
||||
|
||||
func heuristicTemplate(name string) (string, bool) {
|
||||
tokens, sep := splitTokens(name)
|
||||
if sep == 0 {
|
||||
if looksVariable(name) {
|
||||
return "*", true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
s := string(sep)
|
||||
changed := false
|
||||
var result []string
|
||||
for _, t := range tokens {
|
||||
if looksVariable(t) {
|
||||
changed = true
|
||||
if len(result) == 0 || result[len(result)-1] != "*" {
|
||||
result = append(result, "*")
|
||||
}
|
||||
} else {
|
||||
result = append(result, t)
|
||||
}
|
||||
}
|
||||
|
||||
if !changed {
|
||||
return "", false
|
||||
}
|
||||
|
||||
return strings.Join(result, s), true
|
||||
}
|
||||
|
||||
func templateCandidates(name string) []string {
|
||||
var candidates []string
|
||||
|
||||
@@ -333,6 +397,72 @@ func templateCandidates(name string) []string {
|
||||
return candidates
|
||||
}
|
||||
|
||||
func looksVariable(token string) bool {
|
||||
if len(token) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
hasDigit := false
|
||||
hasLetter := false
|
||||
allHex := true
|
||||
allDigits := true
|
||||
for _, ch := range token {
|
||||
switch {
|
||||
case ch >= '0' && ch <= '9':
|
||||
hasDigit = true
|
||||
case ch >= 'a' && ch <= 'f', ch >= 'A' && ch <= 'F':
|
||||
hasLetter = true
|
||||
allDigits = false
|
||||
case ch >= 'g' && ch <= 'z', ch >= 'G' && ch <= 'Z':
|
||||
hasLetter = true
|
||||
allHex = false
|
||||
allDigits = false
|
||||
case ch == '-':
|
||||
allHex = false
|
||||
allDigits = false
|
||||
default:
|
||||
allHex = false
|
||||
allDigits = false
|
||||
}
|
||||
}
|
||||
|
||||
if len(token) >= 8 && hasDigit && hasLetter {
|
||||
return true
|
||||
}
|
||||
|
||||
if len(token) >= 16 && allHex && hasDigit {
|
||||
return true
|
||||
}
|
||||
|
||||
if isUUIDShape(token) {
|
||||
return true
|
||||
}
|
||||
|
||||
if len(token) >= 8 && allDigits {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func isUUIDShape(s string) bool {
|
||||
if len(s) != 36 {
|
||||
return false
|
||||
}
|
||||
for i, ch := range s {
|
||||
if i == 8 || i == 13 || i == 18 || i == 23 {
|
||||
if ch != '-' {
|
||||
return false
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F')) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func splitTokens(name string) ([]string, byte) {
|
||||
if found := strings.Contains(name, "_"); found {
|
||||
return strings.Split(name, "_"), '_'
|
||||
@@ -344,15 +474,31 @@ func splitTokens(name string) ([]string, byte) {
|
||||
}
|
||||
|
||||
func globMatch(pattern, name string) bool {
|
||||
before, after, ok := strings.Cut(pattern, "*")
|
||||
if !ok {
|
||||
parts := strings.Split(pattern, "*")
|
||||
if len(parts) == 1 {
|
||||
return pattern == name
|
||||
}
|
||||
prefix := before
|
||||
suffix := after
|
||||
return strings.HasPrefix(name, prefix) &&
|
||||
strings.HasSuffix(name, suffix) &&
|
||||
len(name) >= len(prefix)+len(suffix)
|
||||
|
||||
if !strings.HasPrefix(name, parts[0]) {
|
||||
return false
|
||||
}
|
||||
name = name[len(parts[0]):]
|
||||
|
||||
last := parts[len(parts)-1]
|
||||
if !strings.HasSuffix(name, last) {
|
||||
return false
|
||||
}
|
||||
name = name[:len(name)-len(last)]
|
||||
|
||||
for _, part := range parts[1 : len(parts)-1] {
|
||||
idx := strings.Index(name, part)
|
||||
if idx == -1 {
|
||||
return false
|
||||
}
|
||||
name = name[idx+len(part):]
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func bestSource(patterns []*coredata.TrackerPattern) *coredata.CookieSource {
|
||||
|
||||
@@ -23,6 +23,233 @@ import (
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
)
|
||||
|
||||
func TestLooksVariable(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
token string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "long mixed alphanumeric",
|
||||
token: "XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "short mixed alphanumeric below threshold",
|
||||
token: "abc123",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "exactly 8 chars mixed",
|
||||
token: "a1b2c3d4",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "long hex string 16 chars",
|
||||
token: "a1b2c3d4e5f60718",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "short hex string below threshold",
|
||||
token: "abcdef12",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "pure letters not variable",
|
||||
token: "posthog",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "long pure letters not variable",
|
||||
token: "authentication",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "brand name with digit",
|
||||
token: "auth0",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "UUID shape",
|
||||
token: "550e8400-e29b-41d4-a716-446655440000",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "8 digit number",
|
||||
token: "12345678",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "short digit number",
|
||||
token: "12345",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "empty string",
|
||||
token: "",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "single char",
|
||||
token: "x",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "all uppercase letters",
|
||||
token: "MEASUREMENT",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "short brand c15t",
|
||||
token: "c15t",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "GA measurement ID style",
|
||||
token: "G-1234ABCDEF",
|
||||
expected: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(
|
||||
tt.name,
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
assert.Equal(t, tt.expected, looksVariable(tt.token))
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsUUIDShape(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "valid UUID lowercase",
|
||||
input: "550e8400-e29b-41d4-a716-446655440000",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "valid UUID uppercase",
|
||||
input: "550E8400-E29B-41D4-A716-446655440000",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "wrong length",
|
||||
input: "550e8400-e29b-41d4-a716",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "no dashes",
|
||||
input: "550e8400e29b41d4a716446655440000xxxx",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "dashes in wrong positions",
|
||||
input: "550e840-0e29b-41d4-a716-446655440000",
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(
|
||||
tt.name,
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
assert.Equal(t, tt.expected, isUUIDShape(tt.input))
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeuristicTemplate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
template string
|
||||
changed bool
|
||||
}{
|
||||
{
|
||||
name: "posthog hash in middle",
|
||||
input: "ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_window_id",
|
||||
template: "ph_phc_*_window_id",
|
||||
changed: true,
|
||||
},
|
||||
{
|
||||
name: "no variable tokens",
|
||||
input: "probo_consent_given",
|
||||
template: "",
|
||||
changed: false,
|
||||
},
|
||||
{
|
||||
name: "trailing hash",
|
||||
input: "auth0_session_a1b2c3d4e5f6",
|
||||
template: "auth0_session_*",
|
||||
changed: true,
|
||||
},
|
||||
{
|
||||
name: "no separator with variable token",
|
||||
input: "a1b2c3d4e5f6g7h8",
|
||||
template: "*",
|
||||
changed: true,
|
||||
},
|
||||
{
|
||||
name: "no separator without variable token",
|
||||
input: "PHPSESSID",
|
||||
template: "",
|
||||
changed: false,
|
||||
},
|
||||
{
|
||||
name: "consecutive variable tokens collapse to single star",
|
||||
input: "ph_a1b2c3d4_e5f6g7h8_window",
|
||||
template: "ph_*_window",
|
||||
changed: true,
|
||||
},
|
||||
{
|
||||
name: "UUID token replaced",
|
||||
input: "session_550e8400-e29b-41d4-a716-446655440000_data",
|
||||
template: "session_*_data",
|
||||
changed: true,
|
||||
},
|
||||
{
|
||||
name: "leading underscore with hash",
|
||||
input: "_ga_G1234ABCDEF",
|
||||
template: "_ga_*",
|
||||
changed: true,
|
||||
},
|
||||
{
|
||||
name: "dash separator with hash",
|
||||
input: "c15t-consent-a1b2c3d4e5f6",
|
||||
template: "c15t-consent-*",
|
||||
changed: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(
|
||||
tt.name,
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
tmpl, changed := heuristicTemplate(tt.input)
|
||||
assert.Equal(t, tt.changed, changed)
|
||||
if changed {
|
||||
assert.Equal(t, tt.template, tmpl)
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemplateCandidates(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -185,6 +412,30 @@ func TestGlobMatch(t *testing.T) {
|
||||
input: "ph_phc_posthog",
|
||||
match: false,
|
||||
},
|
||||
{
|
||||
name: "multi-star matches",
|
||||
pattern: "ph_*_something_*_end",
|
||||
input: "ph_hash1_something_hash2_end",
|
||||
match: true,
|
||||
},
|
||||
{
|
||||
name: "multi-star wrong middle segment",
|
||||
pattern: "ph_*_something_*_end",
|
||||
input: "ph_hash1_other_hash2_end",
|
||||
match: false,
|
||||
},
|
||||
{
|
||||
name: "multi-star wrong suffix",
|
||||
pattern: "ph_*_something_*_end",
|
||||
input: "ph_hash1_something_hash2_nope",
|
||||
match: false,
|
||||
},
|
||||
{
|
||||
name: "multi-star with underscore-heavy middle",
|
||||
pattern: "a_*_b_*_c",
|
||||
input: "a_x_y_z_b_q_r_c",
|
||||
match: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -542,6 +793,92 @@ func TestFindMergeGroups(t *testing.T) {
|
||||
assert.Len(t, group, 3)
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"single pattern with hash normalized via heuristic",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
patterns := coredata.TrackerPatterns{
|
||||
makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_window_id", nil),
|
||||
}
|
||||
|
||||
groups := findMergeGroups(patterns, 3)
|
||||
require.Len(t, groups, 1)
|
||||
|
||||
group, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_window_id", durationBucket: -1}]
|
||||
require.True(t, ok)
|
||||
assert.Len(t, group, 1)
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"same hash different suffixes produce separate heuristic globs",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
patterns := coredata.TrackerPatterns{
|
||||
makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_window_id", nil),
|
||||
makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_posthog", nil),
|
||||
makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_primary_window_exists", nil),
|
||||
}
|
||||
|
||||
groups := findMergeGroups(patterns, 3)
|
||||
require.Len(t, groups, 3)
|
||||
|
||||
windowGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_window_id", durationBucket: -1}]
|
||||
require.True(t, ok)
|
||||
assert.Len(t, windowGroup, 1)
|
||||
|
||||
posthogGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_posthog", durationBucket: -1}]
|
||||
require.True(t, ok)
|
||||
assert.Len(t, posthogGroup, 1)
|
||||
|
||||
primaryGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_primary_window_exists", durationBucket: -1}]
|
||||
require.True(t, ok)
|
||||
assert.Len(t, primaryGroup, 1)
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"patterns without variable tokens still require statistical threshold",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
patterns := coredata.TrackerPatterns{
|
||||
makePattern("foo_bar_aaa", &oneYear),
|
||||
makePattern("foo_bar_bbb", &oneYear),
|
||||
}
|
||||
|
||||
groups := findMergeGroups(patterns, 3)
|
||||
assert.Empty(t, groups)
|
||||
},
|
||||
)
|
||||
|
||||
t.Run(
|
||||
"heuristic and statistical patterns coexist",
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
patterns := coredata.TrackerPatterns{
|
||||
makePattern("ph_phc_XBwJ2pHAf0MoYgh3TNZK32Qk7zLlTldhk4p9llGtZMN_window_id", &oneYear),
|
||||
makePattern("foo_bar_aaa", &oneYear),
|
||||
makePattern("foo_bar_bbb", &oneYear),
|
||||
makePattern("foo_bar_ccc", &oneYear),
|
||||
}
|
||||
|
||||
groups := findMergeGroups(patterns, 3)
|
||||
require.Len(t, groups, 2)
|
||||
|
||||
heuristicGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "ph_phc_*_window_id", durationBucket: durationBucket(&oneYear)}]
|
||||
require.True(t, ok)
|
||||
assert.Len(t, heuristicGroup, 1)
|
||||
|
||||
statGroup, ok := groups[mergeGroupKey{categoryID: gid.Nil, trackerType: coredata.TrackerTypeCookie, template: "foo_bar_*", durationBucket: durationBucket(&oneYear)}]
|
||||
require.True(t, ok)
|
||||
assert.Len(t, statGroup, 3)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func TestDurationBucket(t *testing.T) {
|
||||
|
||||
@@ -240,16 +240,17 @@ WHERE
|
||||
AND tracker_type = @tracker_type
|
||||
AND (
|
||||
(match_type = @match_type_glob
|
||||
AND starts_with(@identifier, split_part(pattern, '*', 1))
|
||||
AND right(@identifier, length(split_part(pattern, '*', 2))) = split_part(pattern, '*', 2)
|
||||
AND length(@identifier) >= length(pattern) - 1)
|
||||
AND @identifier LIKE
|
||||
replace(replace(replace(replace(
|
||||
pattern, E'\\', E'\\\\'), '%', E'\\%'), '_', E'\\_'), '*', '%')
|
||||
ESCAPE E'\\')
|
||||
OR (match_type = @match_type_exact AND pattern = @identifier)
|
||||
)
|
||||
ORDER BY
|
||||
CASE WHEN match_type = @match_type_exact AND pattern = @identifier THEN 0
|
||||
ELSE 1
|
||||
END,
|
||||
length(pattern) - 1 DESC
|
||||
length(replace(pattern, '*', '')) DESC
|
||||
LIMIT 1;
|
||||
`
|
||||
|
||||
|
||||
Reference in New Issue
Block a user