diff --git a/pkg/cookiebanner/tracker_mapping_agent_test.go b/pkg/cookiebanner/tracker_mapping_agent_test.go index 9956e9051..aea3ce8c0 100644 --- a/pkg/cookiebanner/tracker_mapping_agent_test.go +++ b/pkg/cookiebanner/tracker_mapping_agent_test.go @@ -130,32 +130,6 @@ func TestNameIsCookieDatabaseAggregator(t *testing.T) { } } -func TestNormalizeAlnum(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - input string - expected string - }{ - {name: "letters lowercased", input: "Letaido", expected: "letaido"}, - {name: "strips punctuation and spaces", input: "letaido.com - Inc", expected: "letaidocominc"}, - {name: "keeps digits", input: "auth0", expected: "auth0"}, - {name: "empty", input: "", expected: ""}, - {name: "only punctuation", input: "-_.:", expected: ""}, - } - - for _, tt := range tests { - t.Run( - tt.name, - func(t *testing.T) { - t.Parallel() - assert.Equal(t, tt.expected, normalizeAlnum(tt.input)) - }, - ) - } -} - func TestBuildAgentPrompt(t *testing.T) { t.Parallel() diff --git a/pkg/cookiebanner/tracker_mapping_worker.go b/pkg/cookiebanner/tracker_mapping_worker.go index f38e3429c..079a5c340 100644 --- a/pkg/cookiebanner/tracker_mapping_worker.go +++ b/pkg/cookiebanner/tracker_mapping_worker.go @@ -28,6 +28,7 @@ import ( "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/gid" "go.probo.inc/probo/pkg/llm" + "go.probo.inc/probo/pkg/strutil" "go.probo.inc/probo/pkg/thirdparty" "go.probo.inc/probo/pkg/uri" ) @@ -715,15 +716,15 @@ func nameMatchesSiteDomain(name, siteOrigin string) bool { return false } - normalizedName := normalizeAlnum(name) + normalizedName := strutil.NormalizeAlnum(name) if normalizedName == "" { return false } label, _, _ := strings.Cut(domain, ".") - return normalizedName == normalizeAlnum(domain) || - normalizedName == normalizeAlnum(label) + return normalizedName == strutil.NormalizeAlnum(domain) || + normalizedName == strutil.NormalizeAlnum(label) } // cookieDatabaseAggregators holds alphanumeric-normalised names of pure @@ -746,18 +747,18 @@ var cookieDatabaseAggregators = map[string]struct{}{ // is a known cookie-database directory operator that must never be // attributed a tracker. The agent may return either a brand name // ("Cookiepedia") or a domain form ("cookiedatabase.org"); the latter -// would survive a plain normalised lookup because normalizeAlnum folds +// would survive a plain normalised lookup because NormalizeAlnum folds // the eTLD into the key (e.g. "cookiedatabaseorg"). To catch both forms // the candidate is also reduced to its primary domain label before the // alphanumeric-normalised lookup. The comparison is alphanumeric- // normalised so spacing, punctuation, and casing differences do not // matter. func nameIsCookieDatabaseAggregator(name string) bool { - if _, ok := cookieDatabaseAggregators[normalizeAlnum(name)]; ok { + if _, ok := cookieDatabaseAggregators[strutil.NormalizeAlnum(name)]; ok { return true } - label := normalizeAlnum(uri.DomainLabel(name)) + label := strutil.NormalizeAlnum(uri.DomainLabel(name)) if label == "" { return false } @@ -767,22 +768,6 @@ func nameIsCookieDatabaseAggregator(name string) bool { return ok } -// normalizeAlnum lowercases s and keeps only ASCII letters and digits, -// so vendor names and domains can be compared free of spacing, -// punctuation, and casing differences (e.g. "Letaido" and "letaido.com" -// both reduce to a comparable form). -func normalizeAlnum(s string) string { - var b strings.Builder - - for _, r := range strings.ToLower(s) { - if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { - b.WriteRune(r) - } - } - - return b.String() -} - // persistAgentIdentification writes a confident agent identification: // it resolves or creates the catalog third party and upserts the // catalog pattern row that links to it. It runs inside the caller's diff --git a/pkg/strutil/strutil.go b/pkg/strutil/strutil.go new file mode 100644 index 000000000..a42adcec5 --- /dev/null +++ b/pkg/strutil/strutil.go @@ -0,0 +1,35 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +// Package strutil holds small, dependency-free string helpers shared +// across packages. +package strutil + +import "strings" + +// NormalizeAlnum lowercases s and keeps only ASCII letters and digits, so +// vendor names and domains can be compared free of spacing, punctuation, +// and casing differences (e.g. "Letaido" and "letaido.com" both reduce to +// a comparable form). +func NormalizeAlnum(s string) string { + var b strings.Builder + + for _, r := range strings.ToLower(s) { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + b.WriteRune(r) + } + } + + return b.String() +} diff --git a/pkg/strutil/strutil_test.go b/pkg/strutil/strutil_test.go new file mode 100644 index 000000000..ff812f97c --- /dev/null +++ b/pkg/strutil/strutil_test.go @@ -0,0 +1,48 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package strutil_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.probo.inc/probo/pkg/strutil" +) + +func TestNormalizeAlnum(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + expected string + }{ + {name: "letters lowercased", input: "Letaido", expected: "letaido"}, + {name: "strips punctuation and spaces", input: "letaido.com - Inc", expected: "letaidocominc"}, + {name: "keeps digits", input: "auth0", expected: "auth0"}, + {name: "empty", input: "", expected: ""}, + {name: "only punctuation", input: "-_.:", expected: ""}, + } + + for _, tt := range tests { + t.Run( + tt.name, + func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.expected, strutil.NormalizeAlnum(tt.input)) + }, + ) + } +} diff --git a/pkg/thirdparty/common_third_party_owned_domains.go b/pkg/thirdparty/common_third_party_owned_domains.go index a984ad1af..ca78fd53d 100644 --- a/pkg/thirdparty/common_third_party_owned_domains.go +++ b/pkg/thirdparty/common_third_party_owned_domains.go @@ -18,6 +18,7 @@ import ( "slices" "strings" + "go.probo.inc/probo/pkg/strutil" "go.probo.inc/probo/pkg/uri" ) @@ -157,7 +158,7 @@ func vendorLabels(name, website string) []string { } add(uri.DomainLabel(website)) - add(normalizeAlnum(name)) + add(strutil.NormalizeAlnum(name)) return labels } @@ -213,18 +214,3 @@ func normalizeToETLD1(raw string) string { return uri.ExtractDomain(raw) } - -// normalizeAlnum lowercases a string and drops every non-alphanumeric -// rune, so "Dark Reader" and "DarkReader, Inc." both reduce to a -// comparable label root. -func normalizeAlnum(s string) string { - var b strings.Builder - - for _, r := range strings.ToLower(s) { - if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { - b.WriteRune(r) - } - } - - return b.String() -}