rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extract shared NormalizeAlnum into pkg/strutil

Browse Source 
The alphanumeric-normalisation helper was duplicated verbatim in the
common-third-party owned-domains resolver and the cookiebanner tracker
mapping worker. Hoist it into a new dependency-free strutil package so
both call sites share one implementation and one test.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-06-12 10:59:31 +02:00
parent 474074480a
commit 213ac6ddbe
5 changed files with 92 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
pkg/cookiebanner/tracker_mapping_agent_test.go
View File

@@ -130,32 +130,6 @@ func TestNameIsCookieDatabaseAggregator(t *testing.T) {
}
}
func TestNormalizeAlnum(t *testing.T) {
t.Parallel()
tests := []struct {
name string
input string
expected string
}{
{name: "letters lowercased", input: "Letaido", expected: "letaido"},
{name: "strips punctuation and spaces", input: "letaido.com - Inc", expected: "letaidocominc"},
{name: "keeps digits", input: "auth0", expected: "auth0"},
{name: "empty", input: "", expected: ""},
{name: "only punctuation", input: "-_.:", expected: ""},
}
for _, tt := range tests {
t.Run(
tt.name,
func(t *testing.T) {
t.Parallel()
assert.Equal(t, tt.expected, normalizeAlnum(tt.input))
},
)
}
}
func TestBuildAgentPrompt(t *testing.T) {
t.Parallel()

29
pkg/cookiebanner/tracker_mapping_worker.go
View File

@@ -28,6 +28,7 @@ import (
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/llm"
"go.probo.inc/probo/pkg/strutil"
"go.probo.inc/probo/pkg/thirdparty"
"go.probo.inc/probo/pkg/uri"
)
@@ -715,15 +716,15 @@ func nameMatchesSiteDomain(name, siteOrigin string) bool {
return false
}
normalizedName := normalizeAlnum(name)
normalizedName := strutil.NormalizeAlnum(name)
if normalizedName == "" {
return false
}
label, _, _ := strings.Cut(domain, ".")
return normalizedName == normalizeAlnum(domain) ||
normalizedName == normalizeAlnum(label)
return normalizedName == strutil.NormalizeAlnum(domain) ||
normalizedName == strutil.NormalizeAlnum(label)
}
// cookieDatabaseAggregators holds alphanumeric-normalised names of pure
@@ -746,18 +747,18 @@ var cookieDatabaseAggregators = map[string]struct{}{
// is a known cookie-database directory operator that must never be
// attributed a tracker. The agent may return either a brand name
// ("Cookiepedia") or a domain form ("cookiedatabase.org"); the latter
// would survive a plain normalised lookup because normalizeAlnum folds
// would survive a plain normalised lookup because NormalizeAlnum folds
// the eTLD into the key (e.g. "cookiedatabaseorg"). To catch both forms
// the candidate is also reduced to its primary domain label before the
// alphanumeric-normalised lookup. The comparison is alphanumeric-
// normalised so spacing, punctuation, and casing differences do not
// matter.
func nameIsCookieDatabaseAggregator(name string) bool {
if _, ok := cookieDatabaseAggregators[normalizeAlnum(name)]; ok {
if _, ok := cookieDatabaseAggregators[strutil.NormalizeAlnum(name)]; ok {
return true
}
label := normalizeAlnum(uri.DomainLabel(name))
label := strutil.NormalizeAlnum(uri.DomainLabel(name))
if label == "" {
return false
}
@@ -767,22 +768,6 @@ func nameIsCookieDatabaseAggregator(name string) bool {
return ok
}
// normalizeAlnum lowercases s and keeps only ASCII letters and digits,
// so vendor names and domains can be compared free of spacing,
// punctuation, and casing differences (e.g. "Letaido" and "letaido.com"
// both reduce to a comparable form).
func normalizeAlnum(s string) string {
var b strings.Builder
for _, r := range strings.ToLower(s) {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
b.WriteRune(r)
}
}
return b.String()
}
// persistAgentIdentification writes a confident agent identification:
// it resolves or creates the catalog third party and upserts the
// catalog pattern row that links to it. It runs inside the caller's

35
pkg/strutil/strutil.go Normal file
View File

@@ -0,0 +1,35 @@
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
// Package strutil holds small, dependency-free string helpers shared
// across packages.
package strutil
import "strings"
// NormalizeAlnum lowercases s and keeps only ASCII letters and digits, so
// vendor names and domains can be compared free of spacing, punctuation,
// and casing differences (e.g. "Letaido" and "letaido.com" both reduce to
// a comparable form).
func NormalizeAlnum(s string) string {
var b strings.Builder
for _, r := range strings.ToLower(s) {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
b.WriteRune(r)
}
}
return b.String()
}

48
pkg/strutil/strutil_test.go Normal file
View File

@@ -0,0 +1,48 @@
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package strutil_test
import (
"testing"
"github.com/stretchr/testify/assert"
"go.probo.inc/probo/pkg/strutil"
)
func TestNormalizeAlnum(t *testing.T) {
t.Parallel()
tests := []struct {
name string
input string
expected string
}{
{name: "letters lowercased", input: "Letaido", expected: "letaido"},
{name: "strips punctuation and spaces", input: "letaido.com - Inc", expected: "letaidocominc"},
{name: "keeps digits", input: "auth0", expected: "auth0"},
{name: "empty", input: "", expected: ""},
{name: "only punctuation", input: "-_.:", expected: ""},
}
for _, tt := range tests {
t.Run(
tt.name,
func(t *testing.T) {
t.Parallel()
assert.Equal(t, tt.expected, strutil.NormalizeAlnum(tt.input))
},
)
}
}

18
pkg/thirdparty/common_third_party_owned_domains.go vendored
View File

@@ -18,6 +18,7 @@ import (
"slices"
"strings"
"go.probo.inc/probo/pkg/strutil"
"go.probo.inc/probo/pkg/uri"
)
@@ -157,7 +158,7 @@ func vendorLabels(name, website string) []string {
}
add(uri.DomainLabel(website))
add(normalizeAlnum(name))
add(strutil.NormalizeAlnum(name))
return labels
}
@@ -213,18 +214,3 @@ func normalizeToETLD1(raw string) string {
return uri.ExtractDomain(raw)
}
// normalizeAlnum lowercases a string and drops every non-alphanumeric
// rune, so "Dark Reader" and "DarkReader, Inc." both reduce to a
// comparable label root.
func normalizeAlnum(s string) string {
var b strings.Builder
for _, r := range strings.ToLower(s) {
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
b.WriteRune(r)
}
}
return b.String()
}