Extract shared NormalizeAlnum into pkg/strutil
The alphanumeric-normalisation helper was duplicated verbatim in the common-third-party owned-domains resolver and the cookiebanner tracker mapping worker. Hoist it into a new dependency-free strutil package so both call sites share one implementation and one test. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -130,32 +130,6 @@ func TestNameIsCookieDatabaseAggregator(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNormalizeAlnum(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
input string
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
{name: "letters lowercased", input: "Letaido", expected: "letaido"},
|
|
||||||
{name: "strips punctuation and spaces", input: "letaido.com - Inc", expected: "letaidocominc"},
|
|
||||||
{name: "keeps digits", input: "auth0", expected: "auth0"},
|
|
||||||
{name: "empty", input: "", expected: ""},
|
|
||||||
{name: "only punctuation", input: "-_.:", expected: ""},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(
|
|
||||||
tt.name,
|
|
||||||
func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
assert.Equal(t, tt.expected, normalizeAlnum(tt.input))
|
|
||||||
},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildAgentPrompt(t *testing.T) {
|
func TestBuildAgentPrompt(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ import (
|
|||||||
"go.probo.inc/probo/pkg/coredata"
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
"go.probo.inc/probo/pkg/gid"
|
"go.probo.inc/probo/pkg/gid"
|
||||||
"go.probo.inc/probo/pkg/llm"
|
"go.probo.inc/probo/pkg/llm"
|
||||||
|
"go.probo.inc/probo/pkg/strutil"
|
||||||
"go.probo.inc/probo/pkg/thirdparty"
|
"go.probo.inc/probo/pkg/thirdparty"
|
||||||
"go.probo.inc/probo/pkg/uri"
|
"go.probo.inc/probo/pkg/uri"
|
||||||
)
|
)
|
||||||
@@ -715,15 +716,15 @@ func nameMatchesSiteDomain(name, siteOrigin string) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
normalizedName := normalizeAlnum(name)
|
normalizedName := strutil.NormalizeAlnum(name)
|
||||||
if normalizedName == "" {
|
if normalizedName == "" {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
label, _, _ := strings.Cut(domain, ".")
|
label, _, _ := strings.Cut(domain, ".")
|
||||||
|
|
||||||
return normalizedName == normalizeAlnum(domain) ||
|
return normalizedName == strutil.NormalizeAlnum(domain) ||
|
||||||
normalizedName == normalizeAlnum(label)
|
normalizedName == strutil.NormalizeAlnum(label)
|
||||||
}
|
}
|
||||||
|
|
||||||
// cookieDatabaseAggregators holds alphanumeric-normalised names of pure
|
// cookieDatabaseAggregators holds alphanumeric-normalised names of pure
|
||||||
@@ -746,18 +747,18 @@ var cookieDatabaseAggregators = map[string]struct{}{
|
|||||||
// is a known cookie-database directory operator that must never be
|
// is a known cookie-database directory operator that must never be
|
||||||
// attributed a tracker. The agent may return either a brand name
|
// attributed a tracker. The agent may return either a brand name
|
||||||
// ("Cookiepedia") or a domain form ("cookiedatabase.org"); the latter
|
// ("Cookiepedia") or a domain form ("cookiedatabase.org"); the latter
|
||||||
// would survive a plain normalised lookup because normalizeAlnum folds
|
// would survive a plain normalised lookup because NormalizeAlnum folds
|
||||||
// the eTLD into the key (e.g. "cookiedatabaseorg"). To catch both forms
|
// the eTLD into the key (e.g. "cookiedatabaseorg"). To catch both forms
|
||||||
// the candidate is also reduced to its primary domain label before the
|
// the candidate is also reduced to its primary domain label before the
|
||||||
// alphanumeric-normalised lookup. The comparison is alphanumeric-
|
// alphanumeric-normalised lookup. The comparison is alphanumeric-
|
||||||
// normalised so spacing, punctuation, and casing differences do not
|
// normalised so spacing, punctuation, and casing differences do not
|
||||||
// matter.
|
// matter.
|
||||||
func nameIsCookieDatabaseAggregator(name string) bool {
|
func nameIsCookieDatabaseAggregator(name string) bool {
|
||||||
if _, ok := cookieDatabaseAggregators[normalizeAlnum(name)]; ok {
|
if _, ok := cookieDatabaseAggregators[strutil.NormalizeAlnum(name)]; ok {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
label := normalizeAlnum(uri.DomainLabel(name))
|
label := strutil.NormalizeAlnum(uri.DomainLabel(name))
|
||||||
if label == "" {
|
if label == "" {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
@@ -767,22 +768,6 @@ func nameIsCookieDatabaseAggregator(name string) bool {
|
|||||||
return ok
|
return ok
|
||||||
}
|
}
|
||||||
|
|
||||||
// normalizeAlnum lowercases s and keeps only ASCII letters and digits,
|
|
||||||
// so vendor names and domains can be compared free of spacing,
|
|
||||||
// punctuation, and casing differences (e.g. "Letaido" and "letaido.com"
|
|
||||||
// both reduce to a comparable form).
|
|
||||||
func normalizeAlnum(s string) string {
|
|
||||||
var b strings.Builder
|
|
||||||
|
|
||||||
for _, r := range strings.ToLower(s) {
|
|
||||||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
|
|
||||||
b.WriteRune(r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// persistAgentIdentification writes a confident agent identification:
|
// persistAgentIdentification writes a confident agent identification:
|
||||||
// it resolves or creates the catalog third party and upserts the
|
// it resolves or creates the catalog third party and upserts the
|
||||||
// catalog pattern row that links to it. It runs inside the caller's
|
// catalog pattern row that links to it. It runs inside the caller's
|
||||||
|
|||||||
35
pkg/strutil/strutil.go
Normal file
35
pkg/strutil/strutil.go
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
// Package strutil holds small, dependency-free string helpers shared
|
||||||
|
// across packages.
|
||||||
|
package strutil
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
// NormalizeAlnum lowercases s and keeps only ASCII letters and digits, so
|
||||||
|
// vendor names and domains can be compared free of spacing, punctuation,
|
||||||
|
// and casing differences (e.g. "Letaido" and "letaido.com" both reduce to
|
||||||
|
// a comparable form).
|
||||||
|
func NormalizeAlnum(s string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
|
||||||
|
for _, r := range strings.ToLower(s) {
|
||||||
|
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
|
||||||
|
b.WriteRune(r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
48
pkg/strutil/strutil_test.go
Normal file
48
pkg/strutil/strutil_test.go
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package strutil_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"go.probo.inc/probo/pkg/strutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNormalizeAlnum(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{name: "letters lowercased", input: "Letaido", expected: "letaido"},
|
||||||
|
{name: "strips punctuation and spaces", input: "letaido.com - Inc", expected: "letaidocominc"},
|
||||||
|
{name: "keeps digits", input: "auth0", expected: "auth0"},
|
||||||
|
{name: "empty", input: "", expected: ""},
|
||||||
|
{name: "only punctuation", input: "-_.:", expected: ""},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(
|
||||||
|
tt.name,
|
||||||
|
func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
assert.Equal(t, tt.expected, strutil.NormalizeAlnum(tt.input))
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -18,6 +18,7 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"go.probo.inc/probo/pkg/strutil"
|
||||||
"go.probo.inc/probo/pkg/uri"
|
"go.probo.inc/probo/pkg/uri"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -157,7 +158,7 @@ func vendorLabels(name, website string) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
add(uri.DomainLabel(website))
|
add(uri.DomainLabel(website))
|
||||||
add(normalizeAlnum(name))
|
add(strutil.NormalizeAlnum(name))
|
||||||
|
|
||||||
return labels
|
return labels
|
||||||
}
|
}
|
||||||
@@ -213,18 +214,3 @@ func normalizeToETLD1(raw string) string {
|
|||||||
|
|
||||||
return uri.ExtractDomain(raw)
|
return uri.ExtractDomain(raw)
|
||||||
}
|
}
|
||||||
|
|
||||||
// normalizeAlnum lowercases a string and drops every non-alphanumeric
|
|
||||||
// rune, so "Dark Reader" and "DarkReader, Inc." both reduce to a
|
|
||||||
// comparable label root.
|
|
||||||
func normalizeAlnum(s string) string {
|
|
||||||
var b strings.Builder
|
|
||||||
|
|
||||||
for _, r := range strings.ToLower(s) {
|
|
||||||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') {
|
|
||||||
b.WriteRune(r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|||||||
Reference in New Issue
Block a user