Add a domain-discovery step to the enrichment pipeline so the catalog's domain set, previously written only by the curated seed, grows automatically. A focused agent enumerates the registrable domains a vendor owns and operates - marketing, product and sub-brand, app, API, and CDN/asset domains - from links seen while browsing and from web search, anchored on the website resolved earlier in the run. A deterministic ownership gate reduces the candidates to eTLD+1 and keeps only those that clear a strict confidence floor and match the vendor by domain label. Shared tracker-delivery and CDN infrastructure is dropped unless the vendor itself is that provider, in which case its own brand-matching domain passes a stricter exact-label check. The survivors are upserted into common_third_party_domains in the run's final transaction and recorded in the enrichment payload, feeding the tracker-mapping domain step and disambiguation. Signed-off-by: Émile Ré <emile@probo.com>
160 lines
3.8 KiB
Go
160 lines
3.8 KiB
Go
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
|
//
|
|
// Permission to use, copy, modify, and/or distribute this software for any
|
|
// purpose with or without fee is hereby granted, provided that the above
|
|
// copyright notice and this permission notice appear in all copies.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
|
// PERFORMANCE OF THIS SOFTWARE.
|
|
|
|
package thirdparty
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestResolveOwnedDomains(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const threshold = 0.85
|
|
|
|
domainsOf := func(owned []ownedDomain) []string {
|
|
out := make([]string, len(owned))
|
|
for i, d := range owned {
|
|
out[i] = d.Domain
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
t.Run(
|
|
"keeps website and brand-related domains, drops unrelated and generic",
|
|
func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got := resolveOwnedDomains(
|
|
"Dark Reader",
|
|
"https://darkreader.org",
|
|
DomainsResult{
|
|
Domains: []DomainCandidate{
|
|
{Domain: "darkreader.org", Confidence: 0.95},
|
|
{Domain: "darkreader.ltd", Confidence: 0.9},
|
|
{Domain: "https://api.darkreader.org/v1", Confidence: 0.9},
|
|
{Domain: "cloudflare.com", Confidence: 0.9},
|
|
{Domain: "googleapis.com", Confidence: 0.95},
|
|
},
|
|
},
|
|
threshold,
|
|
)
|
|
|
|
assert.Equal(t, []string{"darkreader.org", "darkreader.ltd"}, domainsOf(got))
|
|
},
|
|
)
|
|
|
|
t.Run(
|
|
"always includes the website domain even when absent from candidates",
|
|
func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got := resolveOwnedDomains(
|
|
"Dark Reader",
|
|
"https://darkreader.org",
|
|
DomainsResult{},
|
|
threshold,
|
|
)
|
|
|
|
assert.Equal(t, []string{"darkreader.org"}, domainsOf(got))
|
|
assert.Equal(t, 1.0, got[0].Confidence)
|
|
assert.Equal(t, "https://darkreader.org", got[0].SourceURL)
|
|
},
|
|
)
|
|
|
|
t.Run(
|
|
"drops candidates below the confidence threshold",
|
|
func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got := resolveOwnedDomains(
|
|
"Dark Reader",
|
|
"https://darkreader.org",
|
|
DomainsResult{
|
|
Domains: []DomainCandidate{
|
|
{Domain: "darkreader.io", Confidence: 0.5},
|
|
},
|
|
},
|
|
threshold,
|
|
)
|
|
|
|
assert.Equal(t, []string{"darkreader.org"}, domainsOf(got))
|
|
},
|
|
)
|
|
|
|
t.Run(
|
|
"keeps a separate brand CDN domain",
|
|
func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got := resolveOwnedDomains(
|
|
"Intercom",
|
|
"https://intercom.com",
|
|
DomainsResult{
|
|
Domains: []DomainCandidate{
|
|
{Domain: "intercomcdn.com", Confidence: 0.9},
|
|
},
|
|
},
|
|
threshold,
|
|
)
|
|
|
|
assert.Equal(t, []string{"intercom.com", "intercomcdn.com"}, domainsOf(got))
|
|
},
|
|
)
|
|
|
|
t.Run(
|
|
"keeps the vendor's own shared-infrastructure domain when the vendor is that provider",
|
|
func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got := resolveOwnedDomains(
|
|
"Fastly",
|
|
"https://fastly.com",
|
|
DomainsResult{
|
|
Domains: []DomainCandidate{
|
|
{Domain: "fastly.net", Confidence: 0.95},
|
|
},
|
|
},
|
|
threshold,
|
|
)
|
|
|
|
assert.Equal(t, []string{"fastly.com", "fastly.net"}, domainsOf(got))
|
|
},
|
|
)
|
|
|
|
t.Run(
|
|
"drops shared-infrastructure domains for an unrelated vendor even at high confidence",
|
|
func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got := resolveOwnedDomains(
|
|
"Dark Reader",
|
|
"https://darkreader.org",
|
|
DomainsResult{
|
|
Domains: []DomainCandidate{
|
|
{Domain: "amazonaws.com", Confidence: 0.99},
|
|
{Domain: "fastly.net", Confidence: 0.99},
|
|
},
|
|
},
|
|
threshold,
|
|
)
|
|
|
|
assert.Equal(t, []string{"darkreader.org"}, domainsOf(got))
|
|
},
|
|
)
|
|
}
|