Files
probo/pkg/thirdparty/common_third_party_owned_domains_test.go
Émile Ré 7649f19e33 Discover and persist common third-party domains
Add a domain-discovery step to the enrichment pipeline so the catalog's
domain set, previously written only by the curated seed, grows
automatically. A focused agent enumerates the registrable domains a
vendor owns and operates - marketing, product and sub-brand, app, API,
and CDN/asset domains - from links seen while browsing and from web
search, anchored on the website resolved earlier in the run.

A deterministic ownership gate reduces the candidates to eTLD+1 and
keeps only those that clear a strict confidence floor and match the
vendor by domain label. Shared tracker-delivery and CDN infrastructure
is dropped unless the vendor itself is that provider, in which case its
own brand-matching domain passes a stricter exact-label check. The
survivors are upserted into common_third_party_domains in the run's
final transaction and recorded in the enrichment payload, feeding the
tracker-mapping domain step and disambiguation.

Signed-off-by: Émile Ré <emile@probo.com>
2026-06-12 14:39:51 +02:00

160 lines
3.8 KiB
Go

// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package thirdparty
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestResolveOwnedDomains(t *testing.T) {
t.Parallel()
const threshold = 0.85
domainsOf := func(owned []ownedDomain) []string {
out := make([]string, len(owned))
for i, d := range owned {
out[i] = d.Domain
}
return out
}
t.Run(
"keeps website and brand-related domains, drops unrelated and generic",
func(t *testing.T) {
t.Parallel()
got := resolveOwnedDomains(
"Dark Reader",
"https://darkreader.org",
DomainsResult{
Domains: []DomainCandidate{
{Domain: "darkreader.org", Confidence: 0.95},
{Domain: "darkreader.ltd", Confidence: 0.9},
{Domain: "https://api.darkreader.org/v1", Confidence: 0.9},
{Domain: "cloudflare.com", Confidence: 0.9},
{Domain: "googleapis.com", Confidence: 0.95},
},
},
threshold,
)
assert.Equal(t, []string{"darkreader.org", "darkreader.ltd"}, domainsOf(got))
},
)
t.Run(
"always includes the website domain even when absent from candidates",
func(t *testing.T) {
t.Parallel()
got := resolveOwnedDomains(
"Dark Reader",
"https://darkreader.org",
DomainsResult{},
threshold,
)
assert.Equal(t, []string{"darkreader.org"}, domainsOf(got))
assert.Equal(t, 1.0, got[0].Confidence)
assert.Equal(t, "https://darkreader.org", got[0].SourceURL)
},
)
t.Run(
"drops candidates below the confidence threshold",
func(t *testing.T) {
t.Parallel()
got := resolveOwnedDomains(
"Dark Reader",
"https://darkreader.org",
DomainsResult{
Domains: []DomainCandidate{
{Domain: "darkreader.io", Confidence: 0.5},
},
},
threshold,
)
assert.Equal(t, []string{"darkreader.org"}, domainsOf(got))
},
)
t.Run(
"keeps a separate brand CDN domain",
func(t *testing.T) {
t.Parallel()
got := resolveOwnedDomains(
"Intercom",
"https://intercom.com",
DomainsResult{
Domains: []DomainCandidate{
{Domain: "intercomcdn.com", Confidence: 0.9},
},
},
threshold,
)
assert.Equal(t, []string{"intercom.com", "intercomcdn.com"}, domainsOf(got))
},
)
t.Run(
"keeps the vendor's own shared-infrastructure domain when the vendor is that provider",
func(t *testing.T) {
t.Parallel()
got := resolveOwnedDomains(
"Fastly",
"https://fastly.com",
DomainsResult{
Domains: []DomainCandidate{
{Domain: "fastly.net", Confidence: 0.95},
},
},
threshold,
)
assert.Equal(t, []string{"fastly.com", "fastly.net"}, domainsOf(got))
},
)
t.Run(
"drops shared-infrastructure domains for an unrelated vendor even at high confidence",
func(t *testing.T) {
t.Parallel()
got := resolveOwnedDomains(
"Dark Reader",
"https://darkreader.org",
DomainsResult{
Domains: []DomainCandidate{
{Domain: "amazonaws.com", Confidence: 0.99},
{Domain: "fastly.net", Confidence: 0.99},
},
},
threshold,
)
assert.Equal(t, []string{"darkreader.org"}, domainsOf(got))
},
)
}