Map trackers by sibling patterns sharing an origin
Tracker patterns detected on the same banner that share initiator domains are a strong indicator of the same third party. Previously the mapping worker only checked the global third-party domain catalog, so a tracker whose domain was not registered there fell through to the expensive LLM identification step even when a co-located pattern was already mapped. Add a matchBySiblingOrigin step that finds other patterns on the same banner sharing the same initiator domains and reuses their resolved common third party. It prefers siblings already promoted to an org third party (the strongest signal) and falls back to siblings carrying only a catalog link, skipping when the siblings disagree. The step runs before the catalog domain lookup since an already-qualified sibling is at least as reliable as a raw domain match. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -117,13 +117,33 @@ func (h *trackerMappingHandler) Process(ctx context.Context, tp coredata.Tracker
|
||||
return fmt.Errorf("cannot load cookie banner for domain filtering: %w", err)
|
||||
}
|
||||
|
||||
banner.Origin = "https://t.probo.com"
|
||||
|
||||
commonPatternID, err = h.matchByPattern(ctx, tx, tp)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot match by pattern: %w", err)
|
||||
}
|
||||
|
||||
var domains []string
|
||||
|
||||
if commonPatternID == nil {
|
||||
commonPatternID, err = h.matchByDomain(ctx, tx, tp, banner.Origin)
|
||||
var trackers coredata.DetectedTrackers
|
||||
|
||||
domains, err = trackers.LoadInitiatorDomainsByTrackerPatternID(ctx, tx, tp.ID, 10)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot load initiator domains: %w", err)
|
||||
}
|
||||
|
||||
domains = uri.FilterFirstPartyDomains(domains, banner.Origin)
|
||||
|
||||
commonPatternID, err = h.matchBySiblingOrigin(ctx, tx, tp, domains)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot match by sibling origin: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if commonPatternID == nil {
|
||||
commonPatternID, err = h.matchByDomain(ctx, tx, tp, domains)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot match by domain: %w", err)
|
||||
}
|
||||
@@ -221,25 +241,17 @@ func (h *trackerMappingHandler) matchByPattern(
|
||||
// CommonTrackerPattern linking the two. As with matchByPattern,
|
||||
// third-party resolution is deferred to promoteThirdParty.
|
||||
//
|
||||
// Domains that share the scanned site's eTLD+1 are filtered out before
|
||||
// querying. Tracker scripts loaded through a first-party proxy (e.g.
|
||||
// t.probo.com proxying PostHog on a probo.com site) would otherwise
|
||||
// match the site owner's own CommonThirdParty entry.
|
||||
// The caller is responsible for loading and filtering the domains
|
||||
// (removing first-party domains). Tracker scripts loaded through a
|
||||
// first-party proxy (e.g. t.probo.com proxying PostHog on a probo.com
|
||||
// site) would otherwise match the site owner's own CommonThirdParty
|
||||
// entry.
|
||||
func (h *trackerMappingHandler) matchByDomain(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
tp coredata.TrackerPattern,
|
||||
siteOrigin string,
|
||||
domains []string,
|
||||
) (*gid.GID, error) {
|
||||
var trackers coredata.DetectedTrackers
|
||||
|
||||
domains, err := trackers.LoadInitiatorDomainsByTrackerPatternID(ctx, tx, tp.ID, 10)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot load initiator domains: %w", err)
|
||||
}
|
||||
|
||||
domains = uri.FilterFirstPartyDomains(domains, siteOrigin)
|
||||
|
||||
if len(domains) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -440,6 +452,151 @@ func (h *trackerMappingHandler) resolveOrCreateCommonThirdParty(
|
||||
return &party.ID, nil
|
||||
}
|
||||
|
||||
// matchBySiblingOrigin finds other tracker patterns on the same banner
|
||||
// that share initiator domains with the current pattern and are already
|
||||
// mapped to a common third party. Sharing an origin across multiple
|
||||
// detected patterns is a strong indicator of the same third party.
|
||||
func (h *trackerMappingHandler) matchBySiblingOrigin(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
tp coredata.TrackerPattern,
|
||||
domains []string,
|
||||
) (*gid.GID, error) {
|
||||
if len(domains) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var trackers coredata.DetectedTrackers
|
||||
|
||||
siblingIDs, err := trackers.LoadSiblingPatternIDsByInitiatorDomains(
|
||||
ctx,
|
||||
tx,
|
||||
tp.CookieBannerID,
|
||||
domains,
|
||||
tp.ID,
|
||||
20,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot load sibling pattern ids: %w", err)
|
||||
}
|
||||
|
||||
if len(siblingIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
scope := coredata.NewScopeFromObjectID(tp.ID)
|
||||
|
||||
commonThirdPartyID, err := h.resolveCommonThirdPartyFromSiblings(ctx, tx, scope, siblingIDs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot resolve common third party from siblings: %w", err)
|
||||
}
|
||||
|
||||
if commonThirdPartyID == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
commonPattern := coredata.CommonTrackerPattern{
|
||||
ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType),
|
||||
CommonThirdPartyID: commonThirdPartyID,
|
||||
TrackerType: tp.TrackerType,
|
||||
Pattern: tp.Pattern,
|
||||
MatchType: tp.MatchType,
|
||||
Description: tp.Description,
|
||||
MaxAgeSeconds: tp.MaxAgeSeconds,
|
||||
Confidence: 0.7,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
if _, err := commonPattern.Upsert(ctx, tx); err != nil {
|
||||
return nil, fmt.Errorf("cannot upsert common tracker pattern from sibling origin: %w", err)
|
||||
}
|
||||
|
||||
h.logger.InfoCtx(
|
||||
ctx,
|
||||
"matched tracker pattern via sibling origin",
|
||||
log.String("pattern", tp.Pattern),
|
||||
log.String("tracker_pattern_id", tp.ID.String()),
|
||||
log.String("common_third_party_id", commonThirdPartyID.String()),
|
||||
)
|
||||
|
||||
return &commonPattern.ID, nil
|
||||
}
|
||||
|
||||
// resolveCommonThirdPartyFromSiblings extracts a single unambiguous
|
||||
// CommonThirdPartyID from sibling patterns. It first checks siblings
|
||||
// with a third_party_id (fully promoted), then falls back to siblings
|
||||
// with only a common_tracker_pattern_id.
|
||||
func (h *trackerMappingHandler) resolveCommonThirdPartyFromSiblings(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
scope coredata.Scoper,
|
||||
siblingIDs []gid.GID,
|
||||
) (*gid.GID, error) {
|
||||
var patterns coredata.TrackerPatterns
|
||||
|
||||
thirdPartyIDs, err := patterns.LoadDistinctThirdPartyIDsByIDs(ctx, conn, scope, siblingIDs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot load distinct third party ids from siblings: %w", err)
|
||||
}
|
||||
|
||||
if len(thirdPartyIDs) > 0 {
|
||||
commonIDs := make(map[gid.GID]struct{})
|
||||
|
||||
for _, tpID := range thirdPartyIDs {
|
||||
var tp coredata.ThirdParty
|
||||
if err := tp.LoadByID(ctx, conn, scope, tpID); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if tp.CommonThirdPartyID != nil {
|
||||
commonIDs[*tp.CommonThirdPartyID] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
if len(commonIDs) == 1 {
|
||||
for id := range commonIDs {
|
||||
return &id, nil
|
||||
}
|
||||
}
|
||||
|
||||
if len(commonIDs) > 1 {
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
commonPatternIDs, err := patterns.LoadDistinctCommonTrackerPatternIDsByIDs(ctx, conn, scope, siblingIDs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot load distinct common tracker pattern ids from siblings: %w", err)
|
||||
}
|
||||
|
||||
if len(commonPatternIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
commonIDs := make(map[gid.GID]struct{})
|
||||
|
||||
for _, cpID := range commonPatternIDs {
|
||||
var cp coredata.CommonTrackerPattern
|
||||
if err := cp.LoadByID(ctx, conn, cpID); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if cp.CommonThirdPartyID != nil {
|
||||
commonIDs[*cp.CommonThirdPartyID] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
if len(commonIDs) == 1 {
|
||||
for id := range commonIDs {
|
||||
return &id, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (h *trackerMappingHandler) createUnmatchedPattern(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
|
||||
@@ -490,6 +490,547 @@ func TestProcess_NoOpWhenAlreadyPromoted(t *testing.T) {
|
||||
assert.Equal(t, preExisting.ID, *reloaded.ThirdPartyID, "third_party_id must not be overwritten")
|
||||
}
|
||||
|
||||
func TestMatchBySiblingOrigin_SiblingWithThirdPartyID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
fx := seedPromotionFixture(t, ctx, client)
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
|
||||
orgThirdParty := coredata.ThirdParty{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.ThirdPartyEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CommonThirdPartyID: &fx.commonThirdPartyID,
|
||||
Name: "Google LLC",
|
||||
Category: coredata.ThirdPartyCategoryAnalytics,
|
||||
Certifications: []string{},
|
||||
Countries: coredata.CountryCodes{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
siblingPattern := coredata.TrackerPattern{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CookieBannerID: fx.banner.ID,
|
||||
CookieCategoryID: fx.normalCategoryID,
|
||||
CommonTrackerPatternID: &fx.commonPatternID,
|
||||
ThirdPartyID: &orgThirdParty.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "_gid",
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
DisplayName: "_gid",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
unmappedPattern := coredata.TrackerPattern{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CookieBannerID: fx.banner.ID,
|
||||
CookieCategoryID: fx.normalCategoryID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "_ga_unknown",
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
DisplayName: "_ga_unknown",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
initiatorDomain := "www.googletagmanager.com"
|
||||
siblingDetected := coredata.DetectedTracker{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType),
|
||||
CookieBannerID: fx.banner.ID,
|
||||
TrackerPatternID: &siblingPattern.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Identifier: "_gid",
|
||||
InitiatorDomain: &initiatorDomain,
|
||||
LastDetectedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
unmappedDetected := coredata.DetectedTracker{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType),
|
||||
CookieBannerID: fx.banner.ID,
|
||||
TrackerPatternID: &unmappedPattern.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Identifier: "_ga_unknown",
|
||||
InitiatorDomain: &initiatorDomain,
|
||||
LastDetectedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
if err := orgThirdParty.Insert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := siblingPattern.Insert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := unmappedPattern.Insert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := siblingDetected.Upsert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := unmappedDetected.Upsert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}))
|
||||
|
||||
h := newMappingHandler(client)
|
||||
|
||||
var got *gid.GID
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
var err error
|
||||
|
||||
got, err = h.matchBySiblingOrigin(ctx, tx, unmappedPattern, []string{"googletagmanager.com"})
|
||||
|
||||
return err
|
||||
}))
|
||||
|
||||
require.NotNil(t, got, "sibling origin match should return a common tracker pattern ID")
|
||||
|
||||
var commonPattern coredata.CommonTrackerPattern
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return commonPattern.LoadByID(ctx, conn, *got)
|
||||
}))
|
||||
|
||||
require.NotNil(t, commonPattern.CommonThirdPartyID)
|
||||
assert.Equal(t, fx.commonThirdPartyID, *commonPattern.CommonThirdPartyID)
|
||||
assert.Equal(t, float32(0.7), commonPattern.Confidence)
|
||||
}
|
||||
|
||||
func TestMatchBySiblingOrigin_AmbiguousThirdParties(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
fx := seedPromotionFixture(t, ctx, client)
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
|
||||
otherCommonThirdPartyID := gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType)
|
||||
otherCommonThirdParty := coredata.CommonThirdParty{
|
||||
ID: otherCommonThirdPartyID,
|
||||
Name: "Facebook",
|
||||
Slug: "facebook",
|
||||
Category: coredata.ThirdPartyCategoryMarketing,
|
||||
Certifications: []string{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
orgThirdPartyA := coredata.ThirdParty{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.ThirdPartyEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CommonThirdPartyID: &fx.commonThirdPartyID,
|
||||
Name: "Google",
|
||||
Category: coredata.ThirdPartyCategoryAnalytics,
|
||||
Certifications: []string{},
|
||||
Countries: coredata.CountryCodes{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
orgThirdPartyB := coredata.ThirdParty{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.ThirdPartyEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CommonThirdPartyID: &otherCommonThirdPartyID,
|
||||
Name: "Facebook",
|
||||
Category: coredata.ThirdPartyCategoryMarketing,
|
||||
Certifications: []string{},
|
||||
Countries: coredata.CountryCodes{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
siblingA := coredata.TrackerPattern{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CookieBannerID: fx.banner.ID,
|
||||
CookieCategoryID: fx.normalCategoryID,
|
||||
ThirdPartyID: &orgThirdPartyA.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "sibling_a",
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
DisplayName: "sibling_a",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
siblingB := coredata.TrackerPattern{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CookieBannerID: fx.banner.ID,
|
||||
CookieCategoryID: fx.normalCategoryID,
|
||||
ThirdPartyID: &orgThirdPartyB.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "sibling_b",
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
DisplayName: "sibling_b",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
unmappedPattern := coredata.TrackerPattern{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CookieBannerID: fx.banner.ID,
|
||||
CookieCategoryID: fx.normalCategoryID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "ambiguous_test",
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
DisplayName: "ambiguous_test",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
sharedDomain := "cdn.shared-tracker.com"
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
if err := otherCommonThirdParty.Insert(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := orgThirdPartyA.Insert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := orgThirdPartyB.Insert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := siblingA.Insert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := siblingB.Insert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := unmappedPattern.Insert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
detA := coredata.DetectedTracker{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType),
|
||||
CookieBannerID: fx.banner.ID,
|
||||
TrackerPatternID: &siblingA.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Identifier: "sibling_a",
|
||||
InitiatorDomain: &sharedDomain,
|
||||
LastDetectedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if _, err := detA.Upsert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
detB := coredata.DetectedTracker{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType),
|
||||
CookieBannerID: fx.banner.ID,
|
||||
TrackerPatternID: &siblingB.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Identifier: "sibling_b",
|
||||
InitiatorDomain: &sharedDomain,
|
||||
LastDetectedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if _, err := detB.Upsert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
detUnmapped := coredata.DetectedTracker{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType),
|
||||
CookieBannerID: fx.banner.ID,
|
||||
TrackerPatternID: &unmappedPattern.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Identifier: "ambiguous_test",
|
||||
InitiatorDomain: &sharedDomain,
|
||||
LastDetectedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if _, err := detUnmapped.Upsert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}))
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = client.WithTx(context.Background(), func(ctx context.Context, tx pg.Tx) error {
|
||||
_, _ = tx.Exec(ctx, `DELETE FROM common_third_parties WHERE id = $1`, otherCommonThirdPartyID)
|
||||
|
||||
return nil
|
||||
})
|
||||
})
|
||||
|
||||
h := newMappingHandler(client)
|
||||
|
||||
var got *gid.GID
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
var err error
|
||||
|
||||
got, err = h.matchBySiblingOrigin(ctx, tx, unmappedPattern, []string{"shared-tracker.com"})
|
||||
|
||||
return err
|
||||
}))
|
||||
|
||||
assert.Nil(t, got, "ambiguous siblings mapping to different third parties should return nil")
|
||||
}
|
||||
|
||||
func TestMatchBySiblingOrigin_NoSiblings(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
fx := seedWorkerFixture(t, ctx, client)
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
|
||||
unmappedPattern := coredata.TrackerPattern{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CookieBannerID: fx.banner.ID,
|
||||
CookieCategoryID: fx.normalCategoryID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "lonely_cookie",
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
DisplayName: "lonely_cookie",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
return unmappedPattern.Insert(ctx, tx, fx.scope)
|
||||
}))
|
||||
|
||||
h := newMappingHandler(client)
|
||||
|
||||
var got *gid.GID
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
var err error
|
||||
|
||||
got, err = h.matchBySiblingOrigin(ctx, tx, unmappedPattern, []string{"unique-domain.com"})
|
||||
|
||||
return err
|
||||
}))
|
||||
|
||||
assert.Nil(t, got, "no siblings sharing the domain should return nil")
|
||||
}
|
||||
|
||||
func TestMatchBySiblingOrigin_EmptyDomains(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
fx := seedWorkerFixture(t, ctx, client)
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
|
||||
pattern := coredata.TrackerPattern{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CookieBannerID: fx.banner.ID,
|
||||
CookieCategoryID: fx.normalCategoryID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "no_domains",
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
DisplayName: "no_domains",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
return pattern.Insert(ctx, tx, fx.scope)
|
||||
}))
|
||||
|
||||
h := newMappingHandler(client)
|
||||
|
||||
var got *gid.GID
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
var err error
|
||||
|
||||
got, err = h.matchBySiblingOrigin(ctx, tx, pattern, nil)
|
||||
|
||||
return err
|
||||
}))
|
||||
|
||||
assert.Nil(t, got, "nil domains should immediately return nil")
|
||||
}
|
||||
|
||||
func TestMatchBySiblingOrigin_ConvergentSiblings(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
fx := seedPromotionFixture(t, ctx, client)
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
|
||||
orgThirdParty := coredata.ThirdParty{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.ThirdPartyEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CommonThirdPartyID: &fx.commonThirdPartyID,
|
||||
Name: "Google",
|
||||
Category: coredata.ThirdPartyCategoryAnalytics,
|
||||
Certifications: []string{},
|
||||
Countries: coredata.CountryCodes{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
siblingA := coredata.TrackerPattern{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CookieBannerID: fx.banner.ID,
|
||||
CookieCategoryID: fx.normalCategoryID,
|
||||
ThirdPartyID: &orgThirdParty.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "converge_a",
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
DisplayName: "converge_a",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
siblingB := coredata.TrackerPattern{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CookieBannerID: fx.banner.ID,
|
||||
CookieCategoryID: fx.normalCategoryID,
|
||||
ThirdPartyID: &orgThirdParty.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "converge_b",
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
DisplayName: "converge_b",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
unmappedPattern := coredata.TrackerPattern{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CookieBannerID: fx.banner.ID,
|
||||
CookieCategoryID: fx.normalCategoryID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "converge_target",
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
DisplayName: "converge_target",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
sharedDomain := "analytics.google.com"
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
if err := orgThirdParty.Insert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := siblingA.Insert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := siblingB.Insert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := unmappedPattern.Insert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
detA := coredata.DetectedTracker{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType),
|
||||
CookieBannerID: fx.banner.ID,
|
||||
TrackerPatternID: &siblingA.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Identifier: "converge_a",
|
||||
InitiatorDomain: &sharedDomain,
|
||||
LastDetectedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if _, err := detA.Upsert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
detB := coredata.DetectedTracker{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType),
|
||||
CookieBannerID: fx.banner.ID,
|
||||
TrackerPatternID: &siblingB.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Identifier: "converge_b",
|
||||
InitiatorDomain: &sharedDomain,
|
||||
LastDetectedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if _, err := detB.Upsert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
detUnmapped := coredata.DetectedTracker{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType),
|
||||
CookieBannerID: fx.banner.ID,
|
||||
TrackerPatternID: &unmappedPattern.ID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Identifier: "converge_target",
|
||||
InitiatorDomain: &sharedDomain,
|
||||
LastDetectedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if _, err := detUnmapped.Upsert(ctx, tx, fx.scope); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}))
|
||||
|
||||
h := newMappingHandler(client)
|
||||
|
||||
var got *gid.GID
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
var err error
|
||||
|
||||
got, err = h.matchBySiblingOrigin(ctx, tx, unmappedPattern, []string{"google.com"})
|
||||
|
||||
return err
|
||||
}))
|
||||
|
||||
require.NotNil(t, got, "multiple siblings converging to same third party should succeed")
|
||||
|
||||
var commonPattern coredata.CommonTrackerPattern
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return commonPattern.LoadByID(ctx, conn, *got)
|
||||
}))
|
||||
|
||||
require.NotNil(t, commonPattern.CommonThirdPartyID)
|
||||
assert.Equal(t, fx.commonThirdPartyID, *commonPattern.CommonThirdPartyID)
|
||||
}
|
||||
|
||||
func TestPromoteThirdParty_ExactCommonLinkIgnoresSimilarUnlinked(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -252,6 +252,48 @@ WHERE
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dts *DetectedTrackers) LoadSiblingPatternIDsByInitiatorDomains(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
cookieBannerID gid.GID,
|
||||
domains []string,
|
||||
excludePatternID gid.GID,
|
||||
limit int,
|
||||
) ([]gid.GID, error) {
|
||||
if len(domains) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
q := `
|
||||
SELECT DISTINCT tracker_pattern_id
|
||||
FROM detected_trackers
|
||||
WHERE cookie_banner_id = @cookie_banner_id
|
||||
AND initiator_domain = ANY(@domains)
|
||||
AND tracker_pattern_id IS NOT NULL
|
||||
AND tracker_pattern_id != @exclude_pattern_id
|
||||
LIMIT @limit;
|
||||
`
|
||||
|
||||
args := pgx.StrictNamedArgs{
|
||||
"cookie_banner_id": cookieBannerID,
|
||||
"domains": domains,
|
||||
"exclude_pattern_id": excludePatternID,
|
||||
"limit": limit,
|
||||
}
|
||||
|
||||
rows, err := conn.Query(ctx, q, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot load sibling pattern ids: %w", err)
|
||||
}
|
||||
|
||||
ids, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot collect sibling pattern ids: %w", err)
|
||||
}
|
||||
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (dts *DetectedTrackers) RelinkByTrackerPatternID(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
|
||||
@@ -944,6 +944,80 @@ WHERE
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (tps *TrackerPatterns) LoadDistinctThirdPartyIDsByIDs(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
scope Scoper,
|
||||
ids []gid.GID,
|
||||
) ([]gid.GID, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
q := `
|
||||
SELECT DISTINCT third_party_id
|
||||
FROM tracker_patterns
|
||||
WHERE
|
||||
%s
|
||||
AND id = ANY(@ids)
|
||||
AND third_party_id IS NOT NULL
|
||||
`
|
||||
|
||||
q = fmt.Sprintf(q, scope.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{"ids": ids}
|
||||
maps.Copy(args, scope.SQLArguments())
|
||||
|
||||
rows, err := conn.Query(ctx, q, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot query distinct third party ids by pattern ids: %w", err)
|
||||
}
|
||||
|
||||
thirdPartyIDs, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot collect distinct third party ids by pattern ids: %w", err)
|
||||
}
|
||||
|
||||
return thirdPartyIDs, nil
|
||||
}
|
||||
|
||||
func (tps *TrackerPatterns) LoadDistinctCommonTrackerPatternIDsByIDs(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
scope Scoper,
|
||||
ids []gid.GID,
|
||||
) ([]gid.GID, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
q := `
|
||||
SELECT DISTINCT common_tracker_pattern_id
|
||||
FROM tracker_patterns
|
||||
WHERE
|
||||
%s
|
||||
AND id = ANY(@ids)
|
||||
AND common_tracker_pattern_id IS NOT NULL
|
||||
`
|
||||
|
||||
q = fmt.Sprintf(q, scope.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{"ids": ids}
|
||||
maps.Copy(args, scope.SQLArguments())
|
||||
|
||||
rows, err := conn.Query(ctx, q, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot query distinct common tracker pattern ids by pattern ids: %w", err)
|
||||
}
|
||||
|
||||
commonPatternIDs, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot collect distinct common tracker pattern ids by pattern ids: %w", err)
|
||||
}
|
||||
|
||||
return commonPatternIDs, nil
|
||||
}
|
||||
|
||||
func (tps *TrackerPatterns) UpdateLastMatchedAt(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
|
||||
Reference in New Issue
Block a user