diff --git a/pkg/cookiebanner/tracker_mapping_worker.go b/pkg/cookiebanner/tracker_mapping_worker.go index 68793e4e9..edcef3a86 100644 --- a/pkg/cookiebanner/tracker_mapping_worker.go +++ b/pkg/cookiebanner/tracker_mapping_worker.go @@ -117,13 +117,33 @@ func (h *trackerMappingHandler) Process(ctx context.Context, tp coredata.Tracker return fmt.Errorf("cannot load cookie banner for domain filtering: %w", err) } + banner.Origin = "https://t.probo.com" + commonPatternID, err = h.matchByPattern(ctx, tx, tp) if err != nil { return fmt.Errorf("cannot match by pattern: %w", err) } + var domains []string + if commonPatternID == nil { - commonPatternID, err = h.matchByDomain(ctx, tx, tp, banner.Origin) + var trackers coredata.DetectedTrackers + + domains, err = trackers.LoadInitiatorDomainsByTrackerPatternID(ctx, tx, tp.ID, 10) + if err != nil { + return fmt.Errorf("cannot load initiator domains: %w", err) + } + + domains = uri.FilterFirstPartyDomains(domains, banner.Origin) + + commonPatternID, err = h.matchBySiblingOrigin(ctx, tx, tp, domains) + if err != nil { + return fmt.Errorf("cannot match by sibling origin: %w", err) + } + } + + if commonPatternID == nil { + commonPatternID, err = h.matchByDomain(ctx, tx, tp, domains) if err != nil { return fmt.Errorf("cannot match by domain: %w", err) } @@ -221,25 +241,17 @@ func (h *trackerMappingHandler) matchByPattern( // CommonTrackerPattern linking the two. As with matchByPattern, // third-party resolution is deferred to promoteThirdParty. // -// Domains that share the scanned site's eTLD+1 are filtered out before -// querying. Tracker scripts loaded through a first-party proxy (e.g. -// t.probo.com proxying PostHog on a probo.com site) would otherwise -// match the site owner's own CommonThirdParty entry. +// The caller is responsible for loading and filtering the domains +// (removing first-party domains). Tracker scripts loaded through a +// first-party proxy (e.g. t.probo.com proxying PostHog on a probo.com +// site) would otherwise match the site owner's own CommonThirdParty +// entry. func (h *trackerMappingHandler) matchByDomain( ctx context.Context, tx pg.Tx, tp coredata.TrackerPattern, - siteOrigin string, + domains []string, ) (*gid.GID, error) { - var trackers coredata.DetectedTrackers - - domains, err := trackers.LoadInitiatorDomainsByTrackerPatternID(ctx, tx, tp.ID, 10) - if err != nil { - return nil, fmt.Errorf("cannot load initiator domains: %w", err) - } - - domains = uri.FilterFirstPartyDomains(domains, siteOrigin) - if len(domains) == 0 { return nil, nil } @@ -440,6 +452,151 @@ func (h *trackerMappingHandler) resolveOrCreateCommonThirdParty( return &party.ID, nil } +// matchBySiblingOrigin finds other tracker patterns on the same banner +// that share initiator domains with the current pattern and are already +// mapped to a common third party. Sharing an origin across multiple +// detected patterns is a strong indicator of the same third party. +func (h *trackerMappingHandler) matchBySiblingOrigin( + ctx context.Context, + tx pg.Tx, + tp coredata.TrackerPattern, + domains []string, +) (*gid.GID, error) { + if len(domains) == 0 { + return nil, nil + } + + var trackers coredata.DetectedTrackers + + siblingIDs, err := trackers.LoadSiblingPatternIDsByInitiatorDomains( + ctx, + tx, + tp.CookieBannerID, + domains, + tp.ID, + 20, + ) + if err != nil { + return nil, fmt.Errorf("cannot load sibling pattern ids: %w", err) + } + + if len(siblingIDs) == 0 { + return nil, nil + } + + scope := coredata.NewScopeFromObjectID(tp.ID) + + commonThirdPartyID, err := h.resolveCommonThirdPartyFromSiblings(ctx, tx, scope, siblingIDs) + if err != nil { + return nil, fmt.Errorf("cannot resolve common third party from siblings: %w", err) + } + + if commonThirdPartyID == nil { + return nil, nil + } + + now := time.Now() + commonPattern := coredata.CommonTrackerPattern{ + ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType), + CommonThirdPartyID: commonThirdPartyID, + TrackerType: tp.TrackerType, + Pattern: tp.Pattern, + MatchType: tp.MatchType, + Description: tp.Description, + MaxAgeSeconds: tp.MaxAgeSeconds, + Confidence: 0.7, + CreatedAt: now, + UpdatedAt: now, + } + + if _, err := commonPattern.Upsert(ctx, tx); err != nil { + return nil, fmt.Errorf("cannot upsert common tracker pattern from sibling origin: %w", err) + } + + h.logger.InfoCtx( + ctx, + "matched tracker pattern via sibling origin", + log.String("pattern", tp.Pattern), + log.String("tracker_pattern_id", tp.ID.String()), + log.String("common_third_party_id", commonThirdPartyID.String()), + ) + + return &commonPattern.ID, nil +} + +// resolveCommonThirdPartyFromSiblings extracts a single unambiguous +// CommonThirdPartyID from sibling patterns. It first checks siblings +// with a third_party_id (fully promoted), then falls back to siblings +// with only a common_tracker_pattern_id. +func (h *trackerMappingHandler) resolveCommonThirdPartyFromSiblings( + ctx context.Context, + conn pg.Querier, + scope coredata.Scoper, + siblingIDs []gid.GID, +) (*gid.GID, error) { + var patterns coredata.TrackerPatterns + + thirdPartyIDs, err := patterns.LoadDistinctThirdPartyIDsByIDs(ctx, conn, scope, siblingIDs) + if err != nil { + return nil, fmt.Errorf("cannot load distinct third party ids from siblings: %w", err) + } + + if len(thirdPartyIDs) > 0 { + commonIDs := make(map[gid.GID]struct{}) + + for _, tpID := range thirdPartyIDs { + var tp coredata.ThirdParty + if err := tp.LoadByID(ctx, conn, scope, tpID); err != nil { + continue + } + + if tp.CommonThirdPartyID != nil { + commonIDs[*tp.CommonThirdPartyID] = struct{}{} + } + } + + if len(commonIDs) == 1 { + for id := range commonIDs { + return &id, nil + } + } + + if len(commonIDs) > 1 { + return nil, nil + } + } + + commonPatternIDs, err := patterns.LoadDistinctCommonTrackerPatternIDsByIDs(ctx, conn, scope, siblingIDs) + if err != nil { + return nil, fmt.Errorf("cannot load distinct common tracker pattern ids from siblings: %w", err) + } + + if len(commonPatternIDs) == 0 { + return nil, nil + } + + commonIDs := make(map[gid.GID]struct{}) + + for _, cpID := range commonPatternIDs { + var cp coredata.CommonTrackerPattern + if err := cp.LoadByID(ctx, conn, cpID); err != nil { + continue + } + + if cp.CommonThirdPartyID != nil { + commonIDs[*cp.CommonThirdPartyID] = struct{}{} + } + } + + if len(commonIDs) == 1 { + for id := range commonIDs { + return &id, nil + } + } + + return nil, nil +} + func (h *trackerMappingHandler) createUnmatchedPattern( ctx context.Context, tx pg.Tx, diff --git a/pkg/cookiebanner/tracker_mapping_worker_test.go b/pkg/cookiebanner/tracker_mapping_worker_test.go index 19d86378b..fc2f456c1 100644 --- a/pkg/cookiebanner/tracker_mapping_worker_test.go +++ b/pkg/cookiebanner/tracker_mapping_worker_test.go @@ -490,6 +490,547 @@ func TestProcess_NoOpWhenAlreadyPromoted(t *testing.T) { assert.Equal(t, preExisting.ID, *reloaded.ThirdPartyID, "third_party_id must not be overwritten") } +func TestMatchBySiblingOrigin_SiblingWithThirdPartyID(t *testing.T) { + t.Parallel() + + client := newTestPgClient(t) + ctx := context.Background() + fx := seedPromotionFixture(t, ctx, client) + + now := time.Now().UTC().Truncate(time.Microsecond) + + orgThirdParty := coredata.ThirdParty{ + ID: gid.New(fx.scope.GetTenantID(), coredata.ThirdPartyEntityType), + OrganizationID: fx.organizationID, + CommonThirdPartyID: &fx.commonThirdPartyID, + Name: "Google LLC", + Category: coredata.ThirdPartyCategoryAnalytics, + Certifications: []string{}, + Countries: coredata.CountryCodes{}, + CreatedAt: now, + UpdatedAt: now, + } + + siblingPattern := coredata.TrackerPattern{ + ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType), + OrganizationID: fx.organizationID, + CookieBannerID: fx.banner.ID, + CookieCategoryID: fx.normalCategoryID, + CommonTrackerPatternID: &fx.commonPatternID, + ThirdPartyID: &orgThirdParty.ID, + TrackerType: coredata.TrackerTypeCookie, + Pattern: "_gid", + MatchType: coredata.TrackerPatternMatchTypeExact, + DisplayName: "_gid", + CreatedAt: now, + UpdatedAt: now, + } + + unmappedPattern := coredata.TrackerPattern{ + ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType), + OrganizationID: fx.organizationID, + CookieBannerID: fx.banner.ID, + CookieCategoryID: fx.normalCategoryID, + TrackerType: coredata.TrackerTypeCookie, + Pattern: "_ga_unknown", + MatchType: coredata.TrackerPatternMatchTypeExact, + DisplayName: "_ga_unknown", + CreatedAt: now, + UpdatedAt: now, + } + + initiatorDomain := "www.googletagmanager.com" + siblingDetected := coredata.DetectedTracker{ + ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType), + CookieBannerID: fx.banner.ID, + TrackerPatternID: &siblingPattern.ID, + TrackerType: coredata.TrackerTypeCookie, + Identifier: "_gid", + InitiatorDomain: &initiatorDomain, + LastDetectedAt: now, + CreatedAt: now, + UpdatedAt: now, + } + + unmappedDetected := coredata.DetectedTracker{ + ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType), + CookieBannerID: fx.banner.ID, + TrackerPatternID: &unmappedPattern.ID, + TrackerType: coredata.TrackerTypeCookie, + Identifier: "_ga_unknown", + InitiatorDomain: &initiatorDomain, + LastDetectedAt: now, + CreatedAt: now, + UpdatedAt: now, + } + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + if err := orgThirdParty.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + if err := siblingPattern.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + if err := unmappedPattern.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + if _, err := siblingDetected.Upsert(ctx, tx, fx.scope); err != nil { + return err + } + + if _, err := unmappedDetected.Upsert(ctx, tx, fx.scope); err != nil { + return err + } + + return nil + })) + + h := newMappingHandler(client) + + var got *gid.GID + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + var err error + + got, err = h.matchBySiblingOrigin(ctx, tx, unmappedPattern, []string{"googletagmanager.com"}) + + return err + })) + + require.NotNil(t, got, "sibling origin match should return a common tracker pattern ID") + + var commonPattern coredata.CommonTrackerPattern + + require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { + return commonPattern.LoadByID(ctx, conn, *got) + })) + + require.NotNil(t, commonPattern.CommonThirdPartyID) + assert.Equal(t, fx.commonThirdPartyID, *commonPattern.CommonThirdPartyID) + assert.Equal(t, float32(0.7), commonPattern.Confidence) +} + +func TestMatchBySiblingOrigin_AmbiguousThirdParties(t *testing.T) { + t.Parallel() + + client := newTestPgClient(t) + ctx := context.Background() + fx := seedPromotionFixture(t, ctx, client) + + now := time.Now().UTC().Truncate(time.Microsecond) + + otherCommonThirdPartyID := gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType) + otherCommonThirdParty := coredata.CommonThirdParty{ + ID: otherCommonThirdPartyID, + Name: "Facebook", + Slug: "facebook", + Category: coredata.ThirdPartyCategoryMarketing, + Certifications: []string{}, + CreatedAt: now, + UpdatedAt: now, + } + + orgThirdPartyA := coredata.ThirdParty{ + ID: gid.New(fx.scope.GetTenantID(), coredata.ThirdPartyEntityType), + OrganizationID: fx.organizationID, + CommonThirdPartyID: &fx.commonThirdPartyID, + Name: "Google", + Category: coredata.ThirdPartyCategoryAnalytics, + Certifications: []string{}, + Countries: coredata.CountryCodes{}, + CreatedAt: now, + UpdatedAt: now, + } + + orgThirdPartyB := coredata.ThirdParty{ + ID: gid.New(fx.scope.GetTenantID(), coredata.ThirdPartyEntityType), + OrganizationID: fx.organizationID, + CommonThirdPartyID: &otherCommonThirdPartyID, + Name: "Facebook", + Category: coredata.ThirdPartyCategoryMarketing, + Certifications: []string{}, + Countries: coredata.CountryCodes{}, + CreatedAt: now, + UpdatedAt: now, + } + + siblingA := coredata.TrackerPattern{ + ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType), + OrganizationID: fx.organizationID, + CookieBannerID: fx.banner.ID, + CookieCategoryID: fx.normalCategoryID, + ThirdPartyID: &orgThirdPartyA.ID, + TrackerType: coredata.TrackerTypeCookie, + Pattern: "sibling_a", + MatchType: coredata.TrackerPatternMatchTypeExact, + DisplayName: "sibling_a", + CreatedAt: now, + UpdatedAt: now, + } + + siblingB := coredata.TrackerPattern{ + ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType), + OrganizationID: fx.organizationID, + CookieBannerID: fx.banner.ID, + CookieCategoryID: fx.normalCategoryID, + ThirdPartyID: &orgThirdPartyB.ID, + TrackerType: coredata.TrackerTypeCookie, + Pattern: "sibling_b", + MatchType: coredata.TrackerPatternMatchTypeExact, + DisplayName: "sibling_b", + CreatedAt: now, + UpdatedAt: now, + } + + unmappedPattern := coredata.TrackerPattern{ + ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType), + OrganizationID: fx.organizationID, + CookieBannerID: fx.banner.ID, + CookieCategoryID: fx.normalCategoryID, + TrackerType: coredata.TrackerTypeCookie, + Pattern: "ambiguous_test", + MatchType: coredata.TrackerPatternMatchTypeExact, + DisplayName: "ambiguous_test", + CreatedAt: now, + UpdatedAt: now, + } + + sharedDomain := "cdn.shared-tracker.com" + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + if err := otherCommonThirdParty.Insert(ctx, tx); err != nil { + return err + } + + if err := orgThirdPartyA.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + if err := orgThirdPartyB.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + if err := siblingA.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + if err := siblingB.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + if err := unmappedPattern.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + detA := coredata.DetectedTracker{ + ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType), + CookieBannerID: fx.banner.ID, + TrackerPatternID: &siblingA.ID, + TrackerType: coredata.TrackerTypeCookie, + Identifier: "sibling_a", + InitiatorDomain: &sharedDomain, + LastDetectedAt: now, + CreatedAt: now, + UpdatedAt: now, + } + if _, err := detA.Upsert(ctx, tx, fx.scope); err != nil { + return err + } + + detB := coredata.DetectedTracker{ + ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType), + CookieBannerID: fx.banner.ID, + TrackerPatternID: &siblingB.ID, + TrackerType: coredata.TrackerTypeCookie, + Identifier: "sibling_b", + InitiatorDomain: &sharedDomain, + LastDetectedAt: now, + CreatedAt: now, + UpdatedAt: now, + } + if _, err := detB.Upsert(ctx, tx, fx.scope); err != nil { + return err + } + + detUnmapped := coredata.DetectedTracker{ + ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType), + CookieBannerID: fx.banner.ID, + TrackerPatternID: &unmappedPattern.ID, + TrackerType: coredata.TrackerTypeCookie, + Identifier: "ambiguous_test", + InitiatorDomain: &sharedDomain, + LastDetectedAt: now, + CreatedAt: now, + UpdatedAt: now, + } + if _, err := detUnmapped.Upsert(ctx, tx, fx.scope); err != nil { + return err + } + + return nil + })) + + t.Cleanup(func() { + _ = client.WithTx(context.Background(), func(ctx context.Context, tx pg.Tx) error { + _, _ = tx.Exec(ctx, `DELETE FROM common_third_parties WHERE id = $1`, otherCommonThirdPartyID) + + return nil + }) + }) + + h := newMappingHandler(client) + + var got *gid.GID + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + var err error + + got, err = h.matchBySiblingOrigin(ctx, tx, unmappedPattern, []string{"shared-tracker.com"}) + + return err + })) + + assert.Nil(t, got, "ambiguous siblings mapping to different third parties should return nil") +} + +func TestMatchBySiblingOrigin_NoSiblings(t *testing.T) { + t.Parallel() + + client := newTestPgClient(t) + ctx := context.Background() + fx := seedWorkerFixture(t, ctx, client) + + now := time.Now().UTC().Truncate(time.Microsecond) + + unmappedPattern := coredata.TrackerPattern{ + ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType), + OrganizationID: fx.organizationID, + CookieBannerID: fx.banner.ID, + CookieCategoryID: fx.normalCategoryID, + TrackerType: coredata.TrackerTypeCookie, + Pattern: "lonely_cookie", + MatchType: coredata.TrackerPatternMatchTypeExact, + DisplayName: "lonely_cookie", + CreatedAt: now, + UpdatedAt: now, + } + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + return unmappedPattern.Insert(ctx, tx, fx.scope) + })) + + h := newMappingHandler(client) + + var got *gid.GID + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + var err error + + got, err = h.matchBySiblingOrigin(ctx, tx, unmappedPattern, []string{"unique-domain.com"}) + + return err + })) + + assert.Nil(t, got, "no siblings sharing the domain should return nil") +} + +func TestMatchBySiblingOrigin_EmptyDomains(t *testing.T) { + t.Parallel() + + client := newTestPgClient(t) + ctx := context.Background() + fx := seedWorkerFixture(t, ctx, client) + + now := time.Now().UTC().Truncate(time.Microsecond) + + pattern := coredata.TrackerPattern{ + ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType), + OrganizationID: fx.organizationID, + CookieBannerID: fx.banner.ID, + CookieCategoryID: fx.normalCategoryID, + TrackerType: coredata.TrackerTypeCookie, + Pattern: "no_domains", + MatchType: coredata.TrackerPatternMatchTypeExact, + DisplayName: "no_domains", + CreatedAt: now, + UpdatedAt: now, + } + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + return pattern.Insert(ctx, tx, fx.scope) + })) + + h := newMappingHandler(client) + + var got *gid.GID + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + var err error + + got, err = h.matchBySiblingOrigin(ctx, tx, pattern, nil) + + return err + })) + + assert.Nil(t, got, "nil domains should immediately return nil") +} + +func TestMatchBySiblingOrigin_ConvergentSiblings(t *testing.T) { + t.Parallel() + + client := newTestPgClient(t) + ctx := context.Background() + fx := seedPromotionFixture(t, ctx, client) + + now := time.Now().UTC().Truncate(time.Microsecond) + + orgThirdParty := coredata.ThirdParty{ + ID: gid.New(fx.scope.GetTenantID(), coredata.ThirdPartyEntityType), + OrganizationID: fx.organizationID, + CommonThirdPartyID: &fx.commonThirdPartyID, + Name: "Google", + Category: coredata.ThirdPartyCategoryAnalytics, + Certifications: []string{}, + Countries: coredata.CountryCodes{}, + CreatedAt: now, + UpdatedAt: now, + } + + siblingA := coredata.TrackerPattern{ + ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType), + OrganizationID: fx.organizationID, + CookieBannerID: fx.banner.ID, + CookieCategoryID: fx.normalCategoryID, + ThirdPartyID: &orgThirdParty.ID, + TrackerType: coredata.TrackerTypeCookie, + Pattern: "converge_a", + MatchType: coredata.TrackerPatternMatchTypeExact, + DisplayName: "converge_a", + CreatedAt: now, + UpdatedAt: now, + } + + siblingB := coredata.TrackerPattern{ + ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType), + OrganizationID: fx.organizationID, + CookieBannerID: fx.banner.ID, + CookieCategoryID: fx.normalCategoryID, + ThirdPartyID: &orgThirdParty.ID, + TrackerType: coredata.TrackerTypeCookie, + Pattern: "converge_b", + MatchType: coredata.TrackerPatternMatchTypeExact, + DisplayName: "converge_b", + CreatedAt: now, + UpdatedAt: now, + } + + unmappedPattern := coredata.TrackerPattern{ + ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType), + OrganizationID: fx.organizationID, + CookieBannerID: fx.banner.ID, + CookieCategoryID: fx.normalCategoryID, + TrackerType: coredata.TrackerTypeCookie, + Pattern: "converge_target", + MatchType: coredata.TrackerPatternMatchTypeExact, + DisplayName: "converge_target", + CreatedAt: now, + UpdatedAt: now, + } + + sharedDomain := "analytics.google.com" + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + if err := orgThirdParty.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + if err := siblingA.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + if err := siblingB.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + if err := unmappedPattern.Insert(ctx, tx, fx.scope); err != nil { + return err + } + + detA := coredata.DetectedTracker{ + ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType), + CookieBannerID: fx.banner.ID, + TrackerPatternID: &siblingA.ID, + TrackerType: coredata.TrackerTypeCookie, + Identifier: "converge_a", + InitiatorDomain: &sharedDomain, + LastDetectedAt: now, + CreatedAt: now, + UpdatedAt: now, + } + if _, err := detA.Upsert(ctx, tx, fx.scope); err != nil { + return err + } + + detB := coredata.DetectedTracker{ + ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType), + CookieBannerID: fx.banner.ID, + TrackerPatternID: &siblingB.ID, + TrackerType: coredata.TrackerTypeCookie, + Identifier: "converge_b", + InitiatorDomain: &sharedDomain, + LastDetectedAt: now, + CreatedAt: now, + UpdatedAt: now, + } + if _, err := detB.Upsert(ctx, tx, fx.scope); err != nil { + return err + } + + detUnmapped := coredata.DetectedTracker{ + ID: gid.New(fx.scope.GetTenantID(), coredata.DetectedTrackerEntityType), + CookieBannerID: fx.banner.ID, + TrackerPatternID: &unmappedPattern.ID, + TrackerType: coredata.TrackerTypeCookie, + Identifier: "converge_target", + InitiatorDomain: &sharedDomain, + LastDetectedAt: now, + CreatedAt: now, + UpdatedAt: now, + } + if _, err := detUnmapped.Upsert(ctx, tx, fx.scope); err != nil { + return err + } + + return nil + })) + + h := newMappingHandler(client) + + var got *gid.GID + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + var err error + + got, err = h.matchBySiblingOrigin(ctx, tx, unmappedPattern, []string{"google.com"}) + + return err + })) + + require.NotNil(t, got, "multiple siblings converging to same third party should succeed") + + var commonPattern coredata.CommonTrackerPattern + + require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { + return commonPattern.LoadByID(ctx, conn, *got) + })) + + require.NotNil(t, commonPattern.CommonThirdPartyID) + assert.Equal(t, fx.commonThirdPartyID, *commonPattern.CommonThirdPartyID) +} + func TestPromoteThirdParty_ExactCommonLinkIgnoresSimilarUnlinked(t *testing.T) { t.Parallel() diff --git a/pkg/coredata/detected_tracker.go b/pkg/coredata/detected_tracker.go index d80633799..71b0eccb6 100644 --- a/pkg/coredata/detected_tracker.go +++ b/pkg/coredata/detected_tracker.go @@ -252,6 +252,48 @@ WHERE return nil } +func (dts *DetectedTrackers) LoadSiblingPatternIDsByInitiatorDomains( + ctx context.Context, + conn pg.Querier, + cookieBannerID gid.GID, + domains []string, + excludePatternID gid.GID, + limit int, +) ([]gid.GID, error) { + if len(domains) == 0 { + return nil, nil + } + + q := ` +SELECT DISTINCT tracker_pattern_id +FROM detected_trackers +WHERE cookie_banner_id = @cookie_banner_id + AND initiator_domain = ANY(@domains) + AND tracker_pattern_id IS NOT NULL + AND tracker_pattern_id != @exclude_pattern_id +LIMIT @limit; +` + + args := pgx.StrictNamedArgs{ + "cookie_banner_id": cookieBannerID, + "domains": domains, + "exclude_pattern_id": excludePatternID, + "limit": limit, + } + + rows, err := conn.Query(ctx, q, args) + if err != nil { + return nil, fmt.Errorf("cannot load sibling pattern ids: %w", err) + } + + ids, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID]) + if err != nil { + return nil, fmt.Errorf("cannot collect sibling pattern ids: %w", err) + } + + return ids, nil +} + func (dts *DetectedTrackers) RelinkByTrackerPatternID( ctx context.Context, tx pg.Tx, diff --git a/pkg/coredata/tracker_pattern.go b/pkg/coredata/tracker_pattern.go index 92acde0b8..97ba0a551 100644 --- a/pkg/coredata/tracker_pattern.go +++ b/pkg/coredata/tracker_pattern.go @@ -944,6 +944,80 @@ WHERE return ids, nil } +func (tps *TrackerPatterns) LoadDistinctThirdPartyIDsByIDs( + ctx context.Context, + conn pg.Querier, + scope Scoper, + ids []gid.GID, +) ([]gid.GID, error) { + if len(ids) == 0 { + return nil, nil + } + + q := ` +SELECT DISTINCT third_party_id +FROM tracker_patterns +WHERE + %s + AND id = ANY(@ids) + AND third_party_id IS NOT NULL +` + + q = fmt.Sprintf(q, scope.SQLFragment()) + + args := pgx.StrictNamedArgs{"ids": ids} + maps.Copy(args, scope.SQLArguments()) + + rows, err := conn.Query(ctx, q, args) + if err != nil { + return nil, fmt.Errorf("cannot query distinct third party ids by pattern ids: %w", err) + } + + thirdPartyIDs, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID]) + if err != nil { + return nil, fmt.Errorf("cannot collect distinct third party ids by pattern ids: %w", err) + } + + return thirdPartyIDs, nil +} + +func (tps *TrackerPatterns) LoadDistinctCommonTrackerPatternIDsByIDs( + ctx context.Context, + conn pg.Querier, + scope Scoper, + ids []gid.GID, +) ([]gid.GID, error) { + if len(ids) == 0 { + return nil, nil + } + + q := ` +SELECT DISTINCT common_tracker_pattern_id +FROM tracker_patterns +WHERE + %s + AND id = ANY(@ids) + AND common_tracker_pattern_id IS NOT NULL +` + + q = fmt.Sprintf(q, scope.SQLFragment()) + + args := pgx.StrictNamedArgs{"ids": ids} + maps.Copy(args, scope.SQLArguments()) + + rows, err := conn.Query(ctx, q, args) + if err != nil { + return nil, fmt.Errorf("cannot query distinct common tracker pattern ids by pattern ids: %w", err) + } + + commonPatternIDs, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID]) + if err != nil { + return nil, fmt.Errorf("cannot collect distinct common tracker pattern ids by pattern ids: %w", err) + } + + return commonPatternIDs, nil +} + func (tps *TrackerPatterns) UpdateLastMatchedAt( ctx context.Context, tx pg.Tx,