Add coredata batch loaders for tracker third parties
Surface the third-party links carried by tracker patterns (org-scoped
ThirdParty via third_party_id, or global CommonThirdParty via
common_tracker_pattern_id) requires three new batch loaders and two
filter dimensions, all kept inside their owning entity tables to honour
the no-cross-entity-JOIN rule.
* CommonTrackerPatterns gains LoadByIDs and the ID-only
LoadIDsByCommonThirdPartyID helper, which lets callers translate a
common third party into a set of common_tracker_pattern_id values
without ever JOINing against tracker_patterns.
* CommonThirdParties gains LoadByIDs.
* TrackerPatterns gains LoadDistinctThirdPartyIDsByCookieBannerID and
LoadDistinctCommonTrackerPatternIDsByCookieBannerID, used by the
upcoming CookieBanner.linkedThirdParties resolver to enumerate the
third parties referenced in a banner.
* TrackerPatternFilter gains thirdPartyID and commonTrackerPatternIDs
filter dimensions; the GraphQL layer will dispatch a single
thirdPartyId argument to the right one based on the GID entity-type
prefix.
Service-layer wrappers (cookiebanner.GetCommonTrackerPatternsByIDs,
cookiebanner.LoadCommonTrackerPatternIDsByCommonThirdPartyID,
cookiebanner.LoadDistinctThirdPartyIDsByCookieBannerID,
cookiebanner.LoadDistinctCommonTrackerPatternIDsByCookieBannerID, and
thirdparty.GetCommonThirdPartiesByIDs) expose the new loaders to the
console resolvers and dataloaders that follow.
Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -2709,6 +2709,128 @@ func (s *Service) CountTrackerPatternsForBanner(
|
|||||||
return count, nil
|
return count, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Service) GetCommonTrackerPatternsByIDs(
|
||||||
|
ctx context.Context,
|
||||||
|
ids ...gid.GID,
|
||||||
|
) (coredata.CommonTrackerPatterns, error) {
|
||||||
|
var patterns coredata.CommonTrackerPatterns
|
||||||
|
|
||||||
|
err := s.pg.WithConn(
|
||||||
|
ctx,
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
if err := patterns.LoadByIDs(ctx, conn, ids); err != nil {
|
||||||
|
return fmt.Errorf("cannot load common tracker patterns by ids: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return patterns, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadCommonTrackerPatternIDsByCommonThirdPartyID returns the IDs of
|
||||||
|
// every common tracker pattern referencing the given common third party.
|
||||||
|
// Used by the trackers list filter to translate a CommonThirdParty GID
|
||||||
|
// into a `common_tracker_pattern_id = ANY(...)` constraint without
|
||||||
|
// JOINing across entity tables in coredata.
|
||||||
|
func (s *Service) LoadCommonTrackerPatternIDsByCommonThirdPartyID(
|
||||||
|
ctx context.Context,
|
||||||
|
commonThirdPartyID gid.GID,
|
||||||
|
) ([]gid.GID, error) {
|
||||||
|
var ids []gid.GID
|
||||||
|
|
||||||
|
err := s.pg.WithConn(
|
||||||
|
ctx,
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
var (
|
||||||
|
patterns coredata.CommonTrackerPatterns
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
|
||||||
|
ids, err = patterns.LoadIDsByCommonThirdPartyID(ctx, conn, commonThirdPartyID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot load common tracker pattern ids: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ids, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadDistinctThirdPartyIDsByCookieBannerID returns the distinct
|
||||||
|
// org-scoped third-party IDs referenced by tracker patterns of the
|
||||||
|
// banner. The companion
|
||||||
|
// LoadDistinctCommonTrackerPatternIDsByCookieBannerID covers the
|
||||||
|
// indirect mapping through common_tracker_patterns.
|
||||||
|
func (s *Service) LoadDistinctThirdPartyIDsByCookieBannerID(
|
||||||
|
ctx context.Context,
|
||||||
|
scope coredata.Scoper,
|
||||||
|
cookieBannerID gid.GID,
|
||||||
|
) ([]gid.GID, error) {
|
||||||
|
var ids []gid.GID
|
||||||
|
|
||||||
|
err := s.pg.WithConn(
|
||||||
|
ctx,
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
var (
|
||||||
|
patterns coredata.TrackerPatterns
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
|
||||||
|
ids, err = patterns.LoadDistinctThirdPartyIDsByCookieBannerID(ctx, conn, scope, cookieBannerID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot load distinct third party ids: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ids, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) LoadDistinctCommonTrackerPatternIDsByCookieBannerID(
|
||||||
|
ctx context.Context,
|
||||||
|
scope coredata.Scoper,
|
||||||
|
cookieBannerID gid.GID,
|
||||||
|
) ([]gid.GID, error) {
|
||||||
|
var ids []gid.GID
|
||||||
|
|
||||||
|
err := s.pg.WithConn(
|
||||||
|
ctx,
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
var (
|
||||||
|
patterns coredata.TrackerPatterns
|
||||||
|
err error
|
||||||
|
)
|
||||||
|
|
||||||
|
ids, err = patterns.LoadDistinctCommonTrackerPatternIDsByCookieBannerID(ctx, conn, scope, cookieBannerID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot load distinct common tracker pattern ids: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return ids, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Service) CountDetectedTrackersByPatternID(
|
func (s *Service) CountDetectedTrackersByPatternID(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
scope coredata.Scoper,
|
scope coredata.Scoper,
|
||||||
|
|||||||
@@ -507,6 +507,57 @@ func (t CommonThirdParty) Delete(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (t *CommonThirdParties) LoadByIDs(
|
||||||
|
ctx context.Context,
|
||||||
|
conn pg.Querier,
|
||||||
|
ids []gid.GID,
|
||||||
|
) error {
|
||||||
|
q := `
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
name,
|
||||||
|
slug,
|
||||||
|
category,
|
||||||
|
headquarter_address,
|
||||||
|
legal_name,
|
||||||
|
website_url,
|
||||||
|
privacy_policy_url,
|
||||||
|
service_level_agreement_url,
|
||||||
|
service_software_agreement_url,
|
||||||
|
data_processing_agreement_url,
|
||||||
|
business_associate_agreement_url,
|
||||||
|
subprocessors_list_url,
|
||||||
|
certifications,
|
||||||
|
status_page_url,
|
||||||
|
terms_of_service_url,
|
||||||
|
security_page_url,
|
||||||
|
trust_page_url,
|
||||||
|
logo_file_id,
|
||||||
|
created_at,
|
||||||
|
updated_at
|
||||||
|
FROM
|
||||||
|
common_third_parties
|
||||||
|
WHERE
|
||||||
|
id = ANY(@ids)
|
||||||
|
`
|
||||||
|
|
||||||
|
args := pgx.StrictNamedArgs{"ids": ids}
|
||||||
|
|
||||||
|
rows, err := conn.Query(ctx, q, args)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot query common third parties: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
parties, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[CommonThirdParty])
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot collect common third parties: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
*t = parties
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (t *CommonThirdParties) LoadAll(
|
func (t *CommonThirdParties) LoadAll(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
conn pg.Querier,
|
conn pg.Querier,
|
||||||
|
|||||||
@@ -443,3 +443,76 @@ ORDER BY pattern ASC;
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (ps *CommonTrackerPatterns) LoadByIDs(
|
||||||
|
ctx context.Context,
|
||||||
|
conn pg.Querier,
|
||||||
|
ids []gid.GID,
|
||||||
|
) error {
|
||||||
|
q := `
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
common_third_party_id,
|
||||||
|
tracker_type,
|
||||||
|
pattern,
|
||||||
|
match_type,
|
||||||
|
description,
|
||||||
|
max_age_seconds,
|
||||||
|
confidence,
|
||||||
|
created_at,
|
||||||
|
updated_at
|
||||||
|
FROM
|
||||||
|
common_tracker_patterns
|
||||||
|
WHERE
|
||||||
|
id = ANY(@ids)
|
||||||
|
`
|
||||||
|
|
||||||
|
args := pgx.StrictNamedArgs{"ids": ids}
|
||||||
|
|
||||||
|
rows, err := conn.Query(ctx, q, args)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot query common tracker patterns: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
patterns, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[CommonTrackerPattern])
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot collect common tracker patterns: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
*ps = patterns
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadIDsByCommonThirdPartyID returns just the IDs of the common tracker
|
||||||
|
// patterns linked to the given common third party. Callers use it to feed
|
||||||
|
// a `common_tracker_pattern_id = ANY(...)` filter on tracker_patterns
|
||||||
|
// without crossing the entity boundary.
|
||||||
|
func (ps *CommonTrackerPatterns) LoadIDsByCommonThirdPartyID(
|
||||||
|
ctx context.Context,
|
||||||
|
conn pg.Querier,
|
||||||
|
commonThirdPartyID gid.GID,
|
||||||
|
) ([]gid.GID, error) {
|
||||||
|
q := `
|
||||||
|
SELECT
|
||||||
|
id
|
||||||
|
FROM
|
||||||
|
common_tracker_patterns
|
||||||
|
WHERE
|
||||||
|
common_third_party_id = @common_third_party_id
|
||||||
|
`
|
||||||
|
|
||||||
|
args := pgx.StrictNamedArgs{"common_third_party_id": commonThirdPartyID}
|
||||||
|
|
||||||
|
rows, err := conn.Query(ctx, q, args)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot query common tracker pattern ids: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ids, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot collect common tracker pattern ids: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ids, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -864,6 +864,82 @@ WHERE
|
|||||||
return count, nil
|
return count, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LoadDistinctThirdPartyIDsByCookieBannerID returns the distinct non-null
|
||||||
|
// `third_party_id` values referenced by tracker patterns of the given
|
||||||
|
// banner. Callers feed it to ThirdParty.GetByIDs to power per-banner
|
||||||
|
// pickers without crossing the entity boundary.
|
||||||
|
func (tps *TrackerPatterns) LoadDistinctThirdPartyIDsByCookieBannerID(
|
||||||
|
ctx context.Context,
|
||||||
|
conn pg.Querier,
|
||||||
|
scope Scoper,
|
||||||
|
cookieBannerID gid.GID,
|
||||||
|
) ([]gid.GID, error) {
|
||||||
|
q := `
|
||||||
|
SELECT DISTINCT third_party_id
|
||||||
|
FROM tracker_patterns
|
||||||
|
WHERE
|
||||||
|
%s
|
||||||
|
AND cookie_banner_id = @cookie_banner_id
|
||||||
|
AND third_party_id IS NOT NULL
|
||||||
|
`
|
||||||
|
|
||||||
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
||||||
|
|
||||||
|
args := pgx.StrictNamedArgs{"cookie_banner_id": cookieBannerID}
|
||||||
|
maps.Copy(args, scope.SQLArguments())
|
||||||
|
|
||||||
|
rows, err := conn.Query(ctx, q, args)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot query distinct third party ids: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ids, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot collect distinct third party ids: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ids, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadDistinctCommonTrackerPatternIDsByCookieBannerID returns the
|
||||||
|
// distinct non-null `common_tracker_pattern_id` values referenced by
|
||||||
|
// tracker patterns of the given banner. Callers chain this with
|
||||||
|
// CommonTrackerPatterns.LoadByIDs and CommonThirdParties.LoadByIDs to
|
||||||
|
// resolve the linked common third parties without JOINs.
|
||||||
|
func (tps *TrackerPatterns) LoadDistinctCommonTrackerPatternIDsByCookieBannerID(
|
||||||
|
ctx context.Context,
|
||||||
|
conn pg.Querier,
|
||||||
|
scope Scoper,
|
||||||
|
cookieBannerID gid.GID,
|
||||||
|
) ([]gid.GID, error) {
|
||||||
|
q := `
|
||||||
|
SELECT DISTINCT common_tracker_pattern_id
|
||||||
|
FROM tracker_patterns
|
||||||
|
WHERE
|
||||||
|
%s
|
||||||
|
AND cookie_banner_id = @cookie_banner_id
|
||||||
|
AND common_tracker_pattern_id IS NOT NULL
|
||||||
|
AND third_party_id IS NULL
|
||||||
|
`
|
||||||
|
|
||||||
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
||||||
|
|
||||||
|
args := pgx.StrictNamedArgs{"cookie_banner_id": cookieBannerID}
|
||||||
|
maps.Copy(args, scope.SQLArguments())
|
||||||
|
|
||||||
|
rows, err := conn.Query(ctx, q, args)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot query distinct common tracker pattern ids: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ids, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot collect distinct common tracker pattern ids: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ids, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (tps *TrackerPatterns) UpdateLastMatchedAt(
|
func (tps *TrackerPatterns) UpdateLastMatchedAt(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
tx pg.Tx,
|
tx pg.Tx,
|
||||||
|
|||||||
@@ -26,6 +26,8 @@ type TrackerPatternFilter struct {
|
|||||||
query *string
|
query *string
|
||||||
source *CookieSource
|
source *CookieSource
|
||||||
trackerType *TrackerType
|
trackerType *TrackerType
|
||||||
|
thirdPartyID *gid.GID
|
||||||
|
commonTrackerPatternIDs []gid.GID
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewTrackerPatternFilter(
|
func NewTrackerPatternFilter(
|
||||||
@@ -55,6 +57,22 @@ func (f *TrackerPatternFilter) WithTrackerType(trackerType *TrackerType) *Tracke
|
|||||||
return f
|
return f
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f *TrackerPatternFilter) WithThirdPartyID(thirdPartyID *gid.GID) *TrackerPatternFilter {
|
||||||
|
f.thirdPartyID = thirdPartyID
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithCommonTrackerPatternIDs constrains the result to tracker patterns
|
||||||
|
// whose `common_tracker_pattern_id` is in the given set. Callers
|
||||||
|
// pre-resolve this list (typically via
|
||||||
|
// CommonTrackerPatterns.LoadIDsByCommonThirdPartyID) so the filter stays
|
||||||
|
// inside the tracker_patterns table. Passing an empty (non-nil) slice
|
||||||
|
// yields no rows.
|
||||||
|
func (f *TrackerPatternFilter) WithCommonTrackerPatternIDs(ids []gid.GID) *TrackerPatternFilter {
|
||||||
|
f.commonTrackerPatternIDs = ids
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
func (f *TrackerPatternFilter) SQLFragment() string {
|
func (f *TrackerPatternFilter) SQLFragment() string {
|
||||||
if f == nil {
|
if f == nil {
|
||||||
return "TRUE"
|
return "TRUE"
|
||||||
@@ -103,6 +121,20 @@ func (f *TrackerPatternFilter) SQLFragment() string {
|
|||||||
tracker_type = @filter_tracker_type::tracker_type
|
tracker_type = @filter_tracker_type::tracker_type
|
||||||
ELSE TRUE
|
ELSE TRUE
|
||||||
END
|
END
|
||||||
|
AND
|
||||||
|
CASE
|
||||||
|
WHEN @has_third_party_id_filter::boolean = false THEN TRUE
|
||||||
|
WHEN @has_third_party_id_filter::boolean = true THEN
|
||||||
|
third_party_id = @filter_third_party_id::text
|
||||||
|
ELSE TRUE
|
||||||
|
END
|
||||||
|
AND
|
||||||
|
CASE
|
||||||
|
WHEN @has_common_tracker_pattern_ids_filter::boolean = false THEN TRUE
|
||||||
|
WHEN @has_common_tracker_pattern_ids_filter::boolean = true THEN
|
||||||
|
common_tracker_pattern_id = ANY(@filter_common_tracker_pattern_ids::text[])
|
||||||
|
ELSE TRUE
|
||||||
|
END
|
||||||
)`
|
)`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -123,6 +155,10 @@ func (f *TrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs {
|
|||||||
"filter_source": nil,
|
"filter_source": nil,
|
||||||
"has_tracker_type_filter": false,
|
"has_tracker_type_filter": false,
|
||||||
"filter_tracker_type": nil,
|
"filter_tracker_type": nil,
|
||||||
|
"has_third_party_id_filter": false,
|
||||||
|
"filter_third_party_id": nil,
|
||||||
|
"has_common_tracker_pattern_ids_filter": false,
|
||||||
|
"filter_common_tracker_pattern_ids": []gid.GID{},
|
||||||
}
|
}
|
||||||
|
|
||||||
if f.matchType != nil {
|
if f.matchType != nil {
|
||||||
@@ -154,5 +190,15 @@ func (f *TrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs {
|
|||||||
args["filter_tracker_type"] = string(*f.trackerType)
|
args["filter_tracker_type"] = string(*f.trackerType)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if f.thirdPartyID != nil {
|
||||||
|
args["has_third_party_id_filter"] = true
|
||||||
|
args["filter_third_party_id"] = *f.thirdPartyID
|
||||||
|
}
|
||||||
|
|
||||||
|
if f.commonTrackerPatternIDs != nil {
|
||||||
|
args["has_common_tracker_pattern_ids_filter"] = true
|
||||||
|
args["filter_common_tracker_pattern_ids"] = f.commonTrackerPatternIDs
|
||||||
|
}
|
||||||
|
|
||||||
return args
|
return args
|
||||||
}
|
}
|
||||||
|
|||||||
23
pkg/thirdparty/service.go
vendored
23
pkg/thirdparty/service.go
vendored
@@ -50,6 +50,29 @@ func (s *Service) GenerateLogoURL(
|
|||||||
return &url, nil
|
return &url, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Service) GetCommonThirdPartiesByIDs(
|
||||||
|
ctx context.Context,
|
||||||
|
ids ...gid.GID,
|
||||||
|
) (coredata.CommonThirdParties, error) {
|
||||||
|
var parties coredata.CommonThirdParties
|
||||||
|
|
||||||
|
err := s.pg.WithConn(
|
||||||
|
ctx,
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
if err := parties.LoadByIDs(ctx, conn, ids); err != nil {
|
||||||
|
return fmt.Errorf("cannot load common third parties by ids: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return parties, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Service) Search(ctx context.Context, name string) ([]*coredata.CommonThirdParty, error) {
|
func (s *Service) Search(ctx context.Context, name string) ([]*coredata.CommonThirdParty, error) {
|
||||||
var parties coredata.CommonThirdParties
|
var parties coredata.CommonThirdParties
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user