When the common-third-party enrichment worker discovers owned domains for a vendor, tracker patterns that were detected and left unmatched before those domains were known had nothing to re-trigger them. Add RequestMappingForUnmappedByInitiatorDomains, a global re-arm that stamps mapping_requested_at on still-unmapped patterns whose detected trackers share one of the new domains, and call it from the enrichment worker's persist step for newly-inserted domains only. The mapping worker then re-resolves them through its existing domain-overlap path. Targeting is limited to patterns with no resolved vendor (no org third party and an absent or unlinked catalog row), so a pattern already attributed to this or any other vendor is never disturbed. Signed-off-by: Émile Ré <emile@probo.com>
1665 lines
42 KiB
Go
1665 lines
42 KiB
Go
// Copyright (c) 2026 Probo Inc <hello@probo.com>.
|
|
//
|
|
// Permission to use, copy, modify, and/or distribute this software for any
|
|
// purpose with or without fee is hereby granted, provided that the above
|
|
// copyright notice and this permission notice appear in all copies.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
|
// PERFORMANCE OF THIS SOFTWARE.
|
|
|
|
package coredata
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"maps"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgconn"
|
|
"go.gearno.de/kit/pg"
|
|
"go.probo.inc/probo/pkg/gid"
|
|
"go.probo.inc/probo/pkg/iam/policy"
|
|
"go.probo.inc/probo/pkg/page"
|
|
)
|
|
|
|
type (
|
|
TrackerPattern struct {
|
|
ID gid.GID `db:"id"`
|
|
OrganizationID gid.GID `db:"organization_id"`
|
|
CookieBannerID gid.GID `db:"cookie_banner_id"`
|
|
CookieCategoryID gid.GID `db:"cookie_category_id"`
|
|
CommonTrackerPatternID *gid.GID `db:"common_tracker_pattern_id"`
|
|
ThirdPartyID *gid.GID `db:"third_party_id"`
|
|
TrackerType TrackerType `db:"tracker_type"`
|
|
Pattern string `db:"pattern"`
|
|
MatchType TrackerPatternMatchType `db:"match_type"`
|
|
DisplayName string `db:"display_name"`
|
|
Description string `db:"description"`
|
|
Excluded bool `db:"excluded"`
|
|
MaxAgeSeconds *int `db:"max_age_seconds"`
|
|
Source *CookieSource `db:"source"`
|
|
LastMatchedAt *time.Time `db:"last_matched_at"`
|
|
MappingRequestedAt *time.Time `db:"mapping_requested_at"`
|
|
CreatedAt time.Time `db:"created_at"`
|
|
UpdatedAt time.Time `db:"updated_at"`
|
|
}
|
|
|
|
TrackerPatterns []*TrackerPattern
|
|
)
|
|
|
|
func (tp *TrackerPattern) CursorKey(field TrackerPatternOrderField) page.CursorKey {
|
|
switch field {
|
|
case TrackerPatternOrderFieldCreatedAt:
|
|
return page.NewCursorKey(tp.ID, tp.CreatedAt)
|
|
case TrackerPatternOrderFieldName:
|
|
return page.NewCursorKey(tp.ID, tp.DisplayName)
|
|
case TrackerPatternOrderFieldLastMatchedAt:
|
|
if tp.LastMatchedAt == nil {
|
|
return page.NewCursorKey(tp.ID, time.Time{})
|
|
}
|
|
|
|
return page.NewCursorKey(tp.ID, *tp.LastMatchedAt)
|
|
case TrackerPatternOrderFieldUpdatedAt:
|
|
return page.NewCursorKey(tp.ID, tp.UpdatedAt)
|
|
case TrackerPatternOrderFieldSource:
|
|
if tp.Source == nil {
|
|
return page.NewCursorKey(tp.ID, "")
|
|
}
|
|
|
|
return page.NewCursorKey(tp.ID, string(*tp.Source))
|
|
}
|
|
|
|
panic(fmt.Sprintf("unsupported order by: %s", field))
|
|
}
|
|
|
|
func (tp *TrackerPattern) AuthorizationAttributes(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
resourceIDs []gid.GID,
|
|
) (policy.AttributesByID, error) {
|
|
q := `SELECT id, organization_id FROM tracker_patterns WHERE id = ANY(@resource_ids::text[])`
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"resource_ids": resourceIDs,
|
|
}
|
|
|
|
rows, err := conn.Query(ctx, q, args)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot query authorization attributes: %w", err)
|
|
}
|
|
|
|
defer rows.Close()
|
|
|
|
attrsByID := make(policy.AttributesByID)
|
|
|
|
for rows.Next() {
|
|
var id, organizationID gid.GID
|
|
|
|
if err := rows.Scan(&id, &organizationID); err != nil {
|
|
return nil, fmt.Errorf("cannot scan authorization attributes: %w", err)
|
|
}
|
|
|
|
attrsByID[id] = policy.Attributes{
|
|
"organization_id": organizationID.String(),
|
|
}
|
|
}
|
|
|
|
if err := rows.Err(); err != nil {
|
|
return nil, fmt.Errorf("cannot iterate authorization attributes: %w", err)
|
|
}
|
|
|
|
return attrsByID, nil
|
|
}
|
|
|
|
func (tp *TrackerPattern) LoadByID(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
scope Scoper,
|
|
trackerPatternID gid.GID,
|
|
) error {
|
|
q := `
|
|
SELECT
|
|
id,
|
|
organization_id,
|
|
cookie_banner_id,
|
|
cookie_category_id,
|
|
common_tracker_pattern_id,
|
|
third_party_id,
|
|
tracker_type,
|
|
pattern,
|
|
match_type,
|
|
display_name,
|
|
description,
|
|
excluded,
|
|
max_age_seconds,
|
|
source,
|
|
last_matched_at,
|
|
mapping_requested_at,
|
|
created_at,
|
|
updated_at
|
|
FROM
|
|
tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND id = @tracker_pattern_id
|
|
LIMIT 1;
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{"tracker_pattern_id": trackerPatternID}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
rows, err := conn.Query(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot query tracker patterns: %w", err)
|
|
}
|
|
|
|
pattern, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[TrackerPattern])
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return ErrResourceNotFound
|
|
}
|
|
|
|
return fmt.Errorf("cannot collect tracker pattern: %w", err)
|
|
}
|
|
|
|
*tp = pattern
|
|
|
|
return nil
|
|
}
|
|
|
|
func (tp *TrackerPattern) LoadByBannerIDTypeAndPattern(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
scope Scoper,
|
|
cookieBannerID gid.GID,
|
|
trackerType TrackerType,
|
|
pattern string,
|
|
maxAgeSeconds *int,
|
|
) error {
|
|
q := `
|
|
SELECT
|
|
id,
|
|
organization_id,
|
|
cookie_banner_id,
|
|
cookie_category_id,
|
|
common_tracker_pattern_id,
|
|
third_party_id,
|
|
tracker_type,
|
|
pattern,
|
|
match_type,
|
|
display_name,
|
|
description,
|
|
excluded,
|
|
max_age_seconds,
|
|
source,
|
|
last_matched_at,
|
|
mapping_requested_at,
|
|
created_at,
|
|
updated_at
|
|
FROM
|
|
tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND cookie_banner_id = @cookie_banner_id
|
|
AND tracker_type = @tracker_type
|
|
AND pattern = @pattern
|
|
AND COALESCE(max_age_seconds, -1) = COALESCE(@max_age_seconds, -1)
|
|
LIMIT 1;
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"cookie_banner_id": cookieBannerID,
|
|
"tracker_type": trackerType,
|
|
"pattern": pattern,
|
|
"max_age_seconds": maxAgeSeconds,
|
|
}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
rows, err := conn.Query(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot query tracker patterns: %w", err)
|
|
}
|
|
|
|
p, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[TrackerPattern])
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return ErrResourceNotFound
|
|
}
|
|
|
|
return fmt.Errorf("cannot collect tracker pattern: %w", err)
|
|
}
|
|
|
|
*tp = p
|
|
|
|
return nil
|
|
}
|
|
|
|
func (tp *TrackerPattern) FindMatchingPattern(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
scope Scoper,
|
|
cookieBannerID gid.GID,
|
|
trackerType TrackerType,
|
|
identifier string,
|
|
) error {
|
|
q := `
|
|
SELECT
|
|
id,
|
|
organization_id,
|
|
cookie_banner_id,
|
|
cookie_category_id,
|
|
common_tracker_pattern_id,
|
|
third_party_id,
|
|
tracker_type,
|
|
pattern,
|
|
match_type,
|
|
display_name,
|
|
description,
|
|
excluded,
|
|
max_age_seconds,
|
|
source,
|
|
last_matched_at,
|
|
mapping_requested_at,
|
|
created_at,
|
|
updated_at
|
|
FROM
|
|
tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND cookie_banner_id = @cookie_banner_id
|
|
AND tracker_type = @tracker_type
|
|
AND (
|
|
(match_type = @match_type_glob
|
|
AND @identifier LIKE
|
|
replace(replace(replace(replace(
|
|
pattern, E'\\', E'\\\\'), '%', E'\\%'), '_', E'\\_'), '*', '%')
|
|
ESCAPE E'\\')
|
|
OR (match_type = @match_type_exact AND pattern = @identifier)
|
|
)
|
|
ORDER BY
|
|
CASE WHEN match_type = @match_type_exact AND pattern = @identifier THEN 0
|
|
ELSE 1
|
|
END,
|
|
length(replace(pattern, '*', '')) DESC
|
|
LIMIT 1;
|
|
`
|
|
|
|
q = strings.Replace(q, "%s", scope.SQLFragment(), 1)
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"cookie_banner_id": cookieBannerID,
|
|
"tracker_type": trackerType,
|
|
"identifier": identifier,
|
|
"match_type_glob": TrackerPatternMatchTypeGlob,
|
|
"match_type_exact": TrackerPatternMatchTypeExact,
|
|
}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
rows, err := conn.Query(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot query tracker patterns: %w", err)
|
|
}
|
|
|
|
pattern, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[TrackerPattern])
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return ErrResourceNotFound
|
|
}
|
|
|
|
return fmt.Errorf("cannot collect tracker pattern: %w", err)
|
|
}
|
|
|
|
*tp = pattern
|
|
|
|
return nil
|
|
}
|
|
|
|
func (tp *TrackerPattern) Insert(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
scope Scoper,
|
|
) error {
|
|
q := `
|
|
INSERT INTO tracker_patterns (
|
|
id,
|
|
tenant_id,
|
|
organization_id,
|
|
cookie_banner_id,
|
|
cookie_category_id,
|
|
common_tracker_pattern_id,
|
|
third_party_id,
|
|
tracker_type,
|
|
pattern,
|
|
match_type,
|
|
display_name,
|
|
description,
|
|
excluded,
|
|
max_age_seconds,
|
|
source,
|
|
last_matched_at,
|
|
mapping_requested_at,
|
|
created_at,
|
|
updated_at
|
|
) VALUES (
|
|
@id,
|
|
@tenant_id,
|
|
@organization_id,
|
|
@cookie_banner_id,
|
|
@cookie_category_id,
|
|
@common_tracker_pattern_id,
|
|
@third_party_id,
|
|
@tracker_type,
|
|
@pattern,
|
|
@match_type,
|
|
@display_name,
|
|
@description,
|
|
@excluded,
|
|
@max_age_seconds,
|
|
@source,
|
|
@last_matched_at,
|
|
@mapping_requested_at,
|
|
@created_at,
|
|
@updated_at
|
|
)
|
|
`
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"id": tp.ID,
|
|
"tenant_id": scope.GetTenantID(),
|
|
"organization_id": tp.OrganizationID,
|
|
"cookie_banner_id": tp.CookieBannerID,
|
|
"cookie_category_id": tp.CookieCategoryID,
|
|
"common_tracker_pattern_id": tp.CommonTrackerPatternID,
|
|
"third_party_id": tp.ThirdPartyID,
|
|
"tracker_type": tp.TrackerType,
|
|
"pattern": tp.Pattern,
|
|
"match_type": tp.MatchType,
|
|
"display_name": tp.DisplayName,
|
|
"description": tp.Description,
|
|
"excluded": tp.Excluded,
|
|
"max_age_seconds": tp.MaxAgeSeconds,
|
|
"source": tp.Source,
|
|
"last_matched_at": tp.LastMatchedAt,
|
|
"mapping_requested_at": tp.MappingRequestedAt,
|
|
"created_at": tp.CreatedAt,
|
|
"updated_at": tp.UpdatedAt,
|
|
}
|
|
|
|
_, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok {
|
|
if pgErr.Code == "23505" && pgErr.ConstraintName == "idx_tracker_patterns_unique_pattern_per_banner" {
|
|
return ErrResourceAlreadyExists
|
|
}
|
|
}
|
|
|
|
return fmt.Errorf("cannot insert tracker pattern: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (tp *TrackerPattern) InsertIfNotExists(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
scope Scoper,
|
|
) (bool, error) {
|
|
q := `
|
|
INSERT INTO tracker_patterns (
|
|
id,
|
|
tenant_id,
|
|
organization_id,
|
|
cookie_banner_id,
|
|
cookie_category_id,
|
|
common_tracker_pattern_id,
|
|
third_party_id,
|
|
tracker_type,
|
|
pattern,
|
|
match_type,
|
|
display_name,
|
|
description,
|
|
excluded,
|
|
max_age_seconds,
|
|
source,
|
|
last_matched_at,
|
|
mapping_requested_at,
|
|
created_at,
|
|
updated_at
|
|
) VALUES (
|
|
@id,
|
|
@tenant_id,
|
|
@organization_id,
|
|
@cookie_banner_id,
|
|
@cookie_category_id,
|
|
@common_tracker_pattern_id,
|
|
@third_party_id,
|
|
@tracker_type,
|
|
@pattern,
|
|
@match_type,
|
|
@display_name,
|
|
@description,
|
|
@excluded,
|
|
@max_age_seconds,
|
|
@source,
|
|
@last_matched_at,
|
|
@mapping_requested_at,
|
|
@created_at,
|
|
@updated_at
|
|
)
|
|
ON CONFLICT (cookie_banner_id, tracker_type, pattern, COALESCE(max_age_seconds, -1)) DO NOTHING
|
|
`
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"id": tp.ID,
|
|
"tenant_id": scope.GetTenantID(),
|
|
"organization_id": tp.OrganizationID,
|
|
"cookie_banner_id": tp.CookieBannerID,
|
|
"cookie_category_id": tp.CookieCategoryID,
|
|
"common_tracker_pattern_id": tp.CommonTrackerPatternID,
|
|
"third_party_id": tp.ThirdPartyID,
|
|
"tracker_type": tp.TrackerType,
|
|
"pattern": tp.Pattern,
|
|
"match_type": tp.MatchType,
|
|
"display_name": tp.DisplayName,
|
|
"description": tp.Description,
|
|
"excluded": tp.Excluded,
|
|
"max_age_seconds": tp.MaxAgeSeconds,
|
|
"source": tp.Source,
|
|
"last_matched_at": tp.LastMatchedAt,
|
|
"mapping_requested_at": tp.MappingRequestedAt,
|
|
"created_at": tp.CreatedAt,
|
|
"updated_at": tp.UpdatedAt,
|
|
}
|
|
|
|
result, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return false, fmt.Errorf("cannot insert tracker pattern: %w", err)
|
|
}
|
|
|
|
return result.RowsAffected() > 0, nil
|
|
}
|
|
|
|
// Update rewrites the editable columns of the receiver's row,
|
|
// including `source`. Callers MUST load the pattern under the same
|
|
// transaction before mutating fields and calling Update, otherwise
|
|
// stale local values will clobber concurrent writes. To advance
|
|
// `source`, gate the assignment behind shouldPromoteSource in
|
|
// pkg/cookiebanner — there is no DB-side ranking.
|
|
func (tp *TrackerPattern) Update(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
scope Scoper,
|
|
) error {
|
|
q := `
|
|
UPDATE tracker_patterns
|
|
SET
|
|
common_tracker_pattern_id = @common_tracker_pattern_id,
|
|
third_party_id = @third_party_id,
|
|
cookie_category_id = @cookie_category_id,
|
|
display_name = @display_name,
|
|
max_age_seconds = @max_age_seconds,
|
|
description = @description,
|
|
excluded = @excluded,
|
|
source = @source,
|
|
last_matched_at = @last_matched_at,
|
|
updated_at = @updated_at
|
|
WHERE
|
|
%s
|
|
AND id = @id
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"id": tp.ID,
|
|
"common_tracker_pattern_id": tp.CommonTrackerPatternID,
|
|
"third_party_id": tp.ThirdPartyID,
|
|
"cookie_category_id": tp.CookieCategoryID,
|
|
"display_name": tp.DisplayName,
|
|
"max_age_seconds": tp.MaxAgeSeconds,
|
|
"description": tp.Description,
|
|
"excluded": tp.Excluded,
|
|
"source": tp.Source,
|
|
"last_matched_at": tp.LastMatchedAt,
|
|
"updated_at": tp.UpdatedAt,
|
|
}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
result, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
if pgErr, ok := errors.AsType[*pgconn.PgError](err); ok {
|
|
if pgErr.Code == "23505" && pgErr.ConstraintName == "idx_tracker_patterns_unique_pattern_per_banner" {
|
|
return ErrResourceAlreadyExists
|
|
}
|
|
}
|
|
|
|
return fmt.Errorf("cannot update tracker pattern: %w", err)
|
|
}
|
|
|
|
if result.RowsAffected() == 0 {
|
|
return ErrResourceNotFound
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// UpdateMapping writes only the columns the tracker-mapping worker
|
|
// resolves — common_tracker_pattern_id, third_party_id and an enriched
|
|
// description — leaving the user-editable fields (display_name,
|
|
// excluded, cookie_category_id, max_age_seconds, source, last_matched_at)
|
|
// untouched.
|
|
//
|
|
// The worker loads the pattern in its claim transaction and commits the
|
|
// resolution in a separate, later transaction, so a full-row Update
|
|
// would write back stale values and clobber any concurrent edit made in
|
|
// between. The description is only filled when still empty in the
|
|
// database, so a concurrently set description is never overwritten.
|
|
func (tp *TrackerPattern) UpdateMapping(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
scope Scoper,
|
|
) error {
|
|
q := `
|
|
UPDATE tracker_patterns
|
|
SET
|
|
common_tracker_pattern_id = @common_tracker_pattern_id,
|
|
third_party_id = @third_party_id,
|
|
description = CASE
|
|
WHEN description = '' THEN @description
|
|
ELSE description
|
|
END,
|
|
updated_at = @updated_at
|
|
WHERE
|
|
%s
|
|
AND id = @id
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"id": tp.ID,
|
|
"common_tracker_pattern_id": tp.CommonTrackerPatternID,
|
|
"third_party_id": tp.ThirdPartyID,
|
|
"description": tp.Description,
|
|
"updated_at": tp.UpdatedAt,
|
|
}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
result, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot update tracker pattern mapping: %w", err)
|
|
}
|
|
|
|
if result.RowsAffected() == 0 {
|
|
return ErrResourceNotFound
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (tp *TrackerPattern) Delete(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
scope Scoper,
|
|
) error {
|
|
q := `
|
|
DELETE FROM tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND id = @id
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{"id": tp.ID}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
_, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot delete tracker pattern: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (tps *TrackerPatterns) LoadAllByCookieBannerID(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
scope Scoper,
|
|
cookieBannerID gid.GID,
|
|
filter *TrackerPatternFilter,
|
|
trackerType *TrackerType,
|
|
) error {
|
|
trackerTypeFragment := "TRUE"
|
|
if trackerType != nil {
|
|
trackerTypeFragment = "tracker_type = @tracker_type"
|
|
}
|
|
|
|
q := `
|
|
SELECT
|
|
id,
|
|
organization_id,
|
|
cookie_banner_id,
|
|
cookie_category_id,
|
|
common_tracker_pattern_id,
|
|
third_party_id,
|
|
tracker_type,
|
|
pattern,
|
|
match_type,
|
|
display_name,
|
|
description,
|
|
excluded,
|
|
max_age_seconds,
|
|
source,
|
|
last_matched_at,
|
|
mapping_requested_at,
|
|
created_at,
|
|
updated_at
|
|
FROM
|
|
tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND cookie_banner_id = @cookie_banner_id
|
|
AND %s
|
|
AND %s
|
|
ORDER BY
|
|
created_at ASC, id ASC;
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment(), trackerTypeFragment, filter.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{"cookie_banner_id": cookieBannerID}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
maps.Copy(args, filter.SQLArguments())
|
|
|
|
if trackerType != nil {
|
|
args["tracker_type"] = *trackerType
|
|
}
|
|
|
|
rows, err := conn.Query(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot query tracker patterns: %w", err)
|
|
}
|
|
|
|
patterns, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[TrackerPattern])
|
|
if err != nil {
|
|
return fmt.Errorf("cannot collect tracker patterns: %w", err)
|
|
}
|
|
|
|
*tps = patterns
|
|
|
|
return nil
|
|
}
|
|
|
|
func (tps *TrackerPatterns) RefreshLastMatchedAtByCookieBannerID(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
scope Scoper,
|
|
cookieBannerID gid.GID,
|
|
) error {
|
|
q := `
|
|
UPDATE tracker_patterns
|
|
SET
|
|
last_matched_at = sub.max_detected
|
|
FROM (
|
|
SELECT tracker_pattern_id, MAX(last_detected_at) AS max_detected
|
|
FROM detected_trackers
|
|
WHERE %[1]s AND cookie_banner_id = @cookie_banner_id
|
|
GROUP BY tracker_pattern_id
|
|
) sub
|
|
WHERE
|
|
tracker_patterns.id = sub.tracker_pattern_id
|
|
AND %[1]s
|
|
AND tracker_patterns.cookie_banner_id = @cookie_banner_id
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{"cookie_banner_id": cookieBannerID}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
_, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot refresh last_matched_at for banner tracker patterns: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (tps *TrackerPatterns) LoadByCookieBannerID(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
scope Scoper,
|
|
cookieBannerID gid.GID,
|
|
cursor *page.Cursor[TrackerPatternOrderField],
|
|
filter *TrackerPatternFilter,
|
|
) error {
|
|
q := `
|
|
SELECT
|
|
id,
|
|
organization_id,
|
|
cookie_banner_id,
|
|
cookie_category_id,
|
|
common_tracker_pattern_id,
|
|
third_party_id,
|
|
tracker_type,
|
|
pattern,
|
|
match_type,
|
|
display_name,
|
|
description,
|
|
excluded,
|
|
max_age_seconds,
|
|
source,
|
|
last_matched_at,
|
|
mapping_requested_at,
|
|
created_at,
|
|
updated_at
|
|
FROM
|
|
tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND cookie_banner_id = @cookie_banner_id
|
|
AND %s
|
|
AND %s
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment(), filter.SQLFragment(), cursor.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"cookie_banner_id": cookieBannerID,
|
|
}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
maps.Copy(args, filter.SQLArguments())
|
|
maps.Copy(args, cursor.SQLArguments())
|
|
|
|
rows, err := conn.Query(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot query tracker patterns: %w", err)
|
|
}
|
|
|
|
patterns, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[TrackerPattern])
|
|
if err != nil {
|
|
return fmt.Errorf("cannot collect tracker patterns: %w", err)
|
|
}
|
|
|
|
*tps = patterns
|
|
|
|
return nil
|
|
}
|
|
|
|
func (tps *TrackerPatterns) CountByCookieBannerID(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
scope Scoper,
|
|
cookieBannerID gid.GID,
|
|
filter *TrackerPatternFilter,
|
|
) (int, error) {
|
|
q := `
|
|
SELECT
|
|
COUNT(id)
|
|
FROM
|
|
tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND cookie_banner_id = @cookie_banner_id
|
|
AND %s
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment(), filter.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"cookie_banner_id": cookieBannerID,
|
|
}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
maps.Copy(args, filter.SQLArguments())
|
|
|
|
row := conn.QueryRow(ctx, q, args)
|
|
|
|
var count int
|
|
if err := row.Scan(&count); err != nil {
|
|
return 0, fmt.Errorf("cannot count tracker patterns: %w", err)
|
|
}
|
|
|
|
return count, nil
|
|
}
|
|
|
|
func (tps *TrackerPatterns) LoadByCookieCategoryID(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
scope Scoper,
|
|
cookieCategoryID gid.GID,
|
|
cursor *page.Cursor[TrackerPatternOrderField],
|
|
) error {
|
|
q := `
|
|
SELECT
|
|
id,
|
|
organization_id,
|
|
cookie_banner_id,
|
|
cookie_category_id,
|
|
common_tracker_pattern_id,
|
|
third_party_id,
|
|
tracker_type,
|
|
pattern,
|
|
match_type,
|
|
display_name,
|
|
description,
|
|
excluded,
|
|
max_age_seconds,
|
|
source,
|
|
last_matched_at,
|
|
mapping_requested_at,
|
|
created_at,
|
|
updated_at
|
|
FROM
|
|
tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND cookie_category_id = @cookie_category_id
|
|
AND %s
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment(), cursor.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{"cookie_category_id": cookieCategoryID}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
maps.Copy(args, cursor.SQLArguments())
|
|
|
|
rows, err := conn.Query(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot query tracker patterns: %w", err)
|
|
}
|
|
|
|
patterns, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[TrackerPattern])
|
|
if err != nil {
|
|
return fmt.Errorf("cannot collect tracker patterns: %w", err)
|
|
}
|
|
|
|
*tps = patterns
|
|
|
|
return nil
|
|
}
|
|
|
|
func (tps *TrackerPatterns) CountByCookieCategoryID(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
scope Scoper,
|
|
cookieCategoryID gid.GID,
|
|
) (int, error) {
|
|
q := `
|
|
SELECT
|
|
COUNT(id)
|
|
FROM
|
|
tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND cookie_category_id = @cookie_category_id
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{"cookie_category_id": cookieCategoryID}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
row := conn.QueryRow(ctx, q, args)
|
|
|
|
var count int
|
|
if err := row.Scan(&count); err != nil {
|
|
return 0, fmt.Errorf("cannot scan count: %w", err)
|
|
}
|
|
|
|
return count, nil
|
|
}
|
|
|
|
// LoadDistinctThirdPartyIDsByCookieBannerID returns the distinct non-null
|
|
// `third_party_id` values referenced by tracker patterns of the given
|
|
// banner. Callers feed it to ThirdParty.GetByIDs to power per-banner
|
|
// pickers without crossing the entity boundary.
|
|
func (tps *TrackerPatterns) LoadDistinctThirdPartyIDsByCookieBannerID(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
scope Scoper,
|
|
cookieBannerID gid.GID,
|
|
) ([]gid.GID, error) {
|
|
q := `
|
|
SELECT DISTINCT third_party_id
|
|
FROM tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND cookie_banner_id = @cookie_banner_id
|
|
AND third_party_id IS NOT NULL
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{"cookie_banner_id": cookieBannerID}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
rows, err := conn.Query(ctx, q, args)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot query distinct third party ids: %w", err)
|
|
}
|
|
|
|
ids, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot collect distinct third party ids: %w", err)
|
|
}
|
|
|
|
return ids, nil
|
|
}
|
|
|
|
// LoadDistinctCommonTrackerPatternIDsByCookieBannerID returns the
|
|
// distinct non-null `common_tracker_pattern_id` values referenced by
|
|
// tracker patterns of the given banner. Callers chain this with
|
|
// CommonTrackerPatterns.LoadByIDs and CommonThirdParties.LoadByIDs to
|
|
// resolve the linked common third parties without JOINs.
|
|
func (tps *TrackerPatterns) LoadDistinctCommonTrackerPatternIDsByCookieBannerID(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
scope Scoper,
|
|
cookieBannerID gid.GID,
|
|
) ([]gid.GID, error) {
|
|
q := `
|
|
SELECT DISTINCT common_tracker_pattern_id
|
|
FROM tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND cookie_banner_id = @cookie_banner_id
|
|
AND common_tracker_pattern_id IS NOT NULL
|
|
AND third_party_id IS NULL
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{"cookie_banner_id": cookieBannerID}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
rows, err := conn.Query(ctx, q, args)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot query distinct common tracker pattern ids: %w", err)
|
|
}
|
|
|
|
ids, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot collect distinct common tracker pattern ids: %w", err)
|
|
}
|
|
|
|
return ids, nil
|
|
}
|
|
|
|
func (tps *TrackerPatterns) LoadDistinctThirdPartyIDsByIDs(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
scope Scoper,
|
|
ids []gid.GID,
|
|
) ([]gid.GID, error) {
|
|
if len(ids) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
q := `
|
|
SELECT DISTINCT third_party_id
|
|
FROM tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND id = ANY(@ids)
|
|
AND third_party_id IS NOT NULL
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{"ids": ids}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
rows, err := conn.Query(ctx, q, args)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot query distinct third party ids by pattern ids: %w", err)
|
|
}
|
|
|
|
thirdPartyIDs, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot collect distinct third party ids by pattern ids: %w", err)
|
|
}
|
|
|
|
return thirdPartyIDs, nil
|
|
}
|
|
|
|
func (tps *TrackerPatterns) LoadDistinctCommonTrackerPatternIDsByIDs(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
scope Scoper,
|
|
ids []gid.GID,
|
|
) ([]gid.GID, error) {
|
|
if len(ids) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
q := `
|
|
SELECT DISTINCT common_tracker_pattern_id
|
|
FROM tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND id = ANY(@ids)
|
|
AND common_tracker_pattern_id IS NOT NULL
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{"ids": ids}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
rows, err := conn.Query(ctx, q, args)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot query distinct common tracker pattern ids by pattern ids: %w", err)
|
|
}
|
|
|
|
commonPatternIDs, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot collect distinct common tracker pattern ids by pattern ids: %w", err)
|
|
}
|
|
|
|
return commonPatternIDs, nil
|
|
}
|
|
|
|
func (tps *TrackerPatterns) UpdateLastMatchedAt(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
scope Scoper,
|
|
patternIDs []gid.GID,
|
|
matchedAt time.Time,
|
|
) error {
|
|
if len(patternIDs) == 0 {
|
|
return nil
|
|
}
|
|
|
|
q := `
|
|
UPDATE tracker_patterns
|
|
SET
|
|
last_matched_at = @matched_at,
|
|
updated_at = @updated_at
|
|
WHERE
|
|
%s
|
|
AND id = ANY(@pattern_ids)
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"pattern_ids": patternIDs,
|
|
"matched_at": matchedAt,
|
|
"updated_at": matchedAt,
|
|
}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
_, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot update last_matched_at for tracker patterns: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (tps *TrackerPatterns) MoveToCategoryByCookieCategoryID(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
scope Scoper,
|
|
sourceCategoryID gid.GID,
|
|
targetCategoryID gid.GID,
|
|
) error {
|
|
q := `
|
|
UPDATE tracker_patterns
|
|
SET
|
|
cookie_category_id = @target_category_id,
|
|
updated_at = @updated_at
|
|
WHERE
|
|
%s
|
|
AND cookie_category_id = @source_category_id
|
|
`
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"source_category_id": sourceCategoryID,
|
|
"target_category_id": targetCategoryID,
|
|
"updated_at": time.Now(),
|
|
}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
_, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot move tracker patterns to category: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// LinkThirdPartyByCommonThirdPartyID points the organization's unlinked
|
|
// tracker patterns at an org ThirdParty when their catalog row resolves
|
|
// to the given common third party. It is the backfill the explicit
|
|
// import action runs so patterns that previously surfaced the catalog
|
|
// (CommonThirdParty) entry now surface the managed org ThirdParty. Only
|
|
// patterns with no third_party_id are touched, so it is idempotent and
|
|
// never overrides an existing link. The common_tracker_patterns
|
|
// subquery only narrows the WHERE clause, keeping the resolution in the
|
|
// database.
|
|
func (tps *TrackerPatterns) LinkThirdPartyByCommonThirdPartyID(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
scope Scoper,
|
|
organizationID gid.GID,
|
|
commonThirdPartyID gid.GID,
|
|
thirdPartyID gid.GID,
|
|
) error {
|
|
q := `
|
|
UPDATE tracker_patterns
|
|
SET
|
|
third_party_id = @third_party_id,
|
|
updated_at = NOW()
|
|
WHERE
|
|
%s
|
|
AND organization_id = @organization_id
|
|
AND third_party_id IS NULL
|
|
AND common_tracker_pattern_id IN (
|
|
SELECT id FROM common_tracker_patterns
|
|
WHERE common_third_party_id = @common_third_party_id
|
|
)
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"organization_id": organizationID,
|
|
"common_third_party_id": commonThirdPartyID,
|
|
"third_party_id": thirdPartyID,
|
|
}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
_, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot link tracker patterns to third party: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (tp *TrackerPattern) LoadNextForMappingForUpdateSkipLocked(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
) error {
|
|
q := `
|
|
SELECT
|
|
id,
|
|
organization_id,
|
|
cookie_banner_id,
|
|
cookie_category_id,
|
|
common_tracker_pattern_id,
|
|
third_party_id,
|
|
tracker_type,
|
|
pattern,
|
|
match_type,
|
|
display_name,
|
|
description,
|
|
excluded,
|
|
max_age_seconds,
|
|
source,
|
|
last_matched_at,
|
|
mapping_requested_at,
|
|
created_at,
|
|
updated_at
|
|
FROM
|
|
tracker_patterns
|
|
WHERE
|
|
mapping_requested_at IS NOT NULL
|
|
ORDER BY
|
|
mapping_requested_at ASC
|
|
FOR UPDATE SKIP LOCKED
|
|
LIMIT 1;
|
|
`
|
|
|
|
rows, err := tx.Query(ctx, q)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot query tracker patterns for mapping: %w", err)
|
|
}
|
|
|
|
pattern, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[TrackerPattern])
|
|
if err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return ErrResourceNotFound
|
|
}
|
|
|
|
return fmt.Errorf("cannot collect tracker pattern for mapping: %w", err)
|
|
}
|
|
|
|
*tp = pattern
|
|
|
|
return nil
|
|
}
|
|
|
|
// ClearMappingRequestedAt removes the row from the mapping queue. It
|
|
// bumps updated_at so the stale-recovery clock starts at claim time,
|
|
// keeping ResetStaleMappings from re-arming a row that is still being
|
|
// processed.
|
|
func (tp *TrackerPattern) ClearMappingRequestedAt(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
) error {
|
|
q := `
|
|
UPDATE tracker_patterns
|
|
SET
|
|
mapping_requested_at = NULL,
|
|
updated_at = NOW()
|
|
WHERE id = @id
|
|
`
|
|
|
|
args := pgx.StrictNamedArgs{"id": tp.ID}
|
|
|
|
_, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot clear mapping requested at: %w", err)
|
|
}
|
|
|
|
tp.MappingRequestedAt = nil
|
|
|
|
return nil
|
|
}
|
|
|
|
// ResetStaleMappings re-arms mapping_requested_at on rows whose mapping
|
|
// was claimed but never completed (no common_tracker_pattern_id) and
|
|
// have been idle longer than staleAfter, so a crashed or timed-out
|
|
// mapping run is retried. A successful Process always assigns a catalog
|
|
// row (the unmatched fallback in createUnmatchedPattern), so a missing
|
|
// common_tracker_pattern_id on a dequeued row marks an interrupted run.
|
|
//
|
|
// Like the claim query, this sweep is intentionally cross-tenant: the
|
|
// mapping worker is a system worker that drains the queue regardless of
|
|
// tenant.
|
|
func ResetStaleMappings(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
staleAfter time.Duration,
|
|
) error {
|
|
q := `
|
|
UPDATE tracker_patterns
|
|
SET
|
|
mapping_requested_at = NOW(),
|
|
updated_at = NOW()
|
|
WHERE
|
|
mapping_requested_at IS NULL
|
|
AND common_tracker_pattern_id IS NULL
|
|
AND updated_at < @stale_before
|
|
`
|
|
|
|
args := pgx.StrictNamedArgs{"stale_before": time.Now().Add(-staleAfter)}
|
|
|
|
_, err := conn.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot reset stale tracker pattern mappings: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (tp *TrackerPattern) SetMappingRequested(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
) error {
|
|
q := `
|
|
UPDATE tracker_patterns
|
|
SET mapping_requested_at = NOW()
|
|
WHERE id = @id
|
|
AND mapping_requested_at IS NULL
|
|
`
|
|
|
|
args := pgx.StrictNamedArgs{"id": tp.ID}
|
|
|
|
_, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot set mapping requested: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// RequestMappingForUnmappedSiblings re-arms mapping_requested_at on
|
|
// sibling tracker patterns of the same banner that share an initiator
|
|
// domain with the just-mapped pattern but are still unpromoted. It is
|
|
// the backward-propagation counterpart to the mapping worker's
|
|
// sibling-origin matching: when a pattern newly resolves a vendor, its
|
|
// siblings that were processed earlier and left unmatched can now be
|
|
// re-evaluated against it.
|
|
//
|
|
// Only unpromoted (third_party_id IS NULL), not-already-queued
|
|
// (mapping_requested_at IS NULL), non-extension siblings are touched, so
|
|
// a fully mapped banner re-enqueues nothing. detected_trackers is used
|
|
// only as a filtering subquery. Returns the number of siblings
|
|
// re-enqueued.
|
|
func (tps *TrackerPatterns) RequestMappingForUnmappedSiblings(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
scope Scoper,
|
|
cookieBannerID gid.GID,
|
|
excludePatternID gid.GID,
|
|
domains []string,
|
|
) (int64, error) {
|
|
if len(domains) == 0 {
|
|
return 0, nil
|
|
}
|
|
|
|
q := `
|
|
UPDATE tracker_patterns
|
|
SET
|
|
mapping_requested_at = NOW(),
|
|
updated_at = NOW()
|
|
WHERE
|
|
%[1]s
|
|
AND cookie_banner_id = @cookie_banner_id
|
|
AND id != @exclude_pattern_id
|
|
AND third_party_id IS NULL
|
|
AND mapping_requested_at IS NULL
|
|
AND (source IS NULL OR source != @extension_source)
|
|
AND id IN (
|
|
SELECT DISTINCT tracker_pattern_id
|
|
FROM detected_trackers
|
|
WHERE %[1]s
|
|
AND cookie_banner_id = @cookie_banner_id
|
|
AND initiator_domain = ANY(@domains)
|
|
AND tracker_pattern_id IS NOT NULL
|
|
)
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"cookie_banner_id": cookieBannerID,
|
|
"exclude_pattern_id": excludePatternID,
|
|
"extension_source": CookieSourceExtension,
|
|
"domains": domains,
|
|
}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
result, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("cannot request mapping for unmapped siblings: %w", err)
|
|
}
|
|
|
|
return result.RowsAffected(), nil
|
|
}
|
|
|
|
// BackfillDescriptionByCommonTrackerPatternID copies an enriched
|
|
// description onto every tracker pattern linked to the given common
|
|
// pattern that does not yet have one. It is invoked by the common-pattern
|
|
// enrichment worker, a global system process, so it is intentionally not
|
|
// tenant-scoped: a single catalog enrichment fans out to all tenants'
|
|
// linked patterns. The description = ” guard guarantees a pattern that
|
|
// already carries a description is never overwritten. Returns the number
|
|
// of patterns backfilled.
|
|
func (tps *TrackerPatterns) BackfillDescriptionByCommonTrackerPatternID(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
commonTrackerPatternID gid.GID,
|
|
description string,
|
|
) (int64, error) {
|
|
q := `
|
|
UPDATE tracker_patterns
|
|
SET
|
|
description = @description,
|
|
updated_at = NOW()
|
|
WHERE
|
|
common_tracker_pattern_id = @common_tracker_pattern_id
|
|
AND description = ''
|
|
`
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"common_tracker_pattern_id": commonTrackerPatternID,
|
|
"description": description,
|
|
}
|
|
|
|
result, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("cannot backfill tracker pattern descriptions: %w", err)
|
|
}
|
|
|
|
return result.RowsAffected(), nil
|
|
}
|
|
|
|
// RequestMappingForUncategorisedByCommonTrackerPatternIDs re-arms mapping
|
|
// on the uncategorised org tracker patterns linked to the given common
|
|
// tracker patterns: it clears their resolved org third party and stamps
|
|
// mapping_requested_at so the mapping worker re-resolves the vendor from
|
|
// the catalog row's (now changed) common third party. Like the
|
|
// description backfill it is a global catalog operation, so it is
|
|
// intentionally not tenant-scoped. Excluded patterns and patterns in
|
|
// user-chosen categories are left untouched - only the uncategorised
|
|
// category is remapped, matching the reset-trackers philosophy. The
|
|
// cookie_categories subquery is used only for filtering. Returns the
|
|
// number of org patterns re-armed.
|
|
func (tps *TrackerPatterns) RequestMappingForUncategorisedByCommonTrackerPatternIDs(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
commonIDs []gid.GID,
|
|
) (int64, error) {
|
|
q := `
|
|
UPDATE tracker_patterns
|
|
SET
|
|
third_party_id = NULL,
|
|
mapping_requested_at = NOW(),
|
|
updated_at = NOW()
|
|
WHERE
|
|
common_tracker_pattern_id = ANY(@common_ids)
|
|
AND excluded = false
|
|
AND cookie_category_id IN (
|
|
SELECT id FROM cookie_categories WHERE kind = @uncategorised_kind
|
|
)
|
|
`
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"common_ids": commonIDs,
|
|
"uncategorised_kind": CookieCategoryKindUncategorised,
|
|
}
|
|
|
|
result, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("cannot request mapping for uncategorised tracker patterns: %w", err)
|
|
}
|
|
|
|
return result.RowsAffected(), nil
|
|
}
|
|
|
|
// RequestMappingForUnmappedByInitiatorDomains re-arms mapping on the
|
|
// still-unmapped org tracker patterns whose detected trackers share one
|
|
// of the given initiator domains, so the mapping worker re-resolves them
|
|
// via its domain-overlap signal. It is invoked by the common-third-party
|
|
// enrichment worker when a vendor gains new owned domains, a global
|
|
// catalog operation, so it is intentionally not tenant-scoped: a single
|
|
// catalog domain benefits all tenants' patterns.
|
|
//
|
|
// Only patterns with no resolved vendor are targeted (third_party_id IS
|
|
// NULL and an absent or unlinked catalog row), so a pattern already
|
|
// linked to the vendor that gained the domain - or to any other vendor -
|
|
// is never disturbed and never re-attributed. Extension-sourced patterns
|
|
// and patterns already queued for mapping are skipped. The
|
|
// common_tracker_patterns and detected_trackers subqueries are used only
|
|
// for filtering. Returns the number of patterns re-armed; an empty
|
|
// domains slice is a no-op.
|
|
func (tps *TrackerPatterns) RequestMappingForUnmappedByInitiatorDomains(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
domains []string,
|
|
) (int64, error) {
|
|
if len(domains) == 0 {
|
|
return 0, nil
|
|
}
|
|
|
|
q := `
|
|
UPDATE tracker_patterns
|
|
SET
|
|
mapping_requested_at = NOW(),
|
|
updated_at = NOW()
|
|
WHERE
|
|
third_party_id IS NULL
|
|
AND mapping_requested_at IS NULL
|
|
AND (source IS NULL OR source != @extension_source)
|
|
AND (
|
|
common_tracker_pattern_id IS NULL
|
|
OR common_tracker_pattern_id IN (
|
|
SELECT id FROM common_tracker_patterns
|
|
WHERE common_third_party_id IS NULL
|
|
)
|
|
)
|
|
AND id IN (
|
|
SELECT DISTINCT tracker_pattern_id
|
|
FROM detected_trackers
|
|
WHERE initiator_domain = ANY(@domains)
|
|
AND tracker_pattern_id IS NOT NULL
|
|
)
|
|
`
|
|
|
|
args := pgx.StrictNamedArgs{
|
|
"extension_source": CookieSourceExtension,
|
|
"domains": domains,
|
|
}
|
|
|
|
result, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("cannot request mapping for unmapped tracker patterns by initiator domains: %w", err)
|
|
}
|
|
|
|
return result.RowsAffected(), nil
|
|
}
|
|
|
|
// ResetAndRequestMappingByCookieCategoryID detaches every pattern in the
|
|
// given category from its catalog row, org third party, and copied
|
|
// description, then re-arms mapping. Operators run this (via proboctl) on
|
|
// a banner's uncategorised category to force a clean re-map when
|
|
// iterating on the mapping agent. Excluded patterns are left untouched -
|
|
// exclusion is a deliberate suppression. The cookie_category_id key
|
|
// scopes the reset to the uncategorised category the caller resolves;
|
|
// the Scoper keeps it tenant-isolated. When keyword is non-nil and
|
|
// non-empty, the reset is further restricted to patterns whose pattern or
|
|
// display name contains it (case-insensitive). Returns the number of
|
|
// patterns reset.
|
|
func (tps *TrackerPatterns) ResetAndRequestMappingByCookieCategoryID(
|
|
ctx context.Context,
|
|
tx pg.Tx,
|
|
scope Scoper,
|
|
cookieCategoryID gid.GID,
|
|
keyword *string,
|
|
) (int64, error) {
|
|
filter := NewTrackerPatternFilter(nil, nil, nil).WithPatternKeyword(keyword)
|
|
|
|
q := `
|
|
UPDATE tracker_patterns
|
|
SET
|
|
common_tracker_pattern_id = NULL,
|
|
third_party_id = NULL,
|
|
description = '',
|
|
mapping_requested_at = NOW(),
|
|
updated_at = NOW()
|
|
WHERE
|
|
%s
|
|
AND cookie_category_id = @cookie_category_id
|
|
AND excluded = false
|
|
AND %s
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment(), filter.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{"cookie_category_id": cookieCategoryID}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
maps.Copy(args, filter.SQLArguments())
|
|
|
|
result, err := tx.Exec(ctx, q, args)
|
|
if err != nil {
|
|
return 0, fmt.Errorf("cannot reset and request mapping by cookie category: %w", err)
|
|
}
|
|
|
|
return result.RowsAffected(), nil
|
|
}
|
|
|
|
// LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID returns every
|
|
// distinct common_tracker_pattern_id referenced by the banner's patterns,
|
|
// regardless of mapping state. Unlike
|
|
// LoadDistinctCommonTrackerPatternIDsByCookieBannerID (which restricts to
|
|
// unmapped patterns for the mapping pipeline), this returns the full set
|
|
// of catalog rows the banner depends on, so an operator can re-describe
|
|
// exactly those before a reset.
|
|
func (tps *TrackerPatterns) LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
scope Scoper,
|
|
cookieBannerID gid.GID,
|
|
) ([]gid.GID, error) {
|
|
q := `
|
|
SELECT DISTINCT common_tracker_pattern_id
|
|
FROM tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND cookie_banner_id = @cookie_banner_id
|
|
AND common_tracker_pattern_id IS NOT NULL
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{"cookie_banner_id": cookieBannerID}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
rows, err := conn.Query(ctx, q, args)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot query linked common tracker pattern ids: %w", err)
|
|
}
|
|
|
|
ids, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot collect linked common tracker pattern ids: %w", err)
|
|
}
|
|
|
|
return ids, nil
|
|
}
|
|
|
|
// LoadAllLinkedCommonTrackerPatternIDsByOrganizationID is the org-wide
|
|
// counterpart of LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID:
|
|
// every distinct catalog row the organization's tracker patterns depend
|
|
// on, regardless of mapping state.
|
|
func (tps *TrackerPatterns) LoadAllLinkedCommonTrackerPatternIDsByOrganizationID(
|
|
ctx context.Context,
|
|
conn pg.Querier,
|
|
scope Scoper,
|
|
organizationID gid.GID,
|
|
) ([]gid.GID, error) {
|
|
q := `
|
|
SELECT DISTINCT common_tracker_pattern_id
|
|
FROM tracker_patterns
|
|
WHERE
|
|
%s
|
|
AND organization_id = @organization_id
|
|
AND common_tracker_pattern_id IS NOT NULL
|
|
`
|
|
|
|
q = fmt.Sprintf(q, scope.SQLFragment())
|
|
|
|
args := pgx.StrictNamedArgs{"organization_id": organizationID}
|
|
maps.Copy(args, scope.SQLArguments())
|
|
|
|
rows, err := conn.Query(ctx, q, args)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot query linked common tracker pattern ids: %w", err)
|
|
}
|
|
|
|
ids, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cannot collect linked common tracker pattern ids: %w", err)
|
|
}
|
|
|
|
return ids, nil
|
|
}
|