Add common catalog query layer and enricher service
Introduce an API-style data layer for the global common tracker pattern and common third party catalogs: typed filters, order fields, CursorKey, cursor-paginated Load and CountAll, plus by-id enrichment re-queue and a scoped reset/remap helper for a banner's tracker patterns. These reuse the same page.Cursor/filter/order types the GraphQL API consumes, so a future proboctl API can back them unchanged. Extract the common-pattern enrichment logic out of the worker into a CommonPatternEnricher service so it can run either from the background queue or synchronously over a known set of ids; the worker becomes a thin poller that delegates to it. Extract the LLM client and tracker-agents config wiring into pkg/agentsbuild so probod and other binaries build agents identically; probod now delegates to it. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -25,6 +25,7 @@ import (
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/iam/policy"
|
||||
"go.probo.inc/probo/pkg/page"
|
||||
)
|
||||
|
||||
type (
|
||||
@@ -644,3 +645,108 @@ WHERE
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *CommonThirdParty) CursorKey(field CommonThirdPartyOrderField) page.CursorKey {
|
||||
switch field {
|
||||
case CommonThirdPartyOrderFieldName:
|
||||
return page.NewCursorKey(t.ID, t.Name)
|
||||
case CommonThirdPartyOrderFieldCreatedAt:
|
||||
return page.NewCursorKey(t.ID, t.CreatedAt)
|
||||
case CommonThirdPartyOrderFieldUpdatedAt:
|
||||
return page.NewCursorKey(t.ID, t.UpdatedAt)
|
||||
}
|
||||
|
||||
panic(fmt.Sprintf("unsupported order by: %s", field))
|
||||
}
|
||||
|
||||
// Load returns a cursor-paginated, filtered page of common third
|
||||
// parties. The catalog is global (no tenant scope); the cursor supplies
|
||||
// the limit and ordering. Unlike LoadAll (capped at 20, name only), this
|
||||
// is the listing entry point a future API/CLI consumes.
|
||||
func (t *CommonThirdParties) Load(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
cursor *page.Cursor[CommonThirdPartyOrderField],
|
||||
filter *CommonThirdPartyFilter,
|
||||
) error {
|
||||
q := `
|
||||
SELECT
|
||||
id,
|
||||
name,
|
||||
slug,
|
||||
category,
|
||||
headquarter_address,
|
||||
legal_name,
|
||||
website_url,
|
||||
privacy_policy_url,
|
||||
service_level_agreement_url,
|
||||
service_software_agreement_url,
|
||||
data_processing_agreement_url,
|
||||
business_associate_agreement_url,
|
||||
subprocessors_list_url,
|
||||
certifications,
|
||||
status_page_url,
|
||||
terms_of_service_url,
|
||||
security_page_url,
|
||||
trust_page_url,
|
||||
logo_file_id,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM
|
||||
common_third_parties
|
||||
WHERE
|
||||
%s
|
||||
AND %s
|
||||
`
|
||||
|
||||
q = fmt.Sprintf(q, filter.SQLFragment(), cursor.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{}
|
||||
maps.Copy(args, filter.SQLArguments())
|
||||
maps.Copy(args, cursor.SQLArguments())
|
||||
|
||||
rows, err := conn.Query(ctx, q, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot query common third parties: %w", err)
|
||||
}
|
||||
|
||||
parties, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[CommonThirdParty])
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot collect common third parties: %w", err)
|
||||
}
|
||||
|
||||
*t = parties
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CountAll returns the number of common third parties matching the
|
||||
// filter, ignoring pagination.
|
||||
func (t *CommonThirdParties) CountAll(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
filter *CommonThirdPartyFilter,
|
||||
) (int, error) {
|
||||
q := `
|
||||
SELECT
|
||||
COUNT(id)
|
||||
FROM
|
||||
common_third_parties
|
||||
WHERE
|
||||
%s
|
||||
`
|
||||
|
||||
q = fmt.Sprintf(q, filter.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{}
|
||||
maps.Copy(args, filter.SQLArguments())
|
||||
|
||||
row := conn.QueryRow(ctx, q, args)
|
||||
|
||||
var count int
|
||||
if err := row.Scan(&count); err != nil {
|
||||
return 0, fmt.Errorf("cannot count common third parties: %w", err)
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
@@ -19,13 +19,25 @@ import (
|
||||
)
|
||||
|
||||
type CommonThirdPartyFilter struct {
|
||||
name *string
|
||||
name *string
|
||||
category *ThirdPartyCategory
|
||||
keyword *string
|
||||
}
|
||||
|
||||
func NewCommonThirdPartyFilter(name *string) *CommonThirdPartyFilter {
|
||||
return &CommonThirdPartyFilter{name: name}
|
||||
}
|
||||
|
||||
func (f *CommonThirdPartyFilter) WithCategory(category *ThirdPartyCategory) *CommonThirdPartyFilter {
|
||||
f.category = category
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *CommonThirdPartyFilter) WithKeyword(keyword *string) *CommonThirdPartyFilter {
|
||||
f.keyword = keyword
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *CommonThirdPartyFilter) SQLFragment() string {
|
||||
return `(
|
||||
CASE
|
||||
@@ -33,14 +45,40 @@ func (f *CommonThirdPartyFilter) SQLFragment() string {
|
||||
name ILIKE '%' || @filter_name || '%'
|
||||
ELSE TRUE
|
||||
END
|
||||
AND
|
||||
CASE
|
||||
WHEN @filter_category::text IS NOT NULL THEN
|
||||
category = @filter_category::third_party_category
|
||||
ELSE TRUE
|
||||
END
|
||||
AND
|
||||
CASE
|
||||
WHEN @filter_keyword::text IS NOT NULL AND @filter_keyword::text != '' THEN
|
||||
(name ILIKE '%' || @filter_keyword || '%'
|
||||
OR slug ILIKE '%' || @filter_keyword || '%')
|
||||
ELSE TRUE
|
||||
END
|
||||
)`
|
||||
}
|
||||
|
||||
func (f *CommonThirdPartyFilter) SQLArguments() pgx.StrictNamedArgs {
|
||||
args := pgx.StrictNamedArgs{"filter_name": nil}
|
||||
args := pgx.StrictNamedArgs{
|
||||
"filter_name": nil,
|
||||
"filter_category": nil,
|
||||
"filter_keyword": nil,
|
||||
}
|
||||
|
||||
if f.name != nil {
|
||||
args["filter_name"] = *f.name
|
||||
}
|
||||
|
||||
if f.category != nil {
|
||||
args["filter_category"] = string(*f.category)
|
||||
}
|
||||
|
||||
if f.keyword != nil {
|
||||
args["filter_keyword"] = *f.keyword
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
89
pkg/coredata/common_third_party_order_field.go
Normal file
89
pkg/coredata/common_third_party_order_field.go
Normal file
@@ -0,0 +1,89 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package coredata
|
||||
|
||||
import (
|
||||
"encoding"
|
||||
"fmt"
|
||||
|
||||
"go.probo.inc/probo/pkg/page"
|
||||
)
|
||||
|
||||
type CommonThirdPartyOrderField string
|
||||
|
||||
const (
|
||||
CommonThirdPartyOrderFieldName CommonThirdPartyOrderField = "NAME"
|
||||
CommonThirdPartyOrderFieldCreatedAt CommonThirdPartyOrderField = "CREATED_AT"
|
||||
CommonThirdPartyOrderFieldUpdatedAt CommonThirdPartyOrderField = "UPDATED_AT"
|
||||
)
|
||||
|
||||
var (
|
||||
_ page.OrderField = CommonThirdPartyOrderField("")
|
||||
_ fmt.Stringer = CommonThirdPartyOrderField("")
|
||||
_ encoding.TextMarshaler = CommonThirdPartyOrderField("")
|
||||
_ encoding.TextUnmarshaler = (*CommonThirdPartyOrderField)(nil)
|
||||
)
|
||||
|
||||
func CommonThirdPartyOrderFields() []CommonThirdPartyOrderField {
|
||||
return []CommonThirdPartyOrderField{
|
||||
CommonThirdPartyOrderFieldName,
|
||||
CommonThirdPartyOrderFieldCreatedAt,
|
||||
CommonThirdPartyOrderFieldUpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func (v CommonThirdPartyOrderField) IsValid() bool {
|
||||
switch v {
|
||||
case
|
||||
CommonThirdPartyOrderFieldName,
|
||||
CommonThirdPartyOrderFieldCreatedAt,
|
||||
CommonThirdPartyOrderFieldUpdatedAt:
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (v CommonThirdPartyOrderField) String() string {
|
||||
return string(v)
|
||||
}
|
||||
|
||||
func (v CommonThirdPartyOrderField) MarshalText() ([]byte, error) {
|
||||
return []byte(v.String()), nil
|
||||
}
|
||||
|
||||
func (v *CommonThirdPartyOrderField) UnmarshalText(text []byte) error {
|
||||
val := CommonThirdPartyOrderField(text)
|
||||
if !val.IsValid() {
|
||||
return fmt.Errorf("invalid CommonThirdPartyOrderField value: %q", string(text))
|
||||
}
|
||||
|
||||
*v = val
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v CommonThirdPartyOrderField) Column() string {
|
||||
switch v {
|
||||
case CommonThirdPartyOrderFieldName:
|
||||
return "name"
|
||||
case CommonThirdPartyOrderFieldCreatedAt:
|
||||
return "created_at"
|
||||
case CommonThirdPartyOrderFieldUpdatedAt:
|
||||
return "updated_at"
|
||||
}
|
||||
|
||||
panic(fmt.Sprintf("unsupported order by: %s", v))
|
||||
}
|
||||
@@ -18,11 +18,13 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/page"
|
||||
)
|
||||
|
||||
type (
|
||||
@@ -693,3 +695,176 @@ WHERE
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *CommonTrackerPattern) CursorKey(field CommonTrackerPatternOrderField) page.CursorKey {
|
||||
switch field {
|
||||
case CommonTrackerPatternOrderFieldPattern:
|
||||
return page.NewCursorKey(p.ID, p.Pattern)
|
||||
case CommonTrackerPatternOrderFieldConfidence:
|
||||
return page.NewCursorKey(p.ID, p.Confidence)
|
||||
case CommonTrackerPatternOrderFieldCreatedAt:
|
||||
return page.NewCursorKey(p.ID, p.CreatedAt)
|
||||
case CommonTrackerPatternOrderFieldUpdatedAt:
|
||||
return page.NewCursorKey(p.ID, p.UpdatedAt)
|
||||
case CommonTrackerPatternOrderFieldEnrichedAt:
|
||||
if p.EnrichedAt == nil {
|
||||
return page.NewCursorKey(p.ID, time.Time{})
|
||||
}
|
||||
|
||||
return page.NewCursorKey(p.ID, *p.EnrichedAt)
|
||||
}
|
||||
|
||||
panic(fmt.Sprintf("unsupported order by: %s", field))
|
||||
}
|
||||
|
||||
// Load returns a cursor-paginated, filtered page of common tracker
|
||||
// patterns. The catalog is global (no tenant scope). The cursor supplies
|
||||
// the limit and ordering; callers wrap the result with page.NewPage.
|
||||
func (ps *CommonTrackerPatterns) Load(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
cursor *page.Cursor[CommonTrackerPatternOrderField],
|
||||
filter *CommonTrackerPatternFilter,
|
||||
) error {
|
||||
q := `
|
||||
SELECT
|
||||
id,
|
||||
common_third_party_id,
|
||||
tracker_type,
|
||||
pattern,
|
||||
match_type,
|
||||
description,
|
||||
max_age_seconds,
|
||||
confidence,
|
||||
enrichment_requested_at,
|
||||
enriched_at,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM
|
||||
common_tracker_patterns
|
||||
WHERE
|
||||
%s
|
||||
AND %s
|
||||
`
|
||||
|
||||
q = fmt.Sprintf(q, filter.SQLFragment(), cursor.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{}
|
||||
maps.Copy(args, filter.SQLArguments())
|
||||
maps.Copy(args, cursor.SQLArguments())
|
||||
|
||||
rows, err := conn.Query(ctx, q, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot query common tracker patterns: %w", err)
|
||||
}
|
||||
|
||||
patterns, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[CommonTrackerPattern])
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot collect common tracker patterns: %w", err)
|
||||
}
|
||||
|
||||
*ps = patterns
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CountAll returns the number of common tracker patterns matching the
|
||||
// filter, ignoring pagination.
|
||||
func (ps *CommonTrackerPatterns) CountAll(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
filter *CommonTrackerPatternFilter,
|
||||
) (int, error) {
|
||||
q := `
|
||||
SELECT
|
||||
COUNT(id)
|
||||
FROM
|
||||
common_tracker_patterns
|
||||
WHERE
|
||||
%s
|
||||
`
|
||||
|
||||
q = fmt.Sprintf(q, filter.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{}
|
||||
maps.Copy(args, filter.SQLArguments())
|
||||
|
||||
row := conn.QueryRow(ctx, q, args)
|
||||
|
||||
var count int
|
||||
if err := row.Scan(&count); err != nil {
|
||||
return 0, fmt.Errorf("cannot count common tracker patterns: %w", err)
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// LoadAllIDs returns every common tracker pattern id matching the filter,
|
||||
// with no pagination. It backs bulk operations (e.g. operator-driven
|
||||
// re-enrichment) that act on the entire matching set.
|
||||
func (ps *CommonTrackerPatterns) LoadAllIDs(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
filter *CommonTrackerPatternFilter,
|
||||
) ([]gid.GID, error) {
|
||||
q := `
|
||||
SELECT
|
||||
id
|
||||
FROM
|
||||
common_tracker_patterns
|
||||
WHERE
|
||||
%s
|
||||
ORDER BY pattern ASC
|
||||
`
|
||||
|
||||
q = fmt.Sprintf(q, filter.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{}
|
||||
maps.Copy(args, filter.SQLArguments())
|
||||
|
||||
rows, err := conn.Query(ctx, q, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot query common tracker pattern ids: %w", err)
|
||||
}
|
||||
|
||||
ids, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot collect common tracker pattern ids: %w", err)
|
||||
}
|
||||
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// RequestEnrichmentByIDs arms enrichment on the given common tracker
|
||||
// patterns. When resetEnriched is true it also clears enriched_at so rows
|
||||
// that previously reached a terminal state are re-processed. Returns the
|
||||
// number of rows re-queued. This is the async fallback path; the
|
||||
// synchronous enricher service is preferred.
|
||||
func (ps *CommonTrackerPatterns) RequestEnrichmentByIDs(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
ids []gid.GID,
|
||||
resetEnriched bool,
|
||||
) (int64, error) {
|
||||
q := `
|
||||
UPDATE common_tracker_patterns
|
||||
SET
|
||||
enrichment_requested_at = NOW(),
|
||||
enriched_at = CASE WHEN @reset_enriched THEN NULL ELSE enriched_at END,
|
||||
updated_at = NOW()
|
||||
WHERE
|
||||
id = ANY(@ids)
|
||||
`
|
||||
|
||||
args := pgx.StrictNamedArgs{
|
||||
"ids": ids,
|
||||
"reset_enriched": resetEnriched,
|
||||
}
|
||||
|
||||
result, err := tx.Exec(ctx, q, args)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot request common tracker pattern enrichment: %w", err)
|
||||
}
|
||||
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
213
pkg/coredata/common_tracker_pattern_filter.go
Normal file
213
pkg/coredata/common_tracker_pattern_filter.go
Normal file
@@ -0,0 +1,213 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package coredata
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
)
|
||||
|
||||
// CommonTrackerPatternEnrichmentState is a synthetic filter over the
|
||||
// enrichment_requested_at / enriched_at columns. It is not a stored
|
||||
// column; it classifies a row's position in the enrichment lifecycle.
|
||||
type CommonTrackerPatternEnrichmentState string
|
||||
|
||||
const (
|
||||
// CommonTrackerPatternEnrichmentStateQueued: a row armed for the
|
||||
// enrichment worker (enrichment_requested_at IS NOT NULL).
|
||||
CommonTrackerPatternEnrichmentStateQueued CommonTrackerPatternEnrichmentState = "QUEUED"
|
||||
// CommonTrackerPatternEnrichmentStateEnriched: a row whose
|
||||
// enrichment has completed (enriched_at IS NOT NULL) and is not
|
||||
// re-queued.
|
||||
CommonTrackerPatternEnrichmentStateEnriched CommonTrackerPatternEnrichmentState = "ENRICHED"
|
||||
// CommonTrackerPatternEnrichmentStateUnenriched: a row never enriched
|
||||
// and not currently queued.
|
||||
CommonTrackerPatternEnrichmentStateUnenriched CommonTrackerPatternEnrichmentState = "UNENRICHED"
|
||||
)
|
||||
|
||||
func (s CommonTrackerPatternEnrichmentState) IsValid() bool {
|
||||
switch s {
|
||||
case
|
||||
CommonTrackerPatternEnrichmentStateQueued,
|
||||
CommonTrackerPatternEnrichmentStateEnriched,
|
||||
CommonTrackerPatternEnrichmentStateUnenriched:
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (s CommonTrackerPatternEnrichmentState) String() string {
|
||||
return string(s)
|
||||
}
|
||||
|
||||
func (s CommonTrackerPatternEnrichmentState) MarshalText() ([]byte, error) {
|
||||
return []byte(s.String()), nil
|
||||
}
|
||||
|
||||
func (s *CommonTrackerPatternEnrichmentState) UnmarshalText(text []byte) error {
|
||||
val := CommonTrackerPatternEnrichmentState(text)
|
||||
if !val.IsValid() {
|
||||
return fmt.Errorf("invalid CommonTrackerPatternEnrichmentState value: %q", string(text))
|
||||
}
|
||||
|
||||
*s = val
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type CommonTrackerPatternFilter struct {
|
||||
trackerType *TrackerType
|
||||
matchType *TrackerPatternMatchType
|
||||
commonThirdPartyID *gid.GID
|
||||
keyword *string
|
||||
linked *bool
|
||||
state *CommonTrackerPatternEnrichmentState
|
||||
}
|
||||
|
||||
func NewCommonTrackerPatternFilter() *CommonTrackerPatternFilter {
|
||||
return &CommonTrackerPatternFilter{}
|
||||
}
|
||||
|
||||
func (f *CommonTrackerPatternFilter) WithTrackerType(trackerType *TrackerType) *CommonTrackerPatternFilter {
|
||||
f.trackerType = trackerType
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *CommonTrackerPatternFilter) WithMatchType(matchType *TrackerPatternMatchType) *CommonTrackerPatternFilter {
|
||||
f.matchType = matchType
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *CommonTrackerPatternFilter) WithCommonThirdPartyID(id *gid.GID) *CommonTrackerPatternFilter {
|
||||
f.commonThirdPartyID = id
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *CommonTrackerPatternFilter) WithKeyword(keyword *string) *CommonTrackerPatternFilter {
|
||||
f.keyword = keyword
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *CommonTrackerPatternFilter) WithLinked(linked *bool) *CommonTrackerPatternFilter {
|
||||
f.linked = linked
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *CommonTrackerPatternFilter) WithState(state *CommonTrackerPatternEnrichmentState) *CommonTrackerPatternFilter {
|
||||
f.state = state
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *CommonTrackerPatternFilter) SQLFragment() string {
|
||||
if f == nil {
|
||||
return "TRUE"
|
||||
}
|
||||
|
||||
return `
|
||||
(
|
||||
CASE
|
||||
WHEN @filter_tracker_type::text IS NOT NULL THEN
|
||||
tracker_type = @filter_tracker_type::tracker_type
|
||||
ELSE TRUE
|
||||
END
|
||||
AND
|
||||
CASE
|
||||
WHEN @filter_match_type::text IS NOT NULL THEN
|
||||
match_type = @filter_match_type::cookie_pattern_match_type
|
||||
ELSE TRUE
|
||||
END
|
||||
AND
|
||||
CASE
|
||||
WHEN @filter_common_third_party_id::text IS NOT NULL THEN
|
||||
common_third_party_id = @filter_common_third_party_id::text
|
||||
ELSE TRUE
|
||||
END
|
||||
AND
|
||||
CASE
|
||||
WHEN @filter_keyword::text IS NOT NULL AND @filter_keyword::text != '' THEN
|
||||
(pattern ILIKE '%' || @filter_keyword || '%'
|
||||
OR description ILIKE '%' || @filter_keyword || '%')
|
||||
ELSE TRUE
|
||||
END
|
||||
AND
|
||||
CASE
|
||||
WHEN @filter_linked::boolean IS NULL THEN TRUE
|
||||
WHEN @filter_linked::boolean THEN common_third_party_id IS NOT NULL
|
||||
ELSE common_third_party_id IS NULL
|
||||
END
|
||||
AND
|
||||
CASE
|
||||
WHEN @filter_state_queued::boolean THEN enrichment_requested_at IS NOT NULL
|
||||
WHEN @filter_state_enriched::boolean THEN
|
||||
enrichment_requested_at IS NULL AND enriched_at IS NOT NULL
|
||||
WHEN @filter_state_unenriched::boolean THEN
|
||||
enrichment_requested_at IS NULL AND enriched_at IS NULL
|
||||
ELSE TRUE
|
||||
END
|
||||
)`
|
||||
}
|
||||
|
||||
func (f *CommonTrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs {
|
||||
args := pgx.StrictNamedArgs{
|
||||
"filter_tracker_type": nil,
|
||||
"filter_match_type": nil,
|
||||
"filter_common_third_party_id": nil,
|
||||
"filter_keyword": nil,
|
||||
"filter_linked": nil,
|
||||
"filter_state_queued": false,
|
||||
"filter_state_enriched": false,
|
||||
"filter_state_unenriched": false,
|
||||
}
|
||||
|
||||
if f == nil {
|
||||
return args
|
||||
}
|
||||
|
||||
if f.trackerType != nil {
|
||||
args["filter_tracker_type"] = string(*f.trackerType)
|
||||
}
|
||||
|
||||
if f.matchType != nil {
|
||||
args["filter_match_type"] = string(*f.matchType)
|
||||
}
|
||||
|
||||
if f.commonThirdPartyID != nil {
|
||||
args["filter_common_third_party_id"] = *f.commonThirdPartyID
|
||||
}
|
||||
|
||||
if f.keyword != nil {
|
||||
args["filter_keyword"] = *f.keyword
|
||||
}
|
||||
|
||||
if f.linked != nil {
|
||||
args["filter_linked"] = *f.linked
|
||||
}
|
||||
|
||||
if f.state != nil {
|
||||
switch *f.state {
|
||||
case CommonTrackerPatternEnrichmentStateQueued:
|
||||
args["filter_state_queued"] = true
|
||||
case CommonTrackerPatternEnrichmentStateEnriched:
|
||||
args["filter_state_enriched"] = true
|
||||
case CommonTrackerPatternEnrichmentStateUnenriched:
|
||||
args["filter_state_unenriched"] = true
|
||||
}
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
99
pkg/coredata/common_tracker_pattern_order_field.go
Normal file
99
pkg/coredata/common_tracker_pattern_order_field.go
Normal file
@@ -0,0 +1,99 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package coredata
|
||||
|
||||
import (
|
||||
"encoding"
|
||||
"fmt"
|
||||
|
||||
"go.probo.inc/probo/pkg/page"
|
||||
)
|
||||
|
||||
type CommonTrackerPatternOrderField string
|
||||
|
||||
const (
|
||||
CommonTrackerPatternOrderFieldPattern CommonTrackerPatternOrderField = "PATTERN"
|
||||
CommonTrackerPatternOrderFieldConfidence CommonTrackerPatternOrderField = "CONFIDENCE"
|
||||
CommonTrackerPatternOrderFieldCreatedAt CommonTrackerPatternOrderField = "CREATED_AT"
|
||||
CommonTrackerPatternOrderFieldUpdatedAt CommonTrackerPatternOrderField = "UPDATED_AT"
|
||||
CommonTrackerPatternOrderFieldEnrichedAt CommonTrackerPatternOrderField = "ENRICHED_AT"
|
||||
)
|
||||
|
||||
var (
|
||||
_ page.OrderField = CommonTrackerPatternOrderField("")
|
||||
_ fmt.Stringer = CommonTrackerPatternOrderField("")
|
||||
_ encoding.TextMarshaler = CommonTrackerPatternOrderField("")
|
||||
_ encoding.TextUnmarshaler = (*CommonTrackerPatternOrderField)(nil)
|
||||
)
|
||||
|
||||
func CommonTrackerPatternOrderFields() []CommonTrackerPatternOrderField {
|
||||
return []CommonTrackerPatternOrderField{
|
||||
CommonTrackerPatternOrderFieldPattern,
|
||||
CommonTrackerPatternOrderFieldConfidence,
|
||||
CommonTrackerPatternOrderFieldCreatedAt,
|
||||
CommonTrackerPatternOrderFieldUpdatedAt,
|
||||
CommonTrackerPatternOrderFieldEnrichedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func (v CommonTrackerPatternOrderField) IsValid() bool {
|
||||
switch v {
|
||||
case
|
||||
CommonTrackerPatternOrderFieldPattern,
|
||||
CommonTrackerPatternOrderFieldConfidence,
|
||||
CommonTrackerPatternOrderFieldCreatedAt,
|
||||
CommonTrackerPatternOrderFieldUpdatedAt,
|
||||
CommonTrackerPatternOrderFieldEnrichedAt:
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (v CommonTrackerPatternOrderField) String() string {
|
||||
return string(v)
|
||||
}
|
||||
|
||||
func (v CommonTrackerPatternOrderField) MarshalText() ([]byte, error) {
|
||||
return []byte(v.String()), nil
|
||||
}
|
||||
|
||||
func (v *CommonTrackerPatternOrderField) UnmarshalText(text []byte) error {
|
||||
val := CommonTrackerPatternOrderField(text)
|
||||
if !val.IsValid() {
|
||||
return fmt.Errorf("invalid CommonTrackerPatternOrderField value: %q", string(text))
|
||||
}
|
||||
|
||||
*v = val
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v CommonTrackerPatternOrderField) Column() string {
|
||||
switch v {
|
||||
case CommonTrackerPatternOrderFieldPattern:
|
||||
return "pattern"
|
||||
case CommonTrackerPatternOrderFieldConfidence:
|
||||
return "confidence"
|
||||
case CommonTrackerPatternOrderFieldCreatedAt:
|
||||
return "created_at"
|
||||
case CommonTrackerPatternOrderFieldUpdatedAt:
|
||||
return "updated_at"
|
||||
case CommonTrackerPatternOrderFieldEnrichedAt:
|
||||
return "COALESCE(enriched_at, '0001-01-01T00:00:00Z'::timestamptz)"
|
||||
}
|
||||
|
||||
panic(fmt.Sprintf("unsupported order by: %s", v))
|
||||
}
|
||||
@@ -295,6 +295,99 @@ LIMIT @limit;
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// LoadAllByTrackerPatternID returns every detected tracker linked to the
|
||||
// pattern, with no pagination. It backs the banner-reset rebuild, which
|
||||
// recreates exact patterns from a glob's detections.
|
||||
func (dts *DetectedTrackers) LoadAllByTrackerPatternID(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
scope Scoper,
|
||||
trackerPatternID gid.GID,
|
||||
) error {
|
||||
q := `
|
||||
SELECT
|
||||
id,
|
||||
cookie_banner_id,
|
||||
tracker_pattern_id,
|
||||
tracker_type,
|
||||
identifier,
|
||||
max_age_seconds,
|
||||
source,
|
||||
value_size,
|
||||
initiator_url,
|
||||
initiator_domain,
|
||||
last_detected_at,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM
|
||||
detected_trackers
|
||||
WHERE
|
||||
%s
|
||||
AND tracker_pattern_id = @tracker_pattern_id
|
||||
ORDER BY
|
||||
identifier ASC, id ASC
|
||||
`
|
||||
|
||||
q = fmt.Sprintf(q, scope.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{"tracker_pattern_id": trackerPatternID}
|
||||
maps.Copy(args, scope.SQLArguments())
|
||||
|
||||
rows, err := conn.Query(ctx, q, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot query detected trackers: %w", err)
|
||||
}
|
||||
|
||||
trackers, err := pgx.CollectRows(rows, pgx.RowToAddrOfStructByName[DetectedTracker])
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot collect detected trackers: %w", err)
|
||||
}
|
||||
|
||||
*dts = trackers
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateTrackerPatternID repoints a single detected tracker at another
|
||||
// pattern. It is the per-row counterpart of RelinkByTrackerPatternID,
|
||||
// used by the banner-reset rebuild where each detection of a glob moves
|
||||
// to its own recreated exact pattern.
|
||||
func (dt *DetectedTracker) UpdateTrackerPatternID(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
scope Scoper,
|
||||
) error {
|
||||
q := `
|
||||
UPDATE detected_trackers
|
||||
SET
|
||||
tracker_pattern_id = @tracker_pattern_id,
|
||||
updated_at = @updated_at
|
||||
WHERE
|
||||
%s
|
||||
AND id = @id
|
||||
`
|
||||
|
||||
q = fmt.Sprintf(q, scope.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{
|
||||
"id": dt.ID,
|
||||
"tracker_pattern_id": dt.TrackerPatternID,
|
||||
"updated_at": time.Now(),
|
||||
}
|
||||
maps.Copy(args, scope.SQLArguments())
|
||||
|
||||
result, err := tx.Exec(ctx, q, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot update detected tracker pattern: %w", err)
|
||||
}
|
||||
|
||||
if result.RowsAffected() == 0 {
|
||||
return ErrResourceNotFound
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (dts *DetectedTrackers) RelinkByTrackerPatternID(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
|
||||
@@ -1382,3 +1382,122 @@ WHERE
|
||||
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// ResetAndRequestMappingByCookieCategoryID detaches every pattern in the
|
||||
// given category from its catalog row, org third party, and copied
|
||||
// description, then re-arms mapping. Operators run this (via proboctl) on
|
||||
// a banner's uncategorised category to force a clean re-map when
|
||||
// iterating on the mapping agent. Excluded patterns are left untouched -
|
||||
// exclusion is a deliberate suppression. The cookie_category_id key
|
||||
// scopes the reset to the uncategorised category the caller resolves;
|
||||
// the Scoper keeps it tenant-isolated. Returns the number of patterns
|
||||
// reset.
|
||||
func (tps *TrackerPatterns) ResetAndRequestMappingByCookieCategoryID(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
scope Scoper,
|
||||
cookieCategoryID gid.GID,
|
||||
) (int64, error) {
|
||||
q := `
|
||||
UPDATE tracker_patterns
|
||||
SET
|
||||
common_tracker_pattern_id = NULL,
|
||||
third_party_id = NULL,
|
||||
description = '',
|
||||
mapping_requested_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE
|
||||
%s
|
||||
AND cookie_category_id = @cookie_category_id
|
||||
AND excluded = false
|
||||
`
|
||||
|
||||
q = fmt.Sprintf(q, scope.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{"cookie_category_id": cookieCategoryID}
|
||||
maps.Copy(args, scope.SQLArguments())
|
||||
|
||||
result, err := tx.Exec(ctx, q, args)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot reset and request mapping by cookie category: %w", err)
|
||||
}
|
||||
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID returns every
|
||||
// distinct common_tracker_pattern_id referenced by the banner's patterns,
|
||||
// regardless of mapping state. Unlike
|
||||
// LoadDistinctCommonTrackerPatternIDsByCookieBannerID (which restricts to
|
||||
// unmapped patterns for the mapping pipeline), this returns the full set
|
||||
// of catalog rows the banner depends on, so an operator can re-describe
|
||||
// exactly those before a reset.
|
||||
func (tps *TrackerPatterns) LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
scope Scoper,
|
||||
cookieBannerID gid.GID,
|
||||
) ([]gid.GID, error) {
|
||||
q := `
|
||||
SELECT DISTINCT common_tracker_pattern_id
|
||||
FROM tracker_patterns
|
||||
WHERE
|
||||
%s
|
||||
AND cookie_banner_id = @cookie_banner_id
|
||||
AND common_tracker_pattern_id IS NOT NULL
|
||||
`
|
||||
|
||||
q = fmt.Sprintf(q, scope.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{"cookie_banner_id": cookieBannerID}
|
||||
maps.Copy(args, scope.SQLArguments())
|
||||
|
||||
rows, err := conn.Query(ctx, q, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot query linked common tracker pattern ids: %w", err)
|
||||
}
|
||||
|
||||
ids, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot collect linked common tracker pattern ids: %w", err)
|
||||
}
|
||||
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// LoadAllLinkedCommonTrackerPatternIDsByOrganizationID is the org-wide
|
||||
// counterpart of LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID:
|
||||
// every distinct catalog row the organization's tracker patterns depend
|
||||
// on, regardless of mapping state.
|
||||
func (tps *TrackerPatterns) LoadAllLinkedCommonTrackerPatternIDsByOrganizationID(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
scope Scoper,
|
||||
organizationID gid.GID,
|
||||
) ([]gid.GID, error) {
|
||||
q := `
|
||||
SELECT DISTINCT common_tracker_pattern_id
|
||||
FROM tracker_patterns
|
||||
WHERE
|
||||
%s
|
||||
AND organization_id = @organization_id
|
||||
AND common_tracker_pattern_id IS NOT NULL
|
||||
`
|
||||
|
||||
q = fmt.Sprintf(q, scope.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{"organization_id": organizationID}
|
||||
maps.Copy(args, scope.SQLArguments())
|
||||
|
||||
rows, err := conn.Query(ctx, q, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot query linked common tracker pattern ids: %w", err)
|
||||
}
|
||||
|
||||
ids, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot collect linked common tracker pattern ids: %w", err)
|
||||
}
|
||||
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user