diff --git a/pkg/coredata/common_tracker_pattern_filter.go b/pkg/coredata/common_tracker_pattern_filter.go index dbfd4f611..cd9584c92 100644 --- a/pkg/coredata/common_tracker_pattern_filter.go +++ b/pkg/coredata/common_tracker_pattern_filter.go @@ -71,6 +71,7 @@ func (s *CommonTrackerPatternEnrichmentState) UnmarshalText(text []byte) error { } type CommonTrackerPatternFilter struct { + ids []gid.GID trackerType *TrackerType matchType *TrackerPatternMatchType commonThirdPartyID *gid.GID @@ -83,6 +84,13 @@ func NewCommonTrackerPatternFilter() *CommonTrackerPatternFilter { return &CommonTrackerPatternFilter{} } +// WithIDs restricts the result to the given pattern IDs. A non-nil but +// empty slice matches nothing. +func (f *CommonTrackerPatternFilter) WithIDs(ids []gid.GID) *CommonTrackerPatternFilter { + f.ids = ids + return f +} + func (f *CommonTrackerPatternFilter) WithTrackerType(trackerType *TrackerType) *CommonTrackerPatternFilter { f.trackerType = trackerType return f @@ -120,6 +128,12 @@ func (f *CommonTrackerPatternFilter) SQLFragment() string { return ` ( + CASE + WHEN @filter_ids::text[] IS NOT NULL THEN + id = ANY(@filter_ids) + ELSE TRUE + END + AND CASE WHEN @filter_tracker_type::text IS NOT NULL THEN tracker_type = @filter_tracker_type::tracker_type @@ -164,6 +178,7 @@ func (f *CommonTrackerPatternFilter) SQLFragment() string { func (f *CommonTrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs { args := pgx.StrictNamedArgs{ + "filter_ids": nil, "filter_tracker_type": nil, "filter_match_type": nil, "filter_common_third_party_id": nil, @@ -178,6 +193,10 @@ func (f *CommonTrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs { return args } + if f.ids != nil { + args["filter_ids"] = f.ids + } + if f.trackerType != nil { args["filter_tracker_type"] = string(*f.trackerType) } diff --git a/pkg/proboctl/cmdutil/cmdutil.go b/pkg/proboctl/cmdutil/cmdutil.go index 2b54df6b6..482ce1a57 100644 --- a/pkg/proboctl/cmdutil/cmdutil.go +++ b/pkg/proboctl/cmdutil/cmdutil.go @@ -33,14 +33,30 @@ type Factory struct { Version string PgDSN string CfgFile string + + pgClient *pg.Client } +// PgClient returns a shared pg client, building it on first use. The client +// is memoized because pg.NewClient registers Prometheus collectors, so +// constructing it more than once panics with a duplicate registration. func (f *Factory) PgClient() (*pg.Client, error) { + if f.pgClient != nil { + return f.pgClient, nil + } + if f.PgDSN == "" { return nil, fmt.Errorf("set --pg-dsn or DATABASE_URL") } - return pgconn.NewPgClientFromDSN(f.PgDSN) + client, err := pgconn.NewPgClientFromDSN(f.PgDSN) + if err != nil { + return nil, err + } + + f.pgClient = client + + return f.pgClient, nil } // ProbodConfig loads the shared probod configuration file (--cfg-file). diff --git a/pkg/proboctl/commonthirdparty/list.go b/pkg/proboctl/commonthirdparty/list.go index 4572bfc39..c240c18da 100644 --- a/pkg/proboctl/commonthirdparty/list.go +++ b/pkg/proboctl/commonthirdparty/list.go @@ -120,9 +120,16 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { return nil } - table := clicmdutil.NewTable("ID", "NAME", "SLUG", "CATEGORY") + table := clicmdutil.NewTable("ID", "NAME", "SLUG", "CATEGORY", "CREATED", "UPDATED") for _, p := range parties { - table.Row(p.ID.String(), p.Name, p.Slug, string(p.Category)) + table.Row( + p.ID.String(), + p.Name, + p.Slug, + string(p.Category), + p.CreatedAt.Format("2006-01-02 15:04:05"), + p.UpdatedAt.Format("2006-01-02 15:04:05"), + ) } _, _ = fmt.Fprintln(f.IOStreams.Out, table.Render()) diff --git a/pkg/proboctl/commontrackerpattern/commontrackerpattern.go b/pkg/proboctl/commontrackerpattern/commontrackerpattern.go index f68574c40..1dd25def3 100644 --- a/pkg/proboctl/commontrackerpattern/commontrackerpattern.go +++ b/pkg/proboctl/commontrackerpattern/commontrackerpattern.go @@ -49,6 +49,8 @@ func enrichmentState(p *coredata.CommonTrackerPattern) string { switch { case p.EnrichmentRequestedAt != nil: return "queued" + case p.EnrichedAt != nil && p.Description == "": + return "enriched (no description)" case p.EnrichedAt != nil: return "enriched" default: diff --git a/pkg/proboctl/commontrackerpattern/list.go b/pkg/proboctl/commontrackerpattern/list.go index 7cd68cdc5..812224239 100644 --- a/pkg/proboctl/commontrackerpattern/list.go +++ b/pkg/proboctl/commontrackerpattern/list.go @@ -29,16 +29,17 @@ import ( func newCmdList(f *cmdutil.Factory) *cobra.Command { var ( - flagTrackerType string - flagMatchType string - flagThirdParty string - flagKeyword string - flagState string - flagLinked bool - flagUnlinked bool - flagSort string - flagOrder string - flagLimit int + flagTrackerType string + flagMatchType string + flagCommonThirdParty string + flagLinkedBanner string + flagLinkedOrg string + flagKeyword string + flagState string + flagWithCommonThirdParty bool + flagSort string + flagOrder string + flagLimit int ) cmd := &cobra.Command{ @@ -51,11 +52,12 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Filter by tracker type (COOKIE, LOCAL_STORAGE, SESSION_STORAGE, INDEXED_DB)") cmd.Flags().StringVar(&flagMatchType, "match-type", "", "Filter by match type (EXACT, GLOB, PREFIX)") - cmd.Flags().StringVar(&flagThirdParty, "third-party", "", "Filter by linked common third party (slug or GID)") + cmd.Flags().StringVar(&flagCommonThirdParty, "common-third-party", "", "Filter by linked common third party (slug or GID)") + cmd.Flags().StringVar(&flagLinkedBanner, "linked-banner", "", "Filter to catalog rows linked to a cookie banner's patterns (GID)") + cmd.Flags().StringVar(&flagLinkedOrg, "linked-org", "", "Filter to catalog rows linked to an organization's patterns (GID)") cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter by pattern/description substring") cmd.Flags().StringVar(&flagState, "state", "", "Filter by enrichment state (queued, enriched, unenriched)") - cmd.Flags().BoolVar(&flagLinked, "linked", false, "Only patterns linked to a common third party") - cmd.Flags().BoolVar(&flagUnlinked, "unlinked", false, "Only patterns not linked to a common third party") + cmd.Flags().BoolVar(&flagWithCommonThirdParty, "with-common-third-party", false, "Filter by whether the pattern is linked to a common third party (true/false); ignored when not set") cmd.Flags().StringVar(&flagSort, "sort", "confidence", "Sort field: pattern, confidence, created, updated, enriched") cmd.Flags().StringVar(&flagOrder, "order", "", "Sort order: asc, desc (default depends on field)") cmd.Flags().IntVarP(&flagLimit, "limit", "L", 50, "Maximum rows to return (0 for all)") @@ -65,8 +67,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { return err } - if flagLinked && flagUnlinked { - return fmt.Errorf("--linked and --unlinked are mutually exclusive") + if flagLinkedBanner != "" && flagLinkedOrg != "" { + return fmt.Errorf("--linked-banner and --linked-org are mutually exclusive") } orderBy, err := parseOrderBy(flagSort, flagOrder) @@ -74,7 +76,12 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { return err } - filter, err := buildListFilter(flagTrackerType, flagMatchType, flagKeyword, flagState, flagLinked, flagUnlinked) + var withCommonThirdParty *bool + if cmd.Flags().Changed("with-common-third-party") { + withCommonThirdParty = &flagWithCommonThirdParty + } + + filter, err := buildListFilter(flagTrackerType, flagMatchType, flagKeyword, flagState, withCommonThirdParty) if err != nil { return err } @@ -91,8 +98,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { if err := pgClient.WithConn( ctx, func(ctx context.Context, conn pg.Querier) error { - if flagThirdParty != "" { - id, err := resolveCommonThirdPartyID(ctx, conn, flagThirdParty) + if flagCommonThirdParty != "" { + id, err := resolveCommonThirdPartyID(ctx, conn, flagCommonThirdParty) if err != nil { return err } @@ -100,6 +107,45 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { filter.WithCommonThirdPartyID(&id) } + switch { + case flagLinkedBanner != "": + bannerID, err := gid.ParseGID(flagLinkedBanner) + if err != nil { + return fmt.Errorf("invalid --linked-banner GID %q: %w", flagLinkedBanner, err) + } + + var tps coredata.TrackerPatterns + + linkedIDs, err := tps.LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID(ctx, conn, coredata.NewScopeFromObjectID(bannerID), bannerID) + if err != nil { + return err + } + + if len(linkedIDs) == 0 { + return nil + } + + filter.WithIDs(linkedIDs) + case flagLinkedOrg != "": + orgID, err := gid.ParseGID(flagLinkedOrg) + if err != nil { + return fmt.Errorf("invalid --linked-org GID %q: %w", flagLinkedOrg, err) + } + + var tps coredata.TrackerPatterns + + linkedIDs, err := tps.LoadAllLinkedCommonTrackerPatternIDsByOrganizationID(ctx, conn, coredata.NewScopeFromObjectID(orgID), orgID) + if err != nil { + return err + } + + if len(linkedIDs) == 0 { + return nil + } + + filter.WithIDs(linkedIDs) + } + rows, err := cmdutil.Paginate( ctx, orderBy, @@ -168,7 +214,7 @@ func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredat return err } - table := clicmdutil.NewTable("ID", "TYPE", "MATCH", "PATTERN", "CONF", "STATE", "THIRD PARTY") + table := clicmdutil.NewTable("ID", "TYPE", "MATCH", "PATTERN", "CONF", "STATE", "THIRD PARTY", "CREATED", "UPDATED") for _, p := range patterns { thirdParty := "" @@ -184,6 +230,8 @@ func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredat fmt.Sprintf("%.2f", p.Confidence), enrichmentState(p), thirdParty, + p.CreatedAt.Format("2006-01-02 15:04:05"), + p.UpdatedAt.Format("2006-01-02 15:04:05"), ) } @@ -239,7 +287,7 @@ func parseOrderBy(sort, order string) (page.OrderBy[coredata.CommonTrackerPatter func buildListFilter( trackerType, matchType, keyword, state string, - linked, unlinked bool, + withCommonThirdParty *bool, ) (*coredata.CommonTrackerPatternFilter, error) { filter := coredata.NewCommonTrackerPatternFilter() @@ -274,13 +322,8 @@ func buildListFilter( filter.WithState(&st) } - switch { - case linked: - v := true - filter.WithLinked(&v) - case unlinked: - v := false - filter.WithLinked(&v) + if withCommonThirdParty != nil { + filter.WithLinked(withCommonThirdParty) } return filter, nil diff --git a/pkg/proboctl/commontrackerpattern/reenrich.go b/pkg/proboctl/commontrackerpattern/reenrich.go index 6629d3935..ce86d0489 100644 --- a/pkg/proboctl/commontrackerpattern/reenrich.go +++ b/pkg/proboctl/commontrackerpattern/reenrich.go @@ -30,19 +30,18 @@ import ( func newCmdReenrich(f *cmdutil.Factory) *cobra.Command { var ( - flagIDs []string - flagThirdParty string - flagTrackerType string - flagKeyword string - flagState string - flagAll bool - flagLinkedBanner string - flagLinkedOrg string - flagConcurrency int - flagResetEnriched bool - flagDryRun bool - flagYes bool - flagEnqueue bool + flagIDs []string + flagLinkedBanner string + flagLinkedOrg string + flagCommonThirdParty string + flagTrackerType string + flagKeyword string + flagState string + flagConcurrency int + flagResetEnriched bool + flagDryRun bool + flagYes bool + flagEnqueue bool ) cmd := &cobra.Command{ @@ -57,13 +56,12 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command { } cmd.Flags().StringSliceVar(&flagIDs, "id", nil, "Common tracker pattern GID(s) to re-enrich (repeatable)") - cmd.Flags().StringVar(&flagThirdParty, "third-party", "", "Select patterns linked to a common third party (slug or GID)") - cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Select patterns of a tracker type") - cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Select patterns matching a pattern/description substring") - cmd.Flags().StringVar(&flagState, "state", "", "Select by enrichment state (queued, enriched, unenriched)") - cmd.Flags().BoolVar(&flagAll, "all", false, "Select every common tracker pattern") cmd.Flags().StringVar(&flagLinkedBanner, "linked-banner", "", "Select catalog rows linked to a cookie banner's patterns (GID)") cmd.Flags().StringVar(&flagLinkedOrg, "linked-org", "", "Select catalog rows linked to an organization's patterns (GID)") + cmd.Flags().StringVar(&flagCommonThirdParty, "common-third-party", "", "Select patterns linked to a common third party (slug or GID)") + cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Filter selected patterns by tracker type") + cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter selected patterns by a pattern/description substring") + cmd.Flags().StringVar(&flagState, "state", "", "Filter selected patterns by enrichment state (queued, enriched, unenriched)") cmd.Flags().IntVar(&flagConcurrency, "concurrency", 4, "Number of patterns to enrich in parallel (sync mode)") cmd.Flags().BoolVar(&flagResetEnriched, "reset-enriched", true, "Clear enriched_at so terminal rows are re-processed") cmd.Flags().BoolVar(&flagDryRun, "dry-run", false, "Print the selected patterns without enriching") @@ -82,13 +80,12 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command { ctx, pgClient, flagIDs, - flagThirdParty, + flagLinkedBanner, + flagLinkedOrg, + flagCommonThirdParty, flagTrackerType, flagKeyword, flagState, - flagAll, - flagLinkedBanner, - flagLinkedOrg, ) if err != nil { return err @@ -159,14 +156,35 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command { return cmd } +// resolveReenrichIDs turns the selection flags into the set of common +// tracker pattern IDs to re-enrich. Exactly one selection anchor must be +// provided: --id, --linked-banner, --linked-org, or --common-third-party. +// The --tracker-type, --keyword, and --state flags further narrow the +// anchor's result, except with --id, where the listed patterns are used +// verbatim. func resolveReenrichIDs( ctx context.Context, pgClient *pg.Client, rawIDs []string, - thirdParty, trackerType, keyword, state string, - all bool, - linkedBanner, linkedOrg string, + linkedBanner, linkedOrg, commonThirdParty string, + trackerType, keyword, state string, ) ([]gid.GID, error) { + anchors := 0 + + for _, set := range []bool{len(rawIDs) > 0, linkedBanner != "", linkedOrg != "", commonThirdParty != ""} { + if set { + anchors++ + } + } + + switch { + case anchors == 0: + return nil, fmt.Errorf("specify exactly one selection anchor: --id, --linked-banner, --linked-org, or --common-third-party") + case anchors > 1: + return nil, fmt.Errorf("--id, --linked-banner, --linked-org, and --common-third-party are mutually exclusive") + } + + // --id selects patterns verbatim; the filtering flags do not apply. if len(rawIDs) > 0 { ids := make([]gid.GID, 0, len(rawIDs)) @@ -187,6 +205,11 @@ func resolveReenrichIDs( err := pgClient.WithConn( ctx, func(ctx context.Context, conn pg.Querier) error { + filter, err := buildReenrichFilter(trackerType, keyword, state) + if err != nil { + return err + } + switch { case linkedBanner != "": bannerID, err := gid.ParseGID(linkedBanner) @@ -196,9 +219,16 @@ func resolveReenrichIDs( var tps coredata.TrackerPatterns - ids, err = tps.LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID(ctx, conn, coredata.NewScopeFromObjectID(bannerID), bannerID) + linkedIDs, err := tps.LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID(ctx, conn, coredata.NewScopeFromObjectID(bannerID), bannerID) + if err != nil { + return err + } - return err + if len(linkedIDs) == 0 { + return nil + } + + filter.WithIDs(linkedIDs) case linkedOrg != "": orgID, err := gid.ParseGID(linkedOrg) if err != nil { @@ -207,25 +237,30 @@ func resolveReenrichIDs( var tps coredata.TrackerPatterns - ids, err = tps.LoadAllLinkedCommonTrackerPatternIDsByOrganizationID(ctx, conn, coredata.NewScopeFromObjectID(orgID), orgID) - - return err - default: - filter, hasSelector, err := buildReenrichFilter(ctx, conn, thirdParty, trackerType, keyword, state) + linkedIDs, err := tps.LoadAllLinkedCommonTrackerPatternIDsByOrganizationID(ctx, conn, coredata.NewScopeFromObjectID(orgID), orgID) if err != nil { return err } - if !hasSelector && !all { - return fmt.Errorf("specify a selector (--id, --third-party, --tracker-type, --state, --keyword, --linked-banner, --linked-org) or --all") + if len(linkedIDs) == 0 { + return nil } - var ps coredata.CommonTrackerPatterns + filter.WithIDs(linkedIDs) + case commonThirdParty != "": + thirdPartyID, err := resolveCommonThirdPartyID(ctx, conn, commonThirdParty) + if err != nil { + return err + } - ids, err = ps.LoadAllIDs(ctx, conn, filter) - - return err + filter.WithCommonThirdPartyID(&thirdPartyID) } + + var ps coredata.CommonTrackerPatterns + + ids, err = ps.LoadAllIDs(ctx, conn, filter) + + return err }, ) if err != nil { @@ -235,54 +270,32 @@ func resolveReenrichIDs( return ids, nil } -func buildReenrichFilter( - ctx context.Context, - conn pg.Querier, - thirdParty, trackerType, keyword, state string, -) (*coredata.CommonTrackerPatternFilter, bool, error) { +func buildReenrichFilter(trackerType, keyword, state string) (*coredata.CommonTrackerPatternFilter, error) { filter := coredata.NewCommonTrackerPatternFilter() - hasSelector := false - - if thirdParty != "" { - id, err := resolveCommonThirdPartyID(ctx, conn, thirdParty) - if err != nil { - return nil, false, err - } - - filter.WithCommonThirdPartyID(&id) - - hasSelector = true - } if trackerType != "" { tt := coredata.TrackerType(trackerType) if !tt.IsValid() { - return nil, false, fmt.Errorf("invalid --tracker-type value %q", trackerType) + return nil, fmt.Errorf("invalid --tracker-type value %q", trackerType) } filter.WithTrackerType(&tt) - - hasSelector = true } if keyword != "" { filter.WithKeyword(&keyword) - - hasSelector = true } if state != "" { st, err := parseEnrichmentState(state) if err != nil { - return nil, false, err + return nil, err } filter.WithState(&st) - - hasSelector = true } - return filter, hasSelector, nil + return filter, nil } func printSample(out io.Writer, ids []gid.GID) {