rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refine proboctl catalog selection flags and listing

Browse Source 
Require exactly one selection anchor (--id, --linked-banner,
--linked-org, or --common-third-party) for common-tracker-pattern
reenrich, dropping the catch-all --all; the tracker-type, keyword, and
state flags now narrow the anchor's result except when explicit --id
values are given. Add --linked-banner, --linked-org, and a tri-state
--with-common-third-party to the list command, replacing the separate
--linked/--unlinked booleans, and rename --third-party to
--common-third-party across both commands.

Support these by adding an ID restriction to CommonTrackerPatternFilter
so linked-banner/linked-org selections can be intersected with the
remaining filters in a single query.

Memoize the pg client on the proboctl Factory to avoid a duplicate
Prometheus collector registration panic when more than one command
path builds a client. Surface timestamps in both listing tables and
flag enriched-but-undescribed rows in the displayed enrichment state.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-06-08 17:01:56 +02:00
parent d5427a824b
commit 4eb352a61d
6 changed files with 194 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
pkg/coredata/common_tracker_pattern_filter.go
View File

@@ -71,6 +71,7 @@ func (s *CommonTrackerPatternEnrichmentState) UnmarshalText(text []byte) error {
}
type CommonTrackerPatternFilter struct {
ids []gid.GID
trackerType *TrackerType
matchType *TrackerPatternMatchType
commonThirdPartyID *gid.GID
@@ -83,6 +84,13 @@ func NewCommonTrackerPatternFilter() *CommonTrackerPatternFilter {
return &CommonTrackerPatternFilter{}
}
// WithIDs restricts the result to the given pattern IDs. A non-nil but
// empty slice matches nothing.
func (f *CommonTrackerPatternFilter) WithIDs(ids []gid.GID) *CommonTrackerPatternFilter {
f.ids = ids
return f
}
func (f *CommonTrackerPatternFilter) WithTrackerType(trackerType *TrackerType) *CommonTrackerPatternFilter {
f.trackerType = trackerType
return f
@@ -120,6 +128,12 @@ func (f *CommonTrackerPatternFilter) SQLFragment() string {
return `
(
CASE
WHEN @filter_ids::text[] IS NOT NULL THEN
id = ANY(@filter_ids)
ELSE TRUE
END
AND
CASE
WHEN @filter_tracker_type::text IS NOT NULL THEN
tracker_type = @filter_tracker_type::tracker_type
@@ -164,6 +178,7 @@ func (f *CommonTrackerPatternFilter) SQLFragment() string {
func (f *CommonTrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs {
args := pgx.StrictNamedArgs{
"filter_ids": nil,
"filter_tracker_type": nil,
"filter_match_type": nil,
"filter_common_third_party_id": nil,
@@ -178,6 +193,10 @@ func (f *CommonTrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs {
return args
}
if f.ids != nil {
args["filter_ids"] = f.ids
}
if f.trackerType != nil {
args["filter_tracker_type"] = string(*f.trackerType)
}

18
pkg/proboctl/cmdutil/cmdutil.go
View File

@@ -33,14 +33,30 @@ type Factory struct {
Version string
PgDSN string
CfgFile string
pgClient *pg.Client
}
// PgClient returns a shared pg client, building it on first use. The client
// is memoized because pg.NewClient registers Prometheus collectors, so
// constructing it more than once panics with a duplicate registration.
func (f *Factory) PgClient() (*pg.Client, error) {
if f.pgClient != nil {
return f.pgClient, nil
}
if f.PgDSN == "" {
return nil, fmt.Errorf("set --pg-dsn or DATABASE_URL")
}
return pgconn.NewPgClientFromDSN(f.PgDSN)
client, err := pgconn.NewPgClientFromDSN(f.PgDSN)
if err != nil {
return nil, err
}
f.pgClient = client
return f.pgClient, nil
}
// ProbodConfig loads the shared probod configuration file (--cfg-file).

11
pkg/proboctl/commonthirdparty/list.go vendored
View File

@@ -120,9 +120,16 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
return nil
}
table := clicmdutil.NewTable("ID", "NAME", "SLUG", "CATEGORY")
table := clicmdutil.NewTable("ID", "NAME", "SLUG", "CATEGORY", "CREATED", "UPDATED")
for _, p := range parties {
table.Row(p.ID.String(), p.Name, p.Slug, string(p.Category))
table.Row(
p.ID.String(),
p.Name,
p.Slug,
string(p.Category),
p.CreatedAt.Format("2006-01-02 15:04:05"),
p.UpdatedAt.Format("2006-01-02 15:04:05"),
)
}
_, _ = fmt.Fprintln(f.IOStreams.Out, table.Render())

2
pkg/proboctl/commontrackerpattern/commontrackerpattern.go
View File

@@ -49,6 +49,8 @@ func enrichmentState(p *coredata.CommonTrackerPattern) string {
switch {
case p.EnrichmentRequestedAt != nil:
return "queued"
case p.EnrichedAt != nil && p.Description == "":
return "enriched (no description)"
case p.EnrichedAt != nil:
return "enriched"
default:

83
pkg/proboctl/commontrackerpattern/list.go
View File

@@ -31,11 +31,12 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
var (
flagTrackerType string
flagMatchType string
flagThirdParty string
flagCommonThirdParty string
flagLinkedBanner string
flagLinkedOrg string
flagKeyword string
flagState string
flagLinked bool
flagUnlinked bool
flagWithCommonThirdParty bool
flagSort string
flagOrder string
flagLimit int
@@ -51,11 +52,12 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Filter by tracker type (COOKIE, LOCAL_STORAGE, SESSION_STORAGE, INDEXED_DB)")
cmd.Flags().StringVar(&flagMatchType, "match-type", "", "Filter by match type (EXACT, GLOB, PREFIX)")
cmd.Flags().StringVar(&flagThirdParty, "third-party", "", "Filter by linked common third party (slug or GID)")
cmd.Flags().StringVar(&flagCommonThirdParty, "common-third-party", "", "Filter by linked common third party (slug or GID)")
cmd.Flags().StringVar(&flagLinkedBanner, "linked-banner", "", "Filter to catalog rows linked to a cookie banner's patterns (GID)")
cmd.Flags().StringVar(&flagLinkedOrg, "linked-org", "", "Filter to catalog rows linked to an organization's patterns (GID)")
cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter by pattern/description substring")
cmd.Flags().StringVar(&flagState, "state", "", "Filter by enrichment state (queued, enriched, unenriched)")
cmd.Flags().BoolVar(&flagLinked, "linked", false, "Only patterns linked to a common third party")
cmd.Flags().BoolVar(&flagUnlinked, "unlinked", false, "Only patterns not linked to a common third party")
cmd.Flags().BoolVar(&flagWithCommonThirdParty, "with-common-third-party", false, "Filter by whether the pattern is linked to a common third party (true/false); ignored when not set")
cmd.Flags().StringVar(&flagSort, "sort", "confidence", "Sort field: pattern, confidence, created, updated, enriched")
cmd.Flags().StringVar(&flagOrder, "order", "", "Sort order: asc, desc (default depends on field)")
cmd.Flags().IntVarP(&flagLimit, "limit", "L", 50, "Maximum rows to return (0 for all)")
@@ -65,8 +67,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
return err
}
if flagLinked && flagUnlinked {
return fmt.Errorf("--linked and --unlinked are mutually exclusive")
if flagLinkedBanner != "" && flagLinkedOrg != "" {
return fmt.Errorf("--linked-banner and --linked-org are mutually exclusive")
}
orderBy, err := parseOrderBy(flagSort, flagOrder)
@@ -74,7 +76,12 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
return err
}
filter, err := buildListFilter(flagTrackerType, flagMatchType, flagKeyword, flagState, flagLinked, flagUnlinked)
var withCommonThirdParty *bool
if cmd.Flags().Changed("with-common-third-party") {
withCommonThirdParty = &flagWithCommonThirdParty
}
filter, err := buildListFilter(flagTrackerType, flagMatchType, flagKeyword, flagState, withCommonThirdParty)
if err != nil {
return err
}
@@ -91,8 +98,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
if err := pgClient.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
if flagThirdParty != "" {
id, err := resolveCommonThirdPartyID(ctx, conn, flagThirdParty)
if flagCommonThirdParty != "" {
id, err := resolveCommonThirdPartyID(ctx, conn, flagCommonThirdParty)
if err != nil {
return err
}
@@ -100,6 +107,45 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
filter.WithCommonThirdPartyID(&id)
}
switch {
case flagLinkedBanner != "":
bannerID, err := gid.ParseGID(flagLinkedBanner)
if err != nil {
return fmt.Errorf("invalid --linked-banner GID %q: %w", flagLinkedBanner, err)
}
var tps coredata.TrackerPatterns
linkedIDs, err := tps.LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID(ctx, conn, coredata.NewScopeFromObjectID(bannerID), bannerID)
if err != nil {
return err
}
if len(linkedIDs) == 0 {
return nil
}
filter.WithIDs(linkedIDs)
case flagLinkedOrg != "":
orgID, err := gid.ParseGID(flagLinkedOrg)
if err != nil {
return fmt.Errorf("invalid --linked-org GID %q: %w", flagLinkedOrg, err)
}
var tps coredata.TrackerPatterns
linkedIDs, err := tps.LoadAllLinkedCommonTrackerPatternIDsByOrganizationID(ctx, conn, coredata.NewScopeFromObjectID(orgID), orgID)
if err != nil {
return err
}
if len(linkedIDs) == 0 {
return nil
}
filter.WithIDs(linkedIDs)
}
rows, err := cmdutil.Paginate(
ctx,
orderBy,
@@ -168,7 +214,7 @@ func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredat
return err
}
table := clicmdutil.NewTable("ID", "TYPE", "MATCH", "PATTERN", "CONF", "STATE", "THIRD PARTY")
table := clicmdutil.NewTable("ID", "TYPE", "MATCH", "PATTERN", "CONF", "STATE", "THIRD PARTY", "CREATED", "UPDATED")
for _, p := range patterns {
thirdParty := ""
@@ -184,6 +230,8 @@ func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredat
fmt.Sprintf("%.2f", p.Confidence),
enrichmentState(p),
thirdParty,
p.CreatedAt.Format("2006-01-02 15:04:05"),
p.UpdatedAt.Format("2006-01-02 15:04:05"),
)
}
@@ -239,7 +287,7 @@ func parseOrderBy(sort, order string) (page.OrderBy[coredata.CommonTrackerPatter
func buildListFilter(
trackerType, matchType, keyword, state string,
linked, unlinked bool,
withCommonThirdParty *bool,
) (*coredata.CommonTrackerPatternFilter, error) {
filter := coredata.NewCommonTrackerPatternFilter()
@@ -274,13 +322,8 @@ func buildListFilter(
filter.WithState(&st)
}
switch {
case linked:
v := true
filter.WithLinked(&v)
case unlinked:
v := false
filter.WithLinked(&v)
if withCommonThirdParty != nil {
filter.WithLinked(withCommonThirdParty)
}
return filter, nil

117
pkg/proboctl/commontrackerpattern/reenrich.go
View File

@@ -31,13 +31,12 @@ import (
func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
var (
flagIDs []string
flagThirdParty string
flagLinkedBanner string
flagLinkedOrg string
flagCommonThirdParty string
flagTrackerType string
flagKeyword string
flagState string
flagAll bool
flagLinkedBanner string
flagLinkedOrg string
flagConcurrency int
flagResetEnriched bool
flagDryRun bool
@@ -57,13 +56,12 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
}
cmd.Flags().StringSliceVar(&flagIDs, "id", nil, "Common tracker pattern GID(s) to re-enrich (repeatable)")
cmd.Flags().StringVar(&flagThirdParty, "third-party", "", "Select patterns linked to a common third party (slug or GID)")
cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Select patterns of a tracker type")
cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Select patterns matching a pattern/description substring")
cmd.Flags().StringVar(&flagState, "state", "", "Select by enrichment state (queued, enriched, unenriched)")
cmd.Flags().BoolVar(&flagAll, "all", false, "Select every common tracker pattern")
cmd.Flags().StringVar(&flagLinkedBanner, "linked-banner", "", "Select catalog rows linked to a cookie banner's patterns (GID)")
cmd.Flags().StringVar(&flagLinkedOrg, "linked-org", "", "Select catalog rows linked to an organization's patterns (GID)")
cmd.Flags().StringVar(&flagCommonThirdParty, "common-third-party", "", "Select patterns linked to a common third party (slug or GID)")
cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Filter selected patterns by tracker type")
cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter selected patterns by a pattern/description substring")
cmd.Flags().StringVar(&flagState, "state", "", "Filter selected patterns by enrichment state (queued, enriched, unenriched)")
cmd.Flags().IntVar(&flagConcurrency, "concurrency", 4, "Number of patterns to enrich in parallel (sync mode)")
cmd.Flags().BoolVar(&flagResetEnriched, "reset-enriched", true, "Clear enriched_at so terminal rows are re-processed")
cmd.Flags().BoolVar(&flagDryRun, "dry-run", false, "Print the selected patterns without enriching")
@@ -82,13 +80,12 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
ctx,
pgClient,
flagIDs,
flagThirdParty,
flagLinkedBanner,
flagLinkedOrg,
flagCommonThirdParty,
flagTrackerType,
flagKeyword,
flagState,
flagAll,
flagLinkedBanner,
flagLinkedOrg,
)
if err != nil {
return err
@@ -159,14 +156,35 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
return cmd
}
// resolveReenrichIDs turns the selection flags into the set of common
// tracker pattern IDs to re-enrich. Exactly one selection anchor must be
// provided: --id, --linked-banner, --linked-org, or --common-third-party.
// The --tracker-type, --keyword, and --state flags further narrow the
// anchor's result, except with --id, where the listed patterns are used
// verbatim.
func resolveReenrichIDs(
ctx context.Context,
pgClient *pg.Client,
rawIDs []string,
thirdParty, trackerType, keyword, state string,
all bool,
linkedBanner, linkedOrg string,
linkedBanner, linkedOrg, commonThirdParty string,
trackerType, keyword, state string,
) ([]gid.GID, error) {
anchors := 0
for _, set := range []bool{len(rawIDs) > 0, linkedBanner != "", linkedOrg != "", commonThirdParty != ""} {
if set {
anchors++
}
}
switch {
case anchors == 0:
return nil, fmt.Errorf("specify exactly one selection anchor: --id, --linked-banner, --linked-org, or --common-third-party")
case anchors > 1:
return nil, fmt.Errorf("--id, --linked-banner, --linked-org, and --common-third-party are mutually exclusive")
}
// --id selects patterns verbatim; the filtering flags do not apply.
if len(rawIDs) > 0 {
ids := make([]gid.GID, 0, len(rawIDs))
@@ -187,6 +205,11 @@ func resolveReenrichIDs(
err := pgClient.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
filter, err := buildReenrichFilter(trackerType, keyword, state)
if err != nil {
return err
}
switch {
case linkedBanner != "":
bannerID, err := gid.ParseGID(linkedBanner)
@@ -196,9 +219,16 @@ func resolveReenrichIDs(
var tps coredata.TrackerPatterns
ids, err = tps.LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID(ctx, conn, coredata.NewScopeFromObjectID(bannerID), bannerID)
linkedIDs, err := tps.LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID(ctx, conn, coredata.NewScopeFromObjectID(bannerID), bannerID)
if err != nil {
return err
}
if len(linkedIDs) == 0 {
return nil
}
filter.WithIDs(linkedIDs)
case linkedOrg != "":
orgID, err := gid.ParseGID(linkedOrg)
if err != nil {
@@ -207,17 +237,23 @@ func resolveReenrichIDs(
var tps coredata.TrackerPatterns
ids, err = tps.LoadAllLinkedCommonTrackerPatternIDsByOrganizationID(ctx, conn, coredata.NewScopeFromObjectID(orgID), orgID)
return err
default:
filter, hasSelector, err := buildReenrichFilter(ctx, conn, thirdParty, trackerType, keyword, state)
linkedIDs, err := tps.LoadAllLinkedCommonTrackerPatternIDsByOrganizationID(ctx, conn, coredata.NewScopeFromObjectID(orgID), orgID)
if err != nil {
return err
}
if !hasSelector && !all {
return fmt.Errorf("specify a selector (--id, --third-party, --tracker-type, --state, --keyword, --linked-banner, --linked-org) or --all")
if len(linkedIDs) == 0 {
return nil
}
filter.WithIDs(linkedIDs)
case commonThirdParty != "":
thirdPartyID, err := resolveCommonThirdPartyID(ctx, conn, commonThirdParty)
if err != nil {
return err
}
filter.WithCommonThirdPartyID(&thirdPartyID)
}
var ps coredata.CommonTrackerPatterns
@@ -225,7 +261,6 @@ func resolveReenrichIDs(
ids, err = ps.LoadAllIDs(ctx, conn, filter)
return err
}
},
)
if err != nil {
@@ -235,54 +270,32 @@ func resolveReenrichIDs(
return ids, nil
}
func buildReenrichFilter(
ctx context.Context,
conn pg.Querier,
thirdParty, trackerType, keyword, state string,
) (*coredata.CommonTrackerPatternFilter, bool, error) {
func buildReenrichFilter(trackerType, keyword, state string) (*coredata.CommonTrackerPatternFilter, error) {
filter := coredata.NewCommonTrackerPatternFilter()
hasSelector := false
if thirdParty != "" {
id, err := resolveCommonThirdPartyID(ctx, conn, thirdParty)
if err != nil {
return nil, false, err
}
filter.WithCommonThirdPartyID(&id)
hasSelector = true
}
if trackerType != "" {
tt := coredata.TrackerType(trackerType)
if !tt.IsValid() {
return nil, false, fmt.Errorf("invalid --tracker-type value %q", trackerType)
return nil, fmt.Errorf("invalid --tracker-type value %q", trackerType)
}
filter.WithTrackerType(&tt)
hasSelector = true
}
if keyword != "" {
filter.WithKeyword(&keyword)
hasSelector = true
}
if state != "" {
st, err := parseEnrichmentState(state)
if err != nil {
return nil, false, err
return nil, err
}
filter.WithState(&st)
hasSelector = true
}
return filter, hasSelector, nil
return filter, nil
}
func printSample(out io.Writer, ids []gid.GID) {