rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refine proboctl catalog selection flags and listing

Browse Source 
Require exactly one selection anchor (--id, --linked-banner,
--linked-org, or --common-third-party) for common-tracker-pattern
reenrich, dropping the catch-all --all; the tracker-type, keyword, and
state flags now narrow the anchor's result except when explicit --id
values are given. Add --linked-banner, --linked-org, and a tri-state
--with-common-third-party to the list command, replacing the separate
--linked/--unlinked booleans, and rename --third-party to
--common-third-party across both commands.

Support these by adding an ID restriction to CommonTrackerPatternFilter
so linked-banner/linked-org selections can be intersected with the
remaining filters in a single query.

Memoize the pg client on the proboctl Factory to avoid a duplicate
Prometheus collector registration panic when more than one command
path builds a client. Surface timestamps in both listing tables and
flag enriched-but-undescribed rows in the displayed enrichment state.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-06-08 17:01:56 +02:00
parent d5427a824b
commit 4eb352a61d
6 changed files with 194 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
pkg/coredata/common_tracker_pattern_filter.go
View File

@@ -71,6 +71,7 @@ func (s *CommonTrackerPatternEnrichmentState) UnmarshalText(text []byte) error {
}
type CommonTrackerPatternFilter struct {
ids []gid.GID
trackerType *TrackerType
matchType *TrackerPatternMatchType
commonThirdPartyID *gid.GID
@@ -83,6 +84,13 @@ func NewCommonTrackerPatternFilter() *CommonTrackerPatternFilter {
return &CommonTrackerPatternFilter{}
}
// WithIDs restricts the result to the given pattern IDs. A non-nil but
// empty slice matches nothing.
func (f *CommonTrackerPatternFilter) WithIDs(ids []gid.GID) *CommonTrackerPatternFilter {
f.ids = ids
return f
}
func (f *CommonTrackerPatternFilter) WithTrackerType(trackerType *TrackerType) *CommonTrackerPatternFilter {
f.trackerType = trackerType
return f
@@ -120,6 +128,12 @@ func (f *CommonTrackerPatternFilter) SQLFragment() string {
return `
(
CASE
WHEN @filter_ids::text[] IS NOT NULL THEN
id = ANY(@filter_ids)
ELSE TRUE
END
AND
CASE
WHEN @filter_tracker_type::text IS NOT NULL THEN
tracker_type = @filter_tracker_type::tracker_type
@@ -164,6 +178,7 @@ func (f *CommonTrackerPatternFilter) SQLFragment() string {
func (f *CommonTrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs {
args := pgx.StrictNamedArgs{
"filter_ids": nil,
"filter_tracker_type": nil,
"filter_match_type": nil,
"filter_common_third_party_id": nil,
@@ -178,6 +193,10 @@ func (f *CommonTrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs {
return args
}
if f.ids != nil {
args["filter_ids"] = f.ids
}
if f.trackerType != nil {
args["filter_tracker_type"] = string(*f.trackerType)
}

18
pkg/proboctl/cmdutil/cmdutil.go
View File

@@ -33,14 +33,30 @@ type Factory struct {
Version string
PgDSN string
CfgFile string
pgClient *pg.Client
}
// PgClient returns a shared pg client, building it on first use. The client
// is memoized because pg.NewClient registers Prometheus collectors, so
// constructing it more than once panics with a duplicate registration.
func (f *Factory) PgClient() (*pg.Client, error) {
if f.pgClient != nil {
return f.pgClient, nil
}
if f.PgDSN == "" {
return nil, fmt.Errorf("set --pg-dsn or DATABASE_URL")
}
return pgconn.NewPgClientFromDSN(f.PgDSN)
client, err := pgconn.NewPgClientFromDSN(f.PgDSN)
if err != nil {
return nil, err
}
f.pgClient = client
return f.pgClient, nil
}
// ProbodConfig loads the shared probod configuration file (--cfg-file).

11
pkg/proboctl/commonthirdparty/list.go vendored
View File

@@ -120,9 +120,16 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
return nil
}
table := clicmdutil.NewTable("ID", "NAME", "SLUG", "CATEGORY")
table := clicmdutil.NewTable("ID", "NAME", "SLUG", "CATEGORY", "CREATED", "UPDATED")
for _, p := range parties {
table.Row(p.ID.String(), p.Name, p.Slug, string(p.Category))
table.Row(
p.ID.String(),
p.Name,
p.Slug,
string(p.Category),
p.CreatedAt.Format("2006-01-02 15:04:05"),
p.UpdatedAt.Format("2006-01-02 15:04:05"),
)
}
_, _ = fmt.Fprintln(f.IOStreams.Out, table.Render())

2
pkg/proboctl/commontrackerpattern/commontrackerpattern.go
View File

@@ -49,6 +49,8 @@ func enrichmentState(p *coredata.CommonTrackerPattern) string {
switch {
case p.EnrichmentRequestedAt != nil:
return "queued"
case p.EnrichedAt != nil && p.Description == "":
return "enriched (no description)"
case p.EnrichedAt != nil:
return "enriched"
default:

97
pkg/proboctl/commontrackerpattern/list.go
View File

@@ -29,16 +29,17 @@ import (
func newCmdList(f *cmdutil.Factory) *cobra.Command {
var (
flagTrackerType string
flagMatchType string
flagThirdParty string
flagKeyword string
flagState string
flagLinked bool
flagUnlinked bool
flagSort string
flagOrder string
flagLimit int
flagTrackerType string
flagMatchType string
flagCommonThirdParty string
flagLinkedBanner string
flagLinkedOrg string
flagKeyword string
flagState string
flagWithCommonThirdParty bool
flagSort string
flagOrder string
flagLimit int
)
cmd := &cobra.Command{
@@ -51,11 +52,12 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Filter by tracker type (COOKIE, LOCAL_STORAGE, SESSION_STORAGE, INDEXED_DB)")
cmd.Flags().StringVar(&flagMatchType, "match-type", "", "Filter by match type (EXACT, GLOB, PREFIX)")
cmd.Flags().StringVar(&flagThirdParty, "third-party", "", "Filter by linked common third party (slug or GID)")
cmd.Flags().StringVar(&flagCommonThirdParty, "common-third-party", "", "Filter by linked common third party (slug or GID)")
cmd.Flags().StringVar(&flagLinkedBanner, "linked-banner", "", "Filter to catalog rows linked to a cookie banner's patterns (GID)")
cmd.Flags().StringVar(&flagLinkedOrg, "linked-org", "", "Filter to catalog rows linked to an organization's patterns (GID)")
cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter by pattern/description substring")
cmd.Flags().StringVar(&flagState, "state", "", "Filter by enrichment state (queued, enriched, unenriched)")
cmd.Flags().BoolVar(&flagLinked, "linked", false, "Only patterns linked to a common third party")
cmd.Flags().BoolVar(&flagUnlinked, "unlinked", false, "Only patterns not linked to a common third party")
cmd.Flags().BoolVar(&flagWithCommonThirdParty, "with-common-third-party", false, "Filter by whether the pattern is linked to a common third party (true/false); ignored when not set")
cmd.Flags().StringVar(&flagSort, "sort", "confidence", "Sort field: pattern, confidence, created, updated, enriched")
cmd.Flags().StringVar(&flagOrder, "order", "", "Sort order: asc, desc (default depends on field)")
cmd.Flags().IntVarP(&flagLimit, "limit", "L", 50, "Maximum rows to return (0 for all)")
@@ -65,8 +67,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
return err
}
if flagLinked && flagUnlinked {
return fmt.Errorf("--linked and --unlinked are mutually exclusive")
if flagLinkedBanner != "" && flagLinkedOrg != "" {
return fmt.Errorf("--linked-banner and --linked-org are mutually exclusive")
}
orderBy, err := parseOrderBy(flagSort, flagOrder)
@@ -74,7 +76,12 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
return err
}
filter, err := buildListFilter(flagTrackerType, flagMatchType, flagKeyword, flagState, flagLinked, flagUnlinked)
var withCommonThirdParty *bool
if cmd.Flags().Changed("with-common-third-party") {
withCommonThirdParty = &flagWithCommonThirdParty
}
filter, err := buildListFilter(flagTrackerType, flagMatchType, flagKeyword, flagState, withCommonThirdParty)
if err != nil {
return err
}
@@ -91,8 +98,8 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
if err := pgClient.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
if flagThirdParty != "" {
id, err := resolveCommonThirdPartyID(ctx, conn, flagThirdParty)
if flagCommonThirdParty != "" {
id, err := resolveCommonThirdPartyID(ctx, conn, flagCommonThirdParty)
if err != nil {
return err
}
@@ -100,6 +107,45 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
filter.WithCommonThirdPartyID(&id)
}
switch {
case flagLinkedBanner != "":
bannerID, err := gid.ParseGID(flagLinkedBanner)
if err != nil {
return fmt.Errorf("invalid --linked-banner GID %q: %w", flagLinkedBanner, err)
}
var tps coredata.TrackerPatterns
linkedIDs, err := tps.LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID(ctx, conn, coredata.NewScopeFromObjectID(bannerID), bannerID)
if err != nil {
return err
}
if len(linkedIDs) == 0 {
return nil
}
filter.WithIDs(linkedIDs)
case flagLinkedOrg != "":
orgID, err := gid.ParseGID(flagLinkedOrg)
if err != nil {
return fmt.Errorf("invalid --linked-org GID %q: %w", flagLinkedOrg, err)
}
var tps coredata.TrackerPatterns
linkedIDs, err := tps.LoadAllLinkedCommonTrackerPatternIDsByOrganizationID(ctx, conn, coredata.NewScopeFromObjectID(orgID), orgID)
if err != nil {
return err
}
if len(linkedIDs) == 0 {
return nil
}
filter.WithIDs(linkedIDs)
}
rows, err := cmdutil.Paginate(
ctx,
orderBy,
@@ -168,7 +214,7 @@ func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredat
return err
}
table := clicmdutil.NewTable("ID", "TYPE", "MATCH", "PATTERN", "CONF", "STATE", "THIRD PARTY")
table := clicmdutil.NewTable("ID", "TYPE", "MATCH", "PATTERN", "CONF", "STATE", "THIRD PARTY", "CREATED", "UPDATED")
for _, p := range patterns {
thirdParty := ""
@@ -184,6 +230,8 @@ func renderPatternTable(cmd *cobra.Command, f *cmdutil.Factory, patterns coredat
fmt.Sprintf("%.2f", p.Confidence),
enrichmentState(p),
thirdParty,
p.CreatedAt.Format("2006-01-02 15:04:05"),
p.UpdatedAt.Format("2006-01-02 15:04:05"),
)
}
@@ -239,7 +287,7 @@ func parseOrderBy(sort, order string) (page.OrderBy[coredata.CommonTrackerPatter
func buildListFilter(
trackerType, matchType, keyword, state string,
linked, unlinked bool,
withCommonThirdParty *bool,
) (*coredata.CommonTrackerPatternFilter, error) {
filter := coredata.NewCommonTrackerPatternFilter()
@@ -274,13 +322,8 @@ func buildListFilter(
filter.WithState(&st)
}
switch {
case linked:
v := true
filter.WithLinked(&v)
case unlinked:
v := false
filter.WithLinked(&v)
if withCommonThirdParty != nil {
filter.WithLinked(withCommonThirdParty)
}
return filter, nil

141
pkg/proboctl/commontrackerpattern/reenrich.go
View File

@@ -30,19 +30,18 @@ import (
func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
var (
flagIDs []string
flagThirdParty string
flagTrackerType string
flagKeyword string
flagState string
flagAll bool
flagLinkedBanner string
flagLinkedOrg string
flagConcurrency int
flagResetEnriched bool
flagDryRun bool
flagYes bool
flagEnqueue bool
flagIDs []string
flagLinkedBanner string
flagLinkedOrg string
flagCommonThirdParty string
flagTrackerType string
flagKeyword string
flagState string
flagConcurrency int
flagResetEnriched bool
flagDryRun bool
flagYes bool
flagEnqueue bool
)
cmd := &cobra.Command{
@@ -57,13 +56,12 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
}
cmd.Flags().StringSliceVar(&flagIDs, "id", nil, "Common tracker pattern GID(s) to re-enrich (repeatable)")
cmd.Flags().StringVar(&flagThirdParty, "third-party", "", "Select patterns linked to a common third party (slug or GID)")
cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Select patterns of a tracker type")
cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Select patterns matching a pattern/description substring")
cmd.Flags().StringVar(&flagState, "state", "", "Select by enrichment state (queued, enriched, unenriched)")
cmd.Flags().BoolVar(&flagAll, "all", false, "Select every common tracker pattern")
cmd.Flags().StringVar(&flagLinkedBanner, "linked-banner", "", "Select catalog rows linked to a cookie banner's patterns (GID)")
cmd.Flags().StringVar(&flagLinkedOrg, "linked-org", "", "Select catalog rows linked to an organization's patterns (GID)")
cmd.Flags().StringVar(&flagCommonThirdParty, "common-third-party", "", "Select patterns linked to a common third party (slug or GID)")
cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Filter selected patterns by tracker type")
cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter selected patterns by a pattern/description substring")
cmd.Flags().StringVar(&flagState, "state", "", "Filter selected patterns by enrichment state (queued, enriched, unenriched)")
cmd.Flags().IntVar(&flagConcurrency, "concurrency", 4, "Number of patterns to enrich in parallel (sync mode)")
cmd.Flags().BoolVar(&flagResetEnriched, "reset-enriched", true, "Clear enriched_at so terminal rows are re-processed")
cmd.Flags().BoolVar(&flagDryRun, "dry-run", false, "Print the selected patterns without enriching")
@@ -82,13 +80,12 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
ctx,
pgClient,
flagIDs,
flagThirdParty,
flagLinkedBanner,
flagLinkedOrg,
flagCommonThirdParty,
flagTrackerType,
flagKeyword,
flagState,
flagAll,
flagLinkedBanner,
flagLinkedOrg,
)
if err != nil {
return err
@@ -159,14 +156,35 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
return cmd
}
// resolveReenrichIDs turns the selection flags into the set of common
// tracker pattern IDs to re-enrich. Exactly one selection anchor must be
// provided: --id, --linked-banner, --linked-org, or --common-third-party.
// The --tracker-type, --keyword, and --state flags further narrow the
// anchor's result, except with --id, where the listed patterns are used
// verbatim.
func resolveReenrichIDs(
ctx context.Context,
pgClient *pg.Client,
rawIDs []string,
thirdParty, trackerType, keyword, state string,
all bool,
linkedBanner, linkedOrg string,
linkedBanner, linkedOrg, commonThirdParty string,
trackerType, keyword, state string,
) ([]gid.GID, error) {
anchors := 0
for _, set := range []bool{len(rawIDs) > 0, linkedBanner != "", linkedOrg != "", commonThirdParty != ""} {
if set {
anchors++
}
}
switch {
case anchors == 0:
return nil, fmt.Errorf("specify exactly one selection anchor: --id, --linked-banner, --linked-org, or --common-third-party")
case anchors > 1:
return nil, fmt.Errorf("--id, --linked-banner, --linked-org, and --common-third-party are mutually exclusive")
}
// --id selects patterns verbatim; the filtering flags do not apply.
if len(rawIDs) > 0 {
ids := make([]gid.GID, 0, len(rawIDs))
@@ -187,6 +205,11 @@ func resolveReenrichIDs(
err := pgClient.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
filter, err := buildReenrichFilter(trackerType, keyword, state)
if err != nil {
return err
}
switch {
case linkedBanner != "":
bannerID, err := gid.ParseGID(linkedBanner)
@@ -196,9 +219,16 @@ func resolveReenrichIDs(
var tps coredata.TrackerPatterns
ids, err = tps.LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID(ctx, conn, coredata.NewScopeFromObjectID(bannerID), bannerID)
linkedIDs, err := tps.LoadAllLinkedCommonTrackerPatternIDsByCookieBannerID(ctx, conn, coredata.NewScopeFromObjectID(bannerID), bannerID)
if err != nil {
return err
}
return err
if len(linkedIDs) == 0 {
return nil
}
filter.WithIDs(linkedIDs)
case linkedOrg != "":
orgID, err := gid.ParseGID(linkedOrg)
if err != nil {
@@ -207,25 +237,30 @@ func resolveReenrichIDs(
var tps coredata.TrackerPatterns
ids, err = tps.LoadAllLinkedCommonTrackerPatternIDsByOrganizationID(ctx, conn, coredata.NewScopeFromObjectID(orgID), orgID)
return err
default:
filter, hasSelector, err := buildReenrichFilter(ctx, conn, thirdParty, trackerType, keyword, state)
linkedIDs, err := tps.LoadAllLinkedCommonTrackerPatternIDsByOrganizationID(ctx, conn, coredata.NewScopeFromObjectID(orgID), orgID)
if err != nil {
return err
}
if !hasSelector && !all {
return fmt.Errorf("specify a selector (--id, --third-party, --tracker-type, --state, --keyword, --linked-banner, --linked-org) or --all")
if len(linkedIDs) == 0 {
return nil
}
var ps coredata.CommonTrackerPatterns
filter.WithIDs(linkedIDs)
case commonThirdParty != "":
thirdPartyID, err := resolveCommonThirdPartyID(ctx, conn, commonThirdParty)
if err != nil {
return err
}
ids, err = ps.LoadAllIDs(ctx, conn, filter)
return err
filter.WithCommonThirdPartyID(&thirdPartyID)
}
var ps coredata.CommonTrackerPatterns
ids, err = ps.LoadAllIDs(ctx, conn, filter)
return err
},
)
if err != nil {
@@ -235,54 +270,32 @@ func resolveReenrichIDs(
return ids, nil
}
func buildReenrichFilter(
ctx context.Context,
conn pg.Querier,
thirdParty, trackerType, keyword, state string,
) (*coredata.CommonTrackerPatternFilter, bool, error) {
func buildReenrichFilter(trackerType, keyword, state string) (*coredata.CommonTrackerPatternFilter, error) {
filter := coredata.NewCommonTrackerPatternFilter()
hasSelector := false
if thirdParty != "" {
id, err := resolveCommonThirdPartyID(ctx, conn, thirdParty)
if err != nil {
return nil, false, err
}
filter.WithCommonThirdPartyID(&id)
hasSelector = true
}
if trackerType != "" {
tt := coredata.TrackerType(trackerType)
if !tt.IsValid() {
return nil, false, fmt.Errorf("invalid --tracker-type value %q", trackerType)
return nil, fmt.Errorf("invalid --tracker-type value %q", trackerType)
}
filter.WithTrackerType(&tt)
hasSelector = true
}
if keyword != "" {
filter.WithKeyword(&keyword)
hasSelector = true
}
if state != "" {
st, err := parseEnrichmentState(state)
if err != nil {
return nil, false, err
return nil, err
}
filter.WithState(&st)
hasSelector = true
}
return filter, hasSelector, nil
return filter, nil
}
func printSample(out io.Writer, ids []gid.GID) {