From b0e8e9b812d91f66492a0dd5c98c9e4ad360c58d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89mile=20R=C3=A9?= Date: Mon, 8 Jun 2026 17:13:16 +0200 Subject: [PATCH] Drop reset-enriched flag and add description filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The --reset-enriched flag was effectively a no-op: the enrichment worker claims rows solely on enrichment_requested_at, and SetEnriched rewrites enriched_at regardless, so clearing it never changed whether a row was re-processed. Remove the flag and the resetEnriched parameter on RequestEnrichmentByIDs, which now only stamps enrichment_requested_at. Add a --without-description filter to the list and reenrich commands, backed by a new described predicate on CommonTrackerPatternFilter, so an operator can target catalog rows that still lack a description. Signed-off-by: Émile Ré --- pkg/coredata/common_tracker_pattern.go | 14 +++--- pkg/coredata/common_tracker_pattern_filter.go | 20 +++++++++ pkg/proboctl/commontrackerpattern/list.go | 15 ++++++- pkg/proboctl/commontrackerpattern/reenrich.go | 44 +++++++++++-------- 4 files changed, 64 insertions(+), 29 deletions(-) diff --git a/pkg/coredata/common_tracker_pattern.go b/pkg/coredata/common_tracker_pattern.go index 43d19c127..3f242fcdc 100644 --- a/pkg/coredata/common_tracker_pattern.go +++ b/pkg/coredata/common_tracker_pattern.go @@ -836,29 +836,27 @@ ORDER BY pattern ASC } // RequestEnrichmentByIDs arms enrichment on the given common tracker -// patterns. When resetEnriched is true it also clears enriched_at so rows -// that previously reached a terminal state are re-processed. Returns the -// number of rows re-queued. This is the async fallback path; the -// synchronous enricher service is preferred. +// patterns by stamping enrichment_requested_at, which is the only column +// the enrichment worker claims on. Already-enriched rows are re-processed +// too: the worker overwrites enriched_at and the description when it runs. +// Returns the number of rows re-queued. This is the async fallback path; +// the synchronous enricher service is preferred. func (ps *CommonTrackerPatterns) RequestEnrichmentByIDs( ctx context.Context, tx pg.Tx, ids []gid.GID, - resetEnriched bool, ) (int64, error) { q := ` UPDATE common_tracker_patterns SET enrichment_requested_at = NOW(), - enriched_at = CASE WHEN @reset_enriched THEN NULL ELSE enriched_at END, updated_at = NOW() WHERE id = ANY(@ids) ` args := pgx.StrictNamedArgs{ - "ids": ids, - "reset_enriched": resetEnriched, + "ids": ids, } result, err := tx.Exec(ctx, q, args) diff --git a/pkg/coredata/common_tracker_pattern_filter.go b/pkg/coredata/common_tracker_pattern_filter.go index cd9584c92..c317f3180 100644 --- a/pkg/coredata/common_tracker_pattern_filter.go +++ b/pkg/coredata/common_tracker_pattern_filter.go @@ -77,6 +77,7 @@ type CommonTrackerPatternFilter struct { commonThirdPartyID *gid.GID keyword *string linked *bool + described *bool state *CommonTrackerPatternEnrichmentState } @@ -116,6 +117,14 @@ func (f *CommonTrackerPatternFilter) WithLinked(linked *bool) *CommonTrackerPatt return f } +// WithDescribed filters on whether the pattern has a non-empty +// description: true keeps only described rows, false keeps only rows with +// a blank description. +func (f *CommonTrackerPatternFilter) WithDescribed(described *bool) *CommonTrackerPatternFilter { + f.described = described + return f +} + func (f *CommonTrackerPatternFilter) WithState(state *CommonTrackerPatternEnrichmentState) *CommonTrackerPatternFilter { f.state = state return f @@ -165,6 +174,12 @@ func (f *CommonTrackerPatternFilter) SQLFragment() string { ELSE common_third_party_id IS NULL END AND + CASE + WHEN @filter_described::boolean IS NULL THEN TRUE + WHEN @filter_described::boolean THEN description != '' + ELSE description = '' + END + AND CASE WHEN @filter_state_queued::boolean THEN enrichment_requested_at IS NOT NULL WHEN @filter_state_enriched::boolean THEN @@ -184,6 +199,7 @@ func (f *CommonTrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs { "filter_common_third_party_id": nil, "filter_keyword": nil, "filter_linked": nil, + "filter_described": nil, "filter_state_queued": false, "filter_state_enriched": false, "filter_state_unenriched": false, @@ -217,6 +233,10 @@ func (f *CommonTrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs { args["filter_linked"] = *f.linked } + if f.described != nil { + args["filter_described"] = *f.described + } + if f.state != nil { switch *f.state { case CommonTrackerPatternEnrichmentStateQueued: diff --git a/pkg/proboctl/commontrackerpattern/list.go b/pkg/proboctl/commontrackerpattern/list.go index 812224239..4a95dbad1 100644 --- a/pkg/proboctl/commontrackerpattern/list.go +++ b/pkg/proboctl/commontrackerpattern/list.go @@ -37,6 +37,7 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { flagKeyword string flagState string flagWithCommonThirdParty bool + flagWithoutDescription bool flagSort string flagOrder string flagLimit int @@ -58,6 +59,7 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter by pattern/description substring") cmd.Flags().StringVar(&flagState, "state", "", "Filter by enrichment state (queued, enriched, unenriched)") cmd.Flags().BoolVar(&flagWithCommonThirdParty, "with-common-third-party", false, "Filter by whether the pattern is linked to a common third party (true/false); ignored when not set") + cmd.Flags().BoolVar(&flagWithoutDescription, "without-description", false, "Only patterns with a blank description") cmd.Flags().StringVar(&flagSort, "sort", "confidence", "Sort field: pattern, confidence, created, updated, enriched") cmd.Flags().StringVar(&flagOrder, "order", "", "Sort order: asc, desc (default depends on field)") cmd.Flags().IntVarP(&flagLimit, "limit", "L", 50, "Maximum rows to return (0 for all)") @@ -81,7 +83,12 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command { withCommonThirdParty = &flagWithCommonThirdParty } - filter, err := buildListFilter(flagTrackerType, flagMatchType, flagKeyword, flagState, withCommonThirdParty) + var described *bool + if flagWithoutDescription { + described = new(false) + } + + filter, err := buildListFilter(flagTrackerType, flagMatchType, flagKeyword, flagState, withCommonThirdParty, described) if err != nil { return err } @@ -287,7 +294,7 @@ func parseOrderBy(sort, order string) (page.OrderBy[coredata.CommonTrackerPatter func buildListFilter( trackerType, matchType, keyword, state string, - withCommonThirdParty *bool, + withCommonThirdParty, described *bool, ) (*coredata.CommonTrackerPatternFilter, error) { filter := coredata.NewCommonTrackerPatternFilter() @@ -326,6 +333,10 @@ func buildListFilter( filter.WithLinked(withCommonThirdParty) } + if described != nil { + filter.WithDescribed(described) + } + return filter, nil } diff --git a/pkg/proboctl/commontrackerpattern/reenrich.go b/pkg/proboctl/commontrackerpattern/reenrich.go index ce86d0489..2542e3691 100644 --- a/pkg/proboctl/commontrackerpattern/reenrich.go +++ b/pkg/proboctl/commontrackerpattern/reenrich.go @@ -30,18 +30,18 @@ import ( func newCmdReenrich(f *cmdutil.Factory) *cobra.Command { var ( - flagIDs []string - flagLinkedBanner string - flagLinkedOrg string - flagCommonThirdParty string - flagTrackerType string - flagKeyword string - flagState string - flagConcurrency int - flagResetEnriched bool - flagDryRun bool - flagYes bool - flagEnqueue bool + flagIDs []string + flagLinkedBanner string + flagLinkedOrg string + flagCommonThirdParty string + flagTrackerType string + flagKeyword string + flagState string + flagWithoutDescription bool + flagConcurrency int + flagDryRun bool + flagYes bool + flagEnqueue bool ) cmd := &cobra.Command{ @@ -62,8 +62,8 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command { cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Filter selected patterns by tracker type") cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter selected patterns by a pattern/description substring") cmd.Flags().StringVar(&flagState, "state", "", "Filter selected patterns by enrichment state (queued, enriched, unenriched)") + cmd.Flags().BoolVar(&flagWithoutDescription, "without-description", false, "Only patterns with a blank description") cmd.Flags().IntVar(&flagConcurrency, "concurrency", 4, "Number of patterns to enrich in parallel (sync mode)") - cmd.Flags().BoolVar(&flagResetEnriched, "reset-enriched", true, "Clear enriched_at so terminal rows are re-processed") cmd.Flags().BoolVar(&flagDryRun, "dry-run", false, "Print the selected patterns without enriching") cmd.Flags().BoolVar(&flagYes, "yes", false, "Skip confirmation") cmd.Flags().BoolVar(&flagEnqueue, "enqueue", false, "Arm the async enrichment worker instead of running the agent in-process") @@ -86,6 +86,7 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command { flagTrackerType, flagKeyword, flagState, + flagWithoutDescription, ) if err != nil { return err @@ -117,7 +118,7 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command { func(ctx context.Context, tx pg.Tx) error { var ps coredata.CommonTrackerPatterns - requeued, err = ps.RequestEnrichmentByIDs(ctx, tx, ids, flagResetEnriched) + requeued, err = ps.RequestEnrichmentByIDs(ctx, tx, ids) return err }, @@ -159,15 +160,16 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command { // resolveReenrichIDs turns the selection flags into the set of common // tracker pattern IDs to re-enrich. Exactly one selection anchor must be // provided: --id, --linked-banner, --linked-org, or --common-third-party. -// The --tracker-type, --keyword, and --state flags further narrow the -// anchor's result, except with --id, where the listed patterns are used -// verbatim. +// The --tracker-type, --keyword, --state, and --without-description flags +// further narrow the anchor's result, except with --id, where the listed +// patterns are used verbatim. func resolveReenrichIDs( ctx context.Context, pgClient *pg.Client, rawIDs []string, linkedBanner, linkedOrg, commonThirdParty string, trackerType, keyword, state string, + withoutDescription bool, ) ([]gid.GID, error) { anchors := 0 @@ -205,7 +207,7 @@ func resolveReenrichIDs( err := pgClient.WithConn( ctx, func(ctx context.Context, conn pg.Querier) error { - filter, err := buildReenrichFilter(trackerType, keyword, state) + filter, err := buildReenrichFilter(trackerType, keyword, state, withoutDescription) if err != nil { return err } @@ -270,7 +272,7 @@ func resolveReenrichIDs( return ids, nil } -func buildReenrichFilter(trackerType, keyword, state string) (*coredata.CommonTrackerPatternFilter, error) { +func buildReenrichFilter(trackerType, keyword, state string, withoutDescription bool) (*coredata.CommonTrackerPatternFilter, error) { filter := coredata.NewCommonTrackerPatternFilter() if trackerType != "" { @@ -295,6 +297,10 @@ func buildReenrichFilter(trackerType, keyword, state string) (*coredata.CommonTr filter.WithState(&st) } + if withoutDescription { + filter.WithDescribed(new(false)) + } + return filter, nil }