rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Drop reset-enriched flag and add description filter

Browse Source 
The --reset-enriched flag was effectively a no-op: the enrichment
worker claims rows solely on enrichment_requested_at, and SetEnriched
rewrites enriched_at regardless, so clearing it never changed whether a
row was re-processed. Remove the flag and the resetEnriched parameter on
RequestEnrichmentByIDs, which now only stamps enrichment_requested_at.

Add a --without-description filter to the list and reenrich commands,
backed by a new described predicate on CommonTrackerPatternFilter, so an
operator can target catalog rows that still lack a description.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-06-08 17:13:16 +02:00
parent 4eb352a61d
commit b0e8e9b812
4 changed files with 64 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
pkg/coredata/common_tracker_pattern.go
View File

@@ -836,29 +836,27 @@ ORDER BY pattern ASC
}
// RequestEnrichmentByIDs arms enrichment on the given common tracker
// patterns. When resetEnriched is true it also clears enriched_at so rows
// that previously reached a terminal state are re-processed. Returns the
// number of rows re-queued. This is the async fallback path; the
// synchronous enricher service is preferred.
// patterns by stamping enrichment_requested_at, which is the only column
// the enrichment worker claims on. Already-enriched rows are re-processed
// too: the worker overwrites enriched_at and the description when it runs.
// Returns the number of rows re-queued. This is the async fallback path;
// the synchronous enricher service is preferred.
func (ps *CommonTrackerPatterns) RequestEnrichmentByIDs(
ctx context.Context,
tx pg.Tx,
ids []gid.GID,
resetEnriched bool,
) (int64, error) {
q := `
UPDATE common_tracker_patterns
SET
enrichment_requested_at = NOW(),
enriched_at = CASE WHEN @reset_enriched THEN NULL ELSE enriched_at END,
updated_at = NOW()
WHERE
id = ANY(@ids)
`
args := pgx.StrictNamedArgs{
"ids": ids,
"reset_enriched": resetEnriched,
"ids": ids,
}
result, err := tx.Exec(ctx, q, args)

20
pkg/coredata/common_tracker_pattern_filter.go
View File

@@ -77,6 +77,7 @@ type CommonTrackerPatternFilter struct {
commonThirdPartyID *gid.GID
keyword *string
linked *bool
described *bool
state *CommonTrackerPatternEnrichmentState
}
@@ -116,6 +117,14 @@ func (f *CommonTrackerPatternFilter) WithLinked(linked *bool) *CommonTrackerPatt
return f
}
// WithDescribed filters on whether the pattern has a non-empty
// description: true keeps only described rows, false keeps only rows with
// a blank description.
func (f *CommonTrackerPatternFilter) WithDescribed(described *bool) *CommonTrackerPatternFilter {
f.described = described
return f
}
func (f *CommonTrackerPatternFilter) WithState(state *CommonTrackerPatternEnrichmentState) *CommonTrackerPatternFilter {
f.state = state
return f
@@ -165,6 +174,12 @@ func (f *CommonTrackerPatternFilter) SQLFragment() string {
ELSE common_third_party_id IS NULL
END
AND
CASE
WHEN @filter_described::boolean IS NULL THEN TRUE
WHEN @filter_described::boolean THEN description != ''
ELSE description = ''
END
AND
CASE
WHEN @filter_state_queued::boolean THEN enrichment_requested_at IS NOT NULL
WHEN @filter_state_enriched::boolean THEN
@@ -184,6 +199,7 @@ func (f *CommonTrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs {
"filter_common_third_party_id": nil,
"filter_keyword": nil,
"filter_linked": nil,
"filter_described": nil,
"filter_state_queued": false,
"filter_state_enriched": false,
"filter_state_unenriched": false,
@@ -217,6 +233,10 @@ func (f *CommonTrackerPatternFilter) SQLArguments() pgx.StrictNamedArgs {
args["filter_linked"] = *f.linked
}
if f.described != nil {
args["filter_described"] = *f.described
}
if f.state != nil {
switch *f.state {
case CommonTrackerPatternEnrichmentStateQueued:

15
pkg/proboctl/commontrackerpattern/list.go
View File

@@ -37,6 +37,7 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
flagKeyword string
flagState string
flagWithCommonThirdParty bool
flagWithoutDescription bool
flagSort string
flagOrder string
flagLimit int
@@ -58,6 +59,7 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter by pattern/description substring")
cmd.Flags().StringVar(&flagState, "state", "", "Filter by enrichment state (queued, enriched, unenriched)")
cmd.Flags().BoolVar(&flagWithCommonThirdParty, "with-common-third-party", false, "Filter by whether the pattern is linked to a common third party (true/false); ignored when not set")
cmd.Flags().BoolVar(&flagWithoutDescription, "without-description", false, "Only patterns with a blank description")
cmd.Flags().StringVar(&flagSort, "sort", "confidence", "Sort field: pattern, confidence, created, updated, enriched")
cmd.Flags().StringVar(&flagOrder, "order", "", "Sort order: asc, desc (default depends on field)")
cmd.Flags().IntVarP(&flagLimit, "limit", "L", 50, "Maximum rows to return (0 for all)")
@@ -81,7 +83,12 @@ func newCmdList(f *cmdutil.Factory) *cobra.Command {
withCommonThirdParty = &flagWithCommonThirdParty
}
filter, err := buildListFilter(flagTrackerType, flagMatchType, flagKeyword, flagState, withCommonThirdParty)
var described *bool
if flagWithoutDescription {
described = new(false)
}
filter, err := buildListFilter(flagTrackerType, flagMatchType, flagKeyword, flagState, withCommonThirdParty, described)
if err != nil {
return err
}
@@ -287,7 +294,7 @@ func parseOrderBy(sort, order string) (page.OrderBy[coredata.CommonTrackerPatter
func buildListFilter(
trackerType, matchType, keyword, state string,
withCommonThirdParty *bool,
withCommonThirdParty, described *bool,
) (*coredata.CommonTrackerPatternFilter, error) {
filter := coredata.NewCommonTrackerPatternFilter()
@@ -326,6 +333,10 @@ func buildListFilter(
filter.WithLinked(withCommonThirdParty)
}
if described != nil {
filter.WithDescribed(described)
}
return filter, nil
}

44
pkg/proboctl/commontrackerpattern/reenrich.go
View File

@@ -30,18 +30,18 @@ import (
func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
var (
flagIDs []string
flagLinkedBanner string
flagLinkedOrg string
flagCommonThirdParty string
flagTrackerType string
flagKeyword string
flagState string
flagConcurrency int
flagResetEnriched bool
flagDryRun bool
flagYes bool
flagEnqueue bool
flagIDs []string
flagLinkedBanner string
flagLinkedOrg string
flagCommonThirdParty string
flagTrackerType string
flagKeyword string
flagState string
flagWithoutDescription bool
flagConcurrency int
flagDryRun bool
flagYes bool
flagEnqueue bool
)
cmd := &cobra.Command{
@@ -62,8 +62,8 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
cmd.Flags().StringVar(&flagTrackerType, "tracker-type", "", "Filter selected patterns by tracker type")
cmd.Flags().StringVar(&flagKeyword, "keyword", "", "Filter selected patterns by a pattern/description substring")
cmd.Flags().StringVar(&flagState, "state", "", "Filter selected patterns by enrichment state (queued, enriched, unenriched)")
cmd.Flags().BoolVar(&flagWithoutDescription, "without-description", false, "Only patterns with a blank description")
cmd.Flags().IntVar(&flagConcurrency, "concurrency", 4, "Number of patterns to enrich in parallel (sync mode)")
cmd.Flags().BoolVar(&flagResetEnriched, "reset-enriched", true, "Clear enriched_at so terminal rows are re-processed")
cmd.Flags().BoolVar(&flagDryRun, "dry-run", false, "Print the selected patterns without enriching")
cmd.Flags().BoolVar(&flagYes, "yes", false, "Skip confirmation")
cmd.Flags().BoolVar(&flagEnqueue, "enqueue", false, "Arm the async enrichment worker instead of running the agent in-process")
@@ -86,6 +86,7 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
flagTrackerType,
flagKeyword,
flagState,
flagWithoutDescription,
)
if err != nil {
return err
@@ -117,7 +118,7 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
func(ctx context.Context, tx pg.Tx) error {
var ps coredata.CommonTrackerPatterns
requeued, err = ps.RequestEnrichmentByIDs(ctx, tx, ids, flagResetEnriched)
requeued, err = ps.RequestEnrichmentByIDs(ctx, tx, ids)
return err
},
@@ -159,15 +160,16 @@ func newCmdReenrich(f *cmdutil.Factory) *cobra.Command {
// resolveReenrichIDs turns the selection flags into the set of common
// tracker pattern IDs to re-enrich. Exactly one selection anchor must be
// provided: --id, --linked-banner, --linked-org, or --common-third-party.
// The --tracker-type, --keyword, and --state flags further narrow the
// anchor's result, except with --id, where the listed patterns are used
// verbatim.
// The --tracker-type, --keyword, --state, and --without-description flags
// further narrow the anchor's result, except with --id, where the listed
// patterns are used verbatim.
func resolveReenrichIDs(
ctx context.Context,
pgClient *pg.Client,
rawIDs []string,
linkedBanner, linkedOrg, commonThirdParty string,
trackerType, keyword, state string,
withoutDescription bool,
) ([]gid.GID, error) {
anchors := 0
@@ -205,7 +207,7 @@ func resolveReenrichIDs(
err := pgClient.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
filter, err := buildReenrichFilter(trackerType, keyword, state)
filter, err := buildReenrichFilter(trackerType, keyword, state, withoutDescription)
if err != nil {
return err
}
@@ -270,7 +272,7 @@ func resolveReenrichIDs(
return ids, nil
}
func buildReenrichFilter(trackerType, keyword, state string) (*coredata.CommonTrackerPatternFilter, error) {
func buildReenrichFilter(trackerType, keyword, state string, withoutDescription bool) (*coredata.CommonTrackerPatternFilter, error) {
filter := coredata.NewCommonTrackerPatternFilter()
if trackerType != "" {
@@ -295,6 +297,10 @@ func buildReenrichFilter(trackerType, keyword, state string) (*coredata.CommonTr
filter.WithState(&st)
}
if withoutDescription {
filter.WithDescribed(new(false))
}
return filter, nil
}