Add common third party reenrich and stats CLIs

Add `proboctl common-third-party reenrich` to re-arm the async
enrichment worker for selected catalog rows, and `stats` to summarize
the catalog by enrichment state and last run status. Rows are selected
verbatim via --id/--slug or across the catalog via
--category/--keyword/--state/--status, gated by --dry-run and --yes.

Extend `list` with --state/--status filters and STATE/STATUS columns,
and `show` with enrichment state, attempts, last run status, error,
per-field provenance, and discovered domains.

Back these with CommonThirdPartyFilter state/status/IDs filters plus
CommonThirdParties.LoadAllIDs and RequestEnrichmentByIDs. The latter
stamps enrichment_requested_at and resets the attempt counter while
preserving the existing payload, so the worker merge keeps curated and
human-edited provenance.

Also simplify exactLabelMatch to use slices.Contains.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-06-11 18:20:36 +02:00
parent 968895bdfd
commit b617741feb
8 changed files with 780 additions and 18 deletions

View File

@@ -655,6 +655,42 @@ LIMIT 20
return nil
}
// LoadAllIDs returns the IDs of every common third party matching the
// filter, ignoring pagination. It is the selection primitive behind bulk
// operator actions such as re-arming enrichment across a filtered set.
func (t *CommonThirdParties) LoadAllIDs(
ctx context.Context,
conn pg.Querier,
filter *CommonThirdPartyFilter,
) ([]gid.GID, error) {
q := `
SELECT
id
FROM
common_third_parties
WHERE
%s
ORDER BY name ASC
`
q = fmt.Sprintf(q, filter.SQLFragment())
args := pgx.StrictNamedArgs{}
maps.Copy(args, filter.SQLArguments())
rows, err := conn.Query(ctx, q, args)
if err != nil {
return nil, fmt.Errorf("cannot query common third party ids: %w", err)
}
ids, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
if err != nil {
return nil, fmt.Errorf("cannot collect common third party ids: %w", err)
}
return ids, nil
}
func (t CommonThirdParty) UpdateLogoFileID(
ctx context.Context,
conn pg.Tx,
@@ -1003,3 +1039,40 @@ WHERE
return nil
}
// RequestEnrichmentByIDs re-arms enrichment on the given common third
// parties by stamping enrichment_requested_at, which is the only column
// the enrichment worker claims on. It resets enrichment_attempts to 0 so
// the row gets a fresh retry budget: the claim path bumps the counter on
// every run, and without a reset a row near the max-attempts ceiling
// would not be re-armed by stale recovery if a re-run crashed. The
// enrichment payload is left in place so the worker's merge keeps prior
// per-field provenance (it only overwrites fields it owns, never curated
// seed data or human edits). Already-enriched rows are re-processed too.
// Returns the number of rows re-queued.
func (t *CommonThirdParties) RequestEnrichmentByIDs(
ctx context.Context,
tx pg.Tx,
ids []gid.GID,
) (int64, error) {
q := `
UPDATE common_third_parties
SET
enrichment_requested_at = NOW(),
enrichment_attempts = 0,
updated_at = NOW()
WHERE
id = ANY(@ids)
`
args := pgx.StrictNamedArgs{
"ids": ids,
}
result, err := tx.Exec(ctx, q, args)
if err != nil {
return 0, fmt.Errorf("cannot request common third party enrichment: %w", err)
}
return result.RowsAffected(), nil
}

View File

@@ -15,19 +15,83 @@
package coredata
import (
"fmt"
"github.com/jackc/pgx/v5"
"go.probo.inc/probo/pkg/gid"
)
// CommonThirdPartyEnrichmentState is a synthetic filter over the
// enrichment_requested_at / enrichment columns. It is not a stored
// column; it classifies a row's position in the enrichment lifecycle.
// Unlike common tracker patterns, a common third party has no
// enriched_at column: a row is "enriched" once it carries an enrichment
// payload (Process always writes one, even on a no-result run).
type CommonThirdPartyEnrichmentState string
const (
// CommonThirdPartyEnrichmentStateQueued: a row armed for the
// enrichment worker (enrichment_requested_at IS NOT NULL).
CommonThirdPartyEnrichmentStateQueued CommonThirdPartyEnrichmentState = "QUEUED"
// CommonThirdPartyEnrichmentStateEnriched: a row whose enrichment has
// completed (enrichment IS NOT NULL) and is not re-queued.
CommonThirdPartyEnrichmentStateEnriched CommonThirdPartyEnrichmentState = "ENRICHED"
// CommonThirdPartyEnrichmentStateUnenriched: a row never enriched and
// not currently queued.
CommonThirdPartyEnrichmentStateUnenriched CommonThirdPartyEnrichmentState = "UNENRICHED"
)
func (s CommonThirdPartyEnrichmentState) IsValid() bool {
switch s {
case
CommonThirdPartyEnrichmentStateQueued,
CommonThirdPartyEnrichmentStateEnriched,
CommonThirdPartyEnrichmentStateUnenriched:
return true
}
return false
}
func (s CommonThirdPartyEnrichmentState) String() string {
return string(s)
}
func (s CommonThirdPartyEnrichmentState) MarshalText() ([]byte, error) {
return []byte(s.String()), nil
}
func (s *CommonThirdPartyEnrichmentState) UnmarshalText(text []byte) error {
val := CommonThirdPartyEnrichmentState(text)
if !val.IsValid() {
return fmt.Errorf("invalid CommonThirdPartyEnrichmentState value: %q", string(text))
}
*s = val
return nil
}
type CommonThirdPartyFilter struct {
name *string
category *ThirdPartyCategory
keyword *string
ids []gid.GID
name *string
category *ThirdPartyCategory
keyword *string
state *CommonThirdPartyEnrichmentState
enrichmentStatus *string
}
func NewCommonThirdPartyFilter(name *string) *CommonThirdPartyFilter {
return &CommonThirdPartyFilter{name: name}
}
// WithIDs restricts the result to the given common third party IDs. A
// non-nil but empty slice matches nothing.
func (f *CommonThirdPartyFilter) WithIDs(ids []gid.GID) *CommonThirdPartyFilter {
f.ids = ids
return f
}
func (f *CommonThirdPartyFilter) WithCategory(category *ThirdPartyCategory) *CommonThirdPartyFilter {
f.category = category
return f
@@ -38,8 +102,27 @@ func (f *CommonThirdPartyFilter) WithKeyword(keyword *string) *CommonThirdPartyF
return f
}
func (f *CommonThirdPartyFilter) WithState(state *CommonThirdPartyEnrichmentState) *CommonThirdPartyFilter {
f.state = state
return f
}
// WithEnrichmentStatus filters on the run-level status recorded in the
// enrichment payload (done, partial, failed). Rows with no payload never
// match.
func (f *CommonThirdPartyFilter) WithEnrichmentStatus(status *string) *CommonThirdPartyFilter {
f.enrichmentStatus = status
return f
}
func (f *CommonThirdPartyFilter) SQLFragment() string {
return `(
CASE
WHEN @filter_ids::text[] IS NOT NULL THEN
id = ANY(@filter_ids)
ELSE TRUE
END
AND
CASE
WHEN @filter_name::text IS NOT NULL THEN
name ILIKE '%' || @filter_name || '%'
@@ -58,14 +141,38 @@ func (f *CommonThirdPartyFilter) SQLFragment() string {
OR slug ILIKE '%' || @filter_keyword || '%')
ELSE TRUE
END
AND
CASE
WHEN @filter_state_queued::boolean THEN enrichment_requested_at IS NOT NULL
WHEN @filter_state_enriched::boolean THEN
enrichment_requested_at IS NULL AND enrichment IS NOT NULL
WHEN @filter_state_unenriched::boolean THEN
enrichment_requested_at IS NULL AND enrichment IS NULL
ELSE TRUE
END
AND
CASE
WHEN @filter_enrichment_status::text IS NOT NULL THEN
enrichment->>'status' = @filter_enrichment_status
ELSE TRUE
END
)`
}
func (f *CommonThirdPartyFilter) SQLArguments() pgx.StrictNamedArgs {
args := pgx.StrictNamedArgs{
"filter_name": nil,
"filter_category": nil,
"filter_keyword": nil,
"filter_ids": nil,
"filter_name": nil,
"filter_category": nil,
"filter_keyword": nil,
"filter_state_queued": false,
"filter_state_enriched": false,
"filter_state_unenriched": false,
"filter_enrichment_status": nil,
}
if f.ids != nil {
args["filter_ids"] = f.ids
}
if f.name != nil {
@@ -80,5 +187,20 @@ func (f *CommonThirdPartyFilter) SQLArguments() pgx.StrictNamedArgs {
args["filter_keyword"] = *f.keyword
}
if f.state != nil {
switch *f.state {
case CommonThirdPartyEnrichmentStateQueued:
args["filter_state_queued"] = true
case CommonThirdPartyEnrichmentStateEnriched:
args["filter_state_enriched"] = true
case CommonThirdPartyEnrichmentStateUnenriched:
args["filter_state_unenriched"] = true
}
}
if f.enrichmentStatus != nil {
args["filter_enrichment_status"] = *f.enrichmentStatus
}
return args
}