Add common third party reenrich and stats CLIs
Add `proboctl common-third-party reenrich` to re-arm the async enrichment worker for selected catalog rows, and `stats` to summarize the catalog by enrichment state and last run status. Rows are selected verbatim via --id/--slug or across the catalog via --category/--keyword/--state/--status, gated by --dry-run and --yes. Extend `list` with --state/--status filters and STATE/STATUS columns, and `show` with enrichment state, attempts, last run status, error, per-field provenance, and discovered domains. Back these with CommonThirdPartyFilter state/status/IDs filters plus CommonThirdParties.LoadAllIDs and RequestEnrichmentByIDs. The latter stamps enrichment_requested_at and resets the attempt counter while preserving the existing payload, so the worker merge keeps curated and human-edited provenance. Also simplify exactLabelMatch to use slices.Contains. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -655,6 +655,42 @@ LIMIT 20
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadAllIDs returns the IDs of every common third party matching the
|
||||
// filter, ignoring pagination. It is the selection primitive behind bulk
|
||||
// operator actions such as re-arming enrichment across a filtered set.
|
||||
func (t *CommonThirdParties) LoadAllIDs(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
filter *CommonThirdPartyFilter,
|
||||
) ([]gid.GID, error) {
|
||||
q := `
|
||||
SELECT
|
||||
id
|
||||
FROM
|
||||
common_third_parties
|
||||
WHERE
|
||||
%s
|
||||
ORDER BY name ASC
|
||||
`
|
||||
|
||||
q = fmt.Sprintf(q, filter.SQLFragment())
|
||||
|
||||
args := pgx.StrictNamedArgs{}
|
||||
maps.Copy(args, filter.SQLArguments())
|
||||
|
||||
rows, err := conn.Query(ctx, q, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot query common third party ids: %w", err)
|
||||
}
|
||||
|
||||
ids, err := pgx.CollectRows(rows, pgx.RowTo[gid.GID])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot collect common third party ids: %w", err)
|
||||
}
|
||||
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (t CommonThirdParty) UpdateLogoFileID(
|
||||
ctx context.Context,
|
||||
conn pg.Tx,
|
||||
@@ -1003,3 +1039,40 @@ WHERE
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RequestEnrichmentByIDs re-arms enrichment on the given common third
|
||||
// parties by stamping enrichment_requested_at, which is the only column
|
||||
// the enrichment worker claims on. It resets enrichment_attempts to 0 so
|
||||
// the row gets a fresh retry budget: the claim path bumps the counter on
|
||||
// every run, and without a reset a row near the max-attempts ceiling
|
||||
// would not be re-armed by stale recovery if a re-run crashed. The
|
||||
// enrichment payload is left in place so the worker's merge keeps prior
|
||||
// per-field provenance (it only overwrites fields it owns, never curated
|
||||
// seed data or human edits). Already-enriched rows are re-processed too.
|
||||
// Returns the number of rows re-queued.
|
||||
func (t *CommonThirdParties) RequestEnrichmentByIDs(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
ids []gid.GID,
|
||||
) (int64, error) {
|
||||
q := `
|
||||
UPDATE common_third_parties
|
||||
SET
|
||||
enrichment_requested_at = NOW(),
|
||||
enrichment_attempts = 0,
|
||||
updated_at = NOW()
|
||||
WHERE
|
||||
id = ANY(@ids)
|
||||
`
|
||||
|
||||
args := pgx.StrictNamedArgs{
|
||||
"ids": ids,
|
||||
}
|
||||
|
||||
result, err := tx.Exec(ctx, q, args)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot request common third party enrichment: %w", err)
|
||||
}
|
||||
|
||||
return result.RowsAffected(), nil
|
||||
}
|
||||
|
||||
@@ -15,19 +15,83 @@
|
||||
package coredata
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
)
|
||||
|
||||
// CommonThirdPartyEnrichmentState is a synthetic filter over the
|
||||
// enrichment_requested_at / enrichment columns. It is not a stored
|
||||
// column; it classifies a row's position in the enrichment lifecycle.
|
||||
// Unlike common tracker patterns, a common third party has no
|
||||
// enriched_at column: a row is "enriched" once it carries an enrichment
|
||||
// payload (Process always writes one, even on a no-result run).
|
||||
type CommonThirdPartyEnrichmentState string
|
||||
|
||||
const (
|
||||
// CommonThirdPartyEnrichmentStateQueued: a row armed for the
|
||||
// enrichment worker (enrichment_requested_at IS NOT NULL).
|
||||
CommonThirdPartyEnrichmentStateQueued CommonThirdPartyEnrichmentState = "QUEUED"
|
||||
// CommonThirdPartyEnrichmentStateEnriched: a row whose enrichment has
|
||||
// completed (enrichment IS NOT NULL) and is not re-queued.
|
||||
CommonThirdPartyEnrichmentStateEnriched CommonThirdPartyEnrichmentState = "ENRICHED"
|
||||
// CommonThirdPartyEnrichmentStateUnenriched: a row never enriched and
|
||||
// not currently queued.
|
||||
CommonThirdPartyEnrichmentStateUnenriched CommonThirdPartyEnrichmentState = "UNENRICHED"
|
||||
)
|
||||
|
||||
func (s CommonThirdPartyEnrichmentState) IsValid() bool {
|
||||
switch s {
|
||||
case
|
||||
CommonThirdPartyEnrichmentStateQueued,
|
||||
CommonThirdPartyEnrichmentStateEnriched,
|
||||
CommonThirdPartyEnrichmentStateUnenriched:
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (s CommonThirdPartyEnrichmentState) String() string {
|
||||
return string(s)
|
||||
}
|
||||
|
||||
func (s CommonThirdPartyEnrichmentState) MarshalText() ([]byte, error) {
|
||||
return []byte(s.String()), nil
|
||||
}
|
||||
|
||||
func (s *CommonThirdPartyEnrichmentState) UnmarshalText(text []byte) error {
|
||||
val := CommonThirdPartyEnrichmentState(text)
|
||||
if !val.IsValid() {
|
||||
return fmt.Errorf("invalid CommonThirdPartyEnrichmentState value: %q", string(text))
|
||||
}
|
||||
|
||||
*s = val
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type CommonThirdPartyFilter struct {
|
||||
name *string
|
||||
category *ThirdPartyCategory
|
||||
keyword *string
|
||||
ids []gid.GID
|
||||
name *string
|
||||
category *ThirdPartyCategory
|
||||
keyword *string
|
||||
state *CommonThirdPartyEnrichmentState
|
||||
enrichmentStatus *string
|
||||
}
|
||||
|
||||
func NewCommonThirdPartyFilter(name *string) *CommonThirdPartyFilter {
|
||||
return &CommonThirdPartyFilter{name: name}
|
||||
}
|
||||
|
||||
// WithIDs restricts the result to the given common third party IDs. A
|
||||
// non-nil but empty slice matches nothing.
|
||||
func (f *CommonThirdPartyFilter) WithIDs(ids []gid.GID) *CommonThirdPartyFilter {
|
||||
f.ids = ids
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *CommonThirdPartyFilter) WithCategory(category *ThirdPartyCategory) *CommonThirdPartyFilter {
|
||||
f.category = category
|
||||
return f
|
||||
@@ -38,8 +102,27 @@ func (f *CommonThirdPartyFilter) WithKeyword(keyword *string) *CommonThirdPartyF
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *CommonThirdPartyFilter) WithState(state *CommonThirdPartyEnrichmentState) *CommonThirdPartyFilter {
|
||||
f.state = state
|
||||
return f
|
||||
}
|
||||
|
||||
// WithEnrichmentStatus filters on the run-level status recorded in the
|
||||
// enrichment payload (done, partial, failed). Rows with no payload never
|
||||
// match.
|
||||
func (f *CommonThirdPartyFilter) WithEnrichmentStatus(status *string) *CommonThirdPartyFilter {
|
||||
f.enrichmentStatus = status
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *CommonThirdPartyFilter) SQLFragment() string {
|
||||
return `(
|
||||
CASE
|
||||
WHEN @filter_ids::text[] IS NOT NULL THEN
|
||||
id = ANY(@filter_ids)
|
||||
ELSE TRUE
|
||||
END
|
||||
AND
|
||||
CASE
|
||||
WHEN @filter_name::text IS NOT NULL THEN
|
||||
name ILIKE '%' || @filter_name || '%'
|
||||
@@ -58,14 +141,38 @@ func (f *CommonThirdPartyFilter) SQLFragment() string {
|
||||
OR slug ILIKE '%' || @filter_keyword || '%')
|
||||
ELSE TRUE
|
||||
END
|
||||
AND
|
||||
CASE
|
||||
WHEN @filter_state_queued::boolean THEN enrichment_requested_at IS NOT NULL
|
||||
WHEN @filter_state_enriched::boolean THEN
|
||||
enrichment_requested_at IS NULL AND enrichment IS NOT NULL
|
||||
WHEN @filter_state_unenriched::boolean THEN
|
||||
enrichment_requested_at IS NULL AND enrichment IS NULL
|
||||
ELSE TRUE
|
||||
END
|
||||
AND
|
||||
CASE
|
||||
WHEN @filter_enrichment_status::text IS NOT NULL THEN
|
||||
enrichment->>'status' = @filter_enrichment_status
|
||||
ELSE TRUE
|
||||
END
|
||||
)`
|
||||
}
|
||||
|
||||
func (f *CommonThirdPartyFilter) SQLArguments() pgx.StrictNamedArgs {
|
||||
args := pgx.StrictNamedArgs{
|
||||
"filter_name": nil,
|
||||
"filter_category": nil,
|
||||
"filter_keyword": nil,
|
||||
"filter_ids": nil,
|
||||
"filter_name": nil,
|
||||
"filter_category": nil,
|
||||
"filter_keyword": nil,
|
||||
"filter_state_queued": false,
|
||||
"filter_state_enriched": false,
|
||||
"filter_state_unenriched": false,
|
||||
"filter_enrichment_status": nil,
|
||||
}
|
||||
|
||||
if f.ids != nil {
|
||||
args["filter_ids"] = f.ids
|
||||
}
|
||||
|
||||
if f.name != nil {
|
||||
@@ -80,5 +187,20 @@ func (f *CommonThirdPartyFilter) SQLArguments() pgx.StrictNamedArgs {
|
||||
args["filter_keyword"] = *f.keyword
|
||||
}
|
||||
|
||||
if f.state != nil {
|
||||
switch *f.state {
|
||||
case CommonThirdPartyEnrichmentStateQueued:
|
||||
args["filter_state_queued"] = true
|
||||
case CommonThirdPartyEnrichmentStateEnriched:
|
||||
args["filter_state_enriched"] = true
|
||||
case CommonThirdPartyEnrichmentStateUnenriched:
|
||||
args["filter_state_unenriched"] = true
|
||||
}
|
||||
}
|
||||
|
||||
if f.enrichmentStatus != nil {
|
||||
args["filter_enrichment_status"] = *f.enrichmentStatus
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user