Unify enrichment tracking and outcome-based status
Make common_tracker_patterns and common_third_parties share one enrichment-tracking model and fix the misleading proboctl status. Both tables now carry the enrichment JSONB provenance payload, an enrichment_attempts counter, and a last_enrichment_attempt_at clock. On common_tracker_patterns the enriched_at done-flag is renamed to last_enrichment_attempt_at and stamped at claim time, so it is truthful to "attempt" rather than "success". A row is considered to have been through the workflow when it carries an enrichment payload, not when a timestamp is set, which lets stale recovery key off the payload being absent with budget remaining, exactly like common_third_parties. The claim path reads the attempt counter and timestamp back via RETURNING so the in-memory receiver matches the database clock instead of a separate app-side time.Now. The enricher builds a per-field provenance payload (description and third-party outcomes plus the mapping attribution) and persists it via UpdateEnrichment, named to mirror the common-third-party sibling. The common pattern enrichment worker gains a max-attempts ceiling so a permanently failing row stops looping. proboctl now shows "enriched" only when every field the last run recorded an outcome for resolved a value, otherwise "partial (X/Y)", replacing the misleading "enriched (no description)" label. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -26,17 +26,25 @@ import (
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
)
|
||||
|
||||
const defaultEnrichmentStaleAfter = 10 * time.Minute
|
||||
const (
|
||||
defaultEnrichmentStaleAfter = 10 * time.Minute
|
||||
|
||||
// defaultEnrichmentMaxAttempts caps how many times a row is retried
|
||||
// before stale recovery leaves it alone, so a permanently failing row
|
||||
// does not loop forever.
|
||||
defaultEnrichmentMaxAttempts = 3
|
||||
)
|
||||
|
||||
// commonPatternEnrichmentHandler is the queue poller for common tracker
|
||||
// pattern enrichment. It owns only the claim/dequeue and stale-recovery
|
||||
// mechanics; the enrichment work itself lives in CommonPatternEnricher so
|
||||
// it can also run synchronously from operator tooling.
|
||||
type commonPatternEnrichmentHandler struct {
|
||||
pg *pg.Client
|
||||
logger *log.Logger
|
||||
enricher *CommonPatternEnricher
|
||||
staleAfter time.Duration
|
||||
pg *pg.Client
|
||||
logger *log.Logger
|
||||
enricher *CommonPatternEnricher
|
||||
staleAfter time.Duration
|
||||
maxAttempts int
|
||||
}
|
||||
|
||||
// NewCommonPatternEnrichmentWorker builds the worker that fills
|
||||
@@ -52,17 +60,23 @@ func NewCommonPatternEnrichmentWorker(
|
||||
enrichmentCfg TrackerEnrichmentAgentConfig,
|
||||
mappingCfg TrackerMappingAgentConfig,
|
||||
staleAfter time.Duration,
|
||||
maxAttempts int,
|
||||
opts ...worker.Option,
|
||||
) *worker.Worker[coredata.CommonTrackerPattern] {
|
||||
if staleAfter <= 0 {
|
||||
staleAfter = defaultEnrichmentStaleAfter
|
||||
}
|
||||
|
||||
if maxAttempts <= 0 {
|
||||
maxAttempts = defaultEnrichmentMaxAttempts
|
||||
}
|
||||
|
||||
h := &commonPatternEnrichmentHandler{
|
||||
pg: pgClient,
|
||||
logger: logger,
|
||||
enricher: NewCommonPatternEnricher(pgClient, logger, enrichmentCfg, mappingCfg),
|
||||
staleAfter: staleAfter,
|
||||
pg: pgClient,
|
||||
logger: logger,
|
||||
enricher: NewCommonPatternEnricher(pgClient, logger, enrichmentCfg, mappingCfg),
|
||||
staleAfter: staleAfter,
|
||||
maxAttempts: maxAttempts,
|
||||
}
|
||||
|
||||
return worker.New(
|
||||
@@ -108,7 +122,7 @@ func (h *commonPatternEnrichmentHandler) RecoverStale(ctx context.Context) error
|
||||
return h.pg.WithConn(
|
||||
ctx,
|
||||
func(ctx context.Context, conn pg.Querier) error {
|
||||
if err := coredata.ResetStaleEnrichments(ctx, conn, h.staleAfter); err != nil {
|
||||
if err := coredata.ResetStaleEnrichments(ctx, conn, h.staleAfter, h.maxAttempts); err != nil {
|
||||
return fmt.Errorf("cannot reset stale common tracker pattern enrichments: %w", err)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user