Unify enrichment tracking and outcome-based status

Make common_tracker_patterns and common_third_parties share one
enrichment-tracking model and fix the misleading proboctl status.

Both tables now carry the enrichment JSONB provenance payload, an
enrichment_attempts counter, and a last_enrichment_attempt_at clock.
On common_tracker_patterns the enriched_at done-flag is renamed to
last_enrichment_attempt_at and stamped at claim time, so it is truthful
to "attempt" rather than "success". A row is considered to have been
through the workflow when it carries an enrichment payload, not when a
timestamp is set, which lets stale recovery key off the payload being
absent with budget remaining, exactly like common_third_parties.

The claim path reads the attempt counter and timestamp back via
RETURNING so the in-memory receiver matches the database clock instead
of a separate app-side time.Now.

The enricher builds a per-field provenance payload (description and
third-party outcomes plus the mapping attribution) and persists it via
UpdateEnrichment, named to mirror the common-third-party sibling. The
common pattern enrichment worker gains a max-attempts ceiling so a
permanently failing row stops looping.

proboctl now shows "enriched" only when every field the last run
recorded an outcome for resolved a value, otherwise "partial (X/Y)",
replacing the misleading "enriched (no description)" label.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-06-15 16:25:53 +02:00
parent a6ed64caaf
commit ef6cead482
17 changed files with 705 additions and 155 deletions

View File

@@ -53,6 +53,7 @@ type (
EnrichmentRequestedAt *time.Time `db:"enrichment_requested_at"`
Enrichment json.RawMessage `db:"enrichment"`
EnrichmentAttempts int `db:"enrichment_attempts"`
LastEnrichmentAttemptAt *time.Time `db:"last_enrichment_attempt_at"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
}
@@ -131,6 +132,7 @@ SELECT
enrichment_requested_at,
enrichment,
enrichment_attempts,
last_enrichment_attempt_at,
created_at,
updated_at
FROM
@@ -191,6 +193,7 @@ SELECT
enrichment_requested_at,
enrichment,
enrichment_attempts,
last_enrichment_attempt_at,
created_at,
updated_at
FROM
@@ -251,6 +254,7 @@ SELECT
enrichment_requested_at,
enrichment,
enrichment_attempts,
last_enrichment_attempt_at,
created_at,
updated_at
FROM
@@ -310,6 +314,7 @@ INSERT INTO common_third_parties (
enrichment_requested_at,
enrichment,
enrichment_attempts,
last_enrichment_attempt_at,
created_at,
updated_at
) VALUES (
@@ -335,6 +340,7 @@ INSERT INTO common_third_parties (
@enrichment_requested_at,
@enrichment,
@enrichment_attempts,
@last_enrichment_attempt_at,
@created_at,
@updated_at
)
@@ -363,6 +369,7 @@ INSERT INTO common_third_parties (
"enrichment_requested_at": t.EnrichmentRequestedAt,
"enrichment": t.Enrichment,
"enrichment_attempts": t.EnrichmentAttempts,
"last_enrichment_attempt_at": t.LastEnrichmentAttemptAt,
"created_at": t.CreatedAt,
"updated_at": t.UpdatedAt,
}
@@ -406,6 +413,7 @@ INSERT INTO common_third_parties (
enrichment_requested_at,
enrichment,
enrichment_attempts,
last_enrichment_attempt_at,
created_at,
updated_at
) VALUES (
@@ -431,6 +439,7 @@ INSERT INTO common_third_parties (
@enrichment_requested_at,
@enrichment,
@enrichment_attempts,
@last_enrichment_attempt_at,
@created_at,
@updated_at
)
@@ -476,6 +485,7 @@ RETURNING
enrichment_requested_at,
enrichment,
enrichment_attempts,
last_enrichment_attempt_at,
created_at,
updated_at
`
@@ -505,6 +515,7 @@ RETURNING
"enrichment_requested_at": t.EnrichmentRequestedAt,
"enrichment": t.Enrichment,
"enrichment_attempts": t.EnrichmentAttempts,
"last_enrichment_attempt_at": t.LastEnrichmentAttemptAt,
"created_at": t.CreatedAt,
"updated_at": t.UpdatedAt,
}
@@ -571,6 +582,7 @@ SELECT
enrichment_requested_at,
enrichment,
enrichment_attempts,
last_enrichment_attempt_at,
created_at,
updated_at
FROM
@@ -625,6 +637,7 @@ SELECT
enrichment_requested_at,
enrichment,
enrichment_attempts,
last_enrichment_attempt_at,
created_at,
updated_at
FROM
@@ -769,6 +782,7 @@ SELECT
enrichment_requested_at,
enrichment,
enrichment_attempts,
last_enrichment_attempt_at,
created_at,
updated_at
FROM
@@ -862,6 +876,7 @@ SELECT
enrichment_requested_at,
enrichment,
enrichment_attempts,
last_enrichment_attempt_at,
created_at,
updated_at
FROM
@@ -895,8 +910,8 @@ LIMIT 1;
}
// ClearEnrichmentRequestedAt removes the row from the enrichment queue
// and bumps the attempt counter. It bumps updated_at so the
// stale-recovery clock starts at claim time, keeping
// and bumps the attempt counter. It stamps last_enrichment_attempt_at so
// the stale-recovery clock starts at claim time, keeping
// ResetStaleCommonThirdPartyEnrichments from re-arming a row that is
// still being processed. The attempt counter is incremented up front so
// a crash between claim and persist still counts against the retry
@@ -910,19 +925,27 @@ UPDATE common_third_parties
SET
enrichment_requested_at = NULL,
enrichment_attempts = enrichment_attempts + 1,
last_enrichment_attempt_at = NOW(),
updated_at = NOW()
WHERE id = @id
RETURNING enrichment_attempts, last_enrichment_attempt_at
`
args := pgx.StrictNamedArgs{"id": t.ID}
_, err := tx.Exec(ctx, q, args)
var (
attempts int
lastAttempt *time.Time
)
err := tx.QueryRow(ctx, q, args).Scan(&attempts, &lastAttempt)
if err != nil {
return fmt.Errorf("cannot clear enrichment requested at: %w", err)
}
t.EnrichmentRequestedAt = nil
t.EnrichmentAttempts++
t.EnrichmentAttempts = attempts
t.LastEnrichmentAttemptAt = lastAttempt
return nil
}
@@ -1024,7 +1047,7 @@ WHERE
AND enrichment IS NULL
AND enrichment_attempts > 0
AND enrichment_attempts < @max_attempts
AND updated_at < @stale_before
AND last_enrichment_attempt_at < @stale_before
`
args := pgx.StrictNamedArgs{