Add stale recovery to tracker mapping worker

The tracker-mapping worker clears mapping_requested_at at claim time, so
a crash or hard failure between Process phases left the pattern dequeued,
unmapped, and with nothing to re-trigger it. Only an incidental sibling
remap could rescue it, so a lone pattern could stay stranded forever.

Implement the worker.StaleRecoverer interface, mirroring the enrichment
worker. ResetStaleMappings re-arms rows that were claimed but never
assigned a catalog row (common_tracker_pattern_id IS NULL) once idle past
a configurable window; a successful Process always assigns one via the
unmatched fallback, so the predicate cleanly detects interrupted runs and
self-heals after a single pass. ClearMappingRequestedAt now bumps
updated_at so the stale clock starts at claim time and the sweep never
recycles an in-flight claim.

Plumb a StaleAfter knob (default 600s) through the config struct, builder
env var, probod wiring, and Helm templates.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-06-01 11:43:53 +02:00
parent 587a4f63cd
commit 952c427d2a
10 changed files with 167 additions and 5 deletions

View File

@@ -1196,13 +1196,19 @@ LIMIT 1;
return nil
}
// ClearMappingRequestedAt removes the row from the mapping queue. It
// bumps updated_at so the stale-recovery clock starts at claim time,
// keeping ResetStaleMappings from re-arming a row that is still being
// processed.
func (tp *TrackerPattern) ClearMappingRequestedAt(
ctx context.Context,
tx pg.Tx,
) error {
q := `
UPDATE tracker_patterns
SET mapping_requested_at = NULL
SET
mapping_requested_at = NULL,
updated_at = NOW()
WHERE id = @id
`
@@ -1218,6 +1224,42 @@ WHERE id = @id
return nil
}
// ResetStaleMappings re-arms mapping_requested_at on rows whose mapping
// was claimed but never completed (no common_tracker_pattern_id) and
// have been idle longer than staleAfter, so a crashed or timed-out
// mapping run is retried. A successful Process always assigns a catalog
// row (the unmatched fallback in createUnmatchedPattern), so a missing
// common_tracker_pattern_id on a dequeued row marks an interrupted run.
//
// Like the claim query, this sweep is intentionally cross-tenant: the
// mapping worker is a system worker that drains the queue regardless of
// tenant.
func ResetStaleMappings(
ctx context.Context,
conn pg.Querier,
staleAfter time.Duration,
) error {
q := `
UPDATE tracker_patterns
SET
mapping_requested_at = NOW(),
updated_at = NOW()
WHERE
mapping_requested_at IS NULL
AND common_tracker_pattern_id IS NULL
AND updated_at < @stale_before
`
args := pgx.StrictNamedArgs{"stale_before": time.Now().Add(-staleAfter)}
_, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot reset stale tracker pattern mappings: %w", err)
}
return nil
}
func (tp *TrackerPattern) SetMappingRequested(
ctx context.Context,
tx pg.Tx,