Add stale recovery to tracker mapping worker
The tracker-mapping worker clears mapping_requested_at at claim time, so a crash or hard failure between Process phases left the pattern dequeued, unmapped, and with nothing to re-trigger it. Only an incidental sibling remap could rescue it, so a lone pattern could stay stranded forever. Implement the worker.StaleRecoverer interface, mirroring the enrichment worker. ResetStaleMappings re-arms rows that were claimed but never assigned a catalog row (common_tracker_pattern_id IS NULL) once idle past a configurable window; a successful Process always assigns one via the unmatched fallback, so the predicate cleanly detects interrupted runs and self-heals after a single pass. ClearMappingRequestedAt now bumps updated_at so the stale clock starts at claim time and the sweep never recycles an in-flight claim. Plumb a StaleAfter knob (default 600s) through the config struct, builder env var, probod wiring, and Helm templates. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -1196,13 +1196,19 @@ LIMIT 1;
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearMappingRequestedAt removes the row from the mapping queue. It
|
||||
// bumps updated_at so the stale-recovery clock starts at claim time,
|
||||
// keeping ResetStaleMappings from re-arming a row that is still being
|
||||
// processed.
|
||||
func (tp *TrackerPattern) ClearMappingRequestedAt(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
) error {
|
||||
q := `
|
||||
UPDATE tracker_patterns
|
||||
SET mapping_requested_at = NULL
|
||||
SET
|
||||
mapping_requested_at = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE id = @id
|
||||
`
|
||||
|
||||
@@ -1218,6 +1224,42 @@ WHERE id = @id
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResetStaleMappings re-arms mapping_requested_at on rows whose mapping
|
||||
// was claimed but never completed (no common_tracker_pattern_id) and
|
||||
// have been idle longer than staleAfter, so a crashed or timed-out
|
||||
// mapping run is retried. A successful Process always assigns a catalog
|
||||
// row (the unmatched fallback in createUnmatchedPattern), so a missing
|
||||
// common_tracker_pattern_id on a dequeued row marks an interrupted run.
|
||||
//
|
||||
// Like the claim query, this sweep is intentionally cross-tenant: the
|
||||
// mapping worker is a system worker that drains the queue regardless of
|
||||
// tenant.
|
||||
func ResetStaleMappings(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
staleAfter time.Duration,
|
||||
) error {
|
||||
q := `
|
||||
UPDATE tracker_patterns
|
||||
SET
|
||||
mapping_requested_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE
|
||||
mapping_requested_at IS NULL
|
||||
AND common_tracker_pattern_id IS NULL
|
||||
AND updated_at < @stale_before
|
||||
`
|
||||
|
||||
args := pgx.StrictNamedArgs{"stale_before": time.Now().Add(-staleAfter)}
|
||||
|
||||
_, err := conn.Exec(ctx, q, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reset stale tracker pattern mappings: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tp *TrackerPattern) SetMappingRequested(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
|
||||
Reference in New Issue
Block a user