Add stale recovery to tracker mapping worker

The tracker-mapping worker clears mapping_requested_at at claim time, so
a crash or hard failure between Process phases left the pattern dequeued,
unmapped, and with nothing to re-trigger it. Only an incidental sibling
remap could rescue it, so a lone pattern could stay stranded forever.

Implement the worker.StaleRecoverer interface, mirroring the enrichment
worker. ResetStaleMappings re-arms rows that were claimed but never
assigned a catalog row (common_tracker_pattern_id IS NULL) once idle past
a configurable window; a successful Process always assigns one via the
unmatched fallback, so the predicate cleanly detects interrupted runs and
self-heals after a single pass. ClearMappingRequestedAt now bumps
updated_at so the stale clock starts at claim time and the sweep never
recycles an in-flight claim.

Plumb a StaleAfter knob (default 600s) through the config struct, builder
env var, probod wiring, and Helm templates.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-06-01 11:43:53 +02:00
parent 587a4f63cd
commit 952c427d2a
10 changed files with 167 additions and 5 deletions

View File

@@ -1196,13 +1196,19 @@ LIMIT 1;
return nil
}
// ClearMappingRequestedAt removes the row from the mapping queue. It
// bumps updated_at so the stale-recovery clock starts at claim time,
// keeping ResetStaleMappings from re-arming a row that is still being
// processed.
func (tp *TrackerPattern) ClearMappingRequestedAt(
ctx context.Context,
tx pg.Tx,
) error {
q := `
UPDATE tracker_patterns
SET mapping_requested_at = NULL
SET
mapping_requested_at = NULL,
updated_at = NOW()
WHERE id = @id
`
@@ -1218,6 +1224,42 @@ WHERE id = @id
return nil
}
// ResetStaleMappings re-arms mapping_requested_at on rows whose mapping
// was claimed but never completed (no common_tracker_pattern_id) and
// have been idle longer than staleAfter, so a crashed or timed-out
// mapping run is retried. A successful Process always assigns a catalog
// row (the unmatched fallback in createUnmatchedPattern), so a missing
// common_tracker_pattern_id on a dequeued row marks an interrupted run.
//
// Like the claim query, this sweep is intentionally cross-tenant: the
// mapping worker is a system worker that drains the queue regardless of
// tenant.
func ResetStaleMappings(
ctx context.Context,
conn pg.Querier,
staleAfter time.Duration,
) error {
q := `
UPDATE tracker_patterns
SET
mapping_requested_at = NOW(),
updated_at = NOW()
WHERE
mapping_requested_at IS NULL
AND common_tracker_pattern_id IS NULL
AND updated_at < @stale_before
`
args := pgx.StrictNamedArgs{"stale_before": time.Now().Add(-staleAfter)}
_, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot reset stale tracker pattern mappings: %w", err)
}
return nil
}
func (tp *TrackerPattern) SetMappingRequested(
ctx context.Context,
tx pg.Tx,

View File

@@ -249,3 +249,78 @@ func TestTrackerPattern_Update_NotFoundForMissingRow(t *testing.T) {
assert.ErrorIs(t, err, coredata.ErrResourceNotFound)
}
// TestResetStaleMappings pins the mapping stale-recovery contract: a row
// dequeued (mapping_requested_at IS NULL) but never finished (no
// common_tracker_pattern_id) and idle past the window is re-armed, while
// a recently claimed row (clock not yet elapsed) and a completed row
// (catalog row assigned) are left untouched. Without this sweep a crash
// between Process phases would strand the pattern unmapped forever.
func TestResetStaleMappings(t *testing.T) {
t.Parallel()
client := newTestPgClient(t)
ctx := context.Background()
fx := seedTrackerPatternFixture(t, ctx, client)
now := time.Now().UTC().Truncate(time.Microsecond)
old := now.Add(-time.Hour)
maxAge := 3600
source := coredata.CookieSourceScript
newPattern := func(pattern string, updatedAt time.Time, commonID *gid.GID) *coredata.TrackerPattern {
tp := &coredata.TrackerPattern{
ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType),
OrganizationID: fx.organizationID,
CookieBannerID: fx.cookieBannerID,
CookieCategoryID: fx.cookieCategoryID,
CommonTrackerPatternID: commonID,
TrackerType: coredata.TrackerTypeCookie,
Pattern: pattern,
MatchType: coredata.TrackerPatternMatchTypeExact,
DisplayName: pattern,
MaxAgeSeconds: &maxAge,
Source: &source,
CreatedAt: old,
UpdatedAt: updatedAt,
}
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
return tp.Insert(ctx, tx, fx.scope)
}))
return tp
}
commonPattern := coredata.CommonTrackerPattern{
ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType),
TrackerType: coredata.TrackerTypeCookie,
Pattern: "mapped_catalog_" + gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType).String(),
MatchType: coredata.TrackerPatternMatchTypeExact,
Confidence: 0.5,
CreatedAt: old,
UpdatedAt: old,
}
insertCommonTrackerPattern(t, ctx, client, commonPattern)
stale := newPattern("stale_unfinished", old, nil)
fresh := newPattern("fresh_unfinished", now, nil)
completed := newPattern("completed_mapping", old, &commonPattern.ID)
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
return coredata.ResetStaleMappings(ctx, conn, 10*time.Minute)
}))
load := func(id gid.GID) coredata.TrackerPattern {
var reloaded coredata.TrackerPattern
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
return reloaded.LoadByID(ctx, conn, fx.scope, id)
}))
return reloaded
}
assert.NotNil(t, load(stale.ID).MappingRequestedAt, "claimed-but-unfinished idle row must be re-armed")
assert.Nil(t, load(fresh.ID).MappingRequestedAt, "recently claimed row must not be re-armed before the window elapses")
assert.Nil(t, load(completed.ID).MappingRequestedAt, "completed mapping (catalog row assigned) must never be re-armed")
}