Add stale recovery to tracker mapping worker
The tracker-mapping worker clears mapping_requested_at at claim time, so a crash or hard failure between Process phases left the pattern dequeued, unmapped, and with nothing to re-trigger it. Only an incidental sibling remap could rescue it, so a lone pattern could stay stranded forever. Implement the worker.StaleRecoverer interface, mirroring the enrichment worker. ResetStaleMappings re-arms rows that were claimed but never assigned a catalog row (common_tracker_pattern_id IS NULL) once idle past a configurable window; a successful Process always assigns one via the unmatched fallback, so the predicate cleanly detects interrupted runs and self-heals after a single pass. ClearMappingRequestedAt now bumps updated_at so the stale clock starts at claim time and the sweep never recycles an in-flight claim. Plumb a StaleAfter knob (default 600s) through the config struct, builder env var, probod wiring, and Helm templates. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -1196,13 +1196,19 @@ LIMIT 1;
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearMappingRequestedAt removes the row from the mapping queue. It
|
||||
// bumps updated_at so the stale-recovery clock starts at claim time,
|
||||
// keeping ResetStaleMappings from re-arming a row that is still being
|
||||
// processed.
|
||||
func (tp *TrackerPattern) ClearMappingRequestedAt(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
) error {
|
||||
q := `
|
||||
UPDATE tracker_patterns
|
||||
SET mapping_requested_at = NULL
|
||||
SET
|
||||
mapping_requested_at = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE id = @id
|
||||
`
|
||||
|
||||
@@ -1218,6 +1224,42 @@ WHERE id = @id
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResetStaleMappings re-arms mapping_requested_at on rows whose mapping
|
||||
// was claimed but never completed (no common_tracker_pattern_id) and
|
||||
// have been idle longer than staleAfter, so a crashed or timed-out
|
||||
// mapping run is retried. A successful Process always assigns a catalog
|
||||
// row (the unmatched fallback in createUnmatchedPattern), so a missing
|
||||
// common_tracker_pattern_id on a dequeued row marks an interrupted run.
|
||||
//
|
||||
// Like the claim query, this sweep is intentionally cross-tenant: the
|
||||
// mapping worker is a system worker that drains the queue regardless of
|
||||
// tenant.
|
||||
func ResetStaleMappings(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
staleAfter time.Duration,
|
||||
) error {
|
||||
q := `
|
||||
UPDATE tracker_patterns
|
||||
SET
|
||||
mapping_requested_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE
|
||||
mapping_requested_at IS NULL
|
||||
AND common_tracker_pattern_id IS NULL
|
||||
AND updated_at < @stale_before
|
||||
`
|
||||
|
||||
args := pgx.StrictNamedArgs{"stale_before": time.Now().Add(-staleAfter)}
|
||||
|
||||
_, err := conn.Exec(ctx, q, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reset stale tracker pattern mappings: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (tp *TrackerPattern) SetMappingRequested(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
|
||||
@@ -249,3 +249,78 @@ func TestTrackerPattern_Update_NotFoundForMissingRow(t *testing.T) {
|
||||
|
||||
assert.ErrorIs(t, err, coredata.ErrResourceNotFound)
|
||||
}
|
||||
|
||||
// TestResetStaleMappings pins the mapping stale-recovery contract: a row
|
||||
// dequeued (mapping_requested_at IS NULL) but never finished (no
|
||||
// common_tracker_pattern_id) and idle past the window is re-armed, while
|
||||
// a recently claimed row (clock not yet elapsed) and a completed row
|
||||
// (catalog row assigned) are left untouched. Without this sweep a crash
|
||||
// between Process phases would strand the pattern unmapped forever.
|
||||
func TestResetStaleMappings(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := newTestPgClient(t)
|
||||
ctx := context.Background()
|
||||
fx := seedTrackerPatternFixture(t, ctx, client)
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Microsecond)
|
||||
old := now.Add(-time.Hour)
|
||||
maxAge := 3600
|
||||
source := coredata.CookieSourceScript
|
||||
|
||||
newPattern := func(pattern string, updatedAt time.Time, commonID *gid.GID) *coredata.TrackerPattern {
|
||||
tp := &coredata.TrackerPattern{
|
||||
ID: gid.New(fx.scope.GetTenantID(), coredata.TrackerPatternEntityType),
|
||||
OrganizationID: fx.organizationID,
|
||||
CookieBannerID: fx.cookieBannerID,
|
||||
CookieCategoryID: fx.cookieCategoryID,
|
||||
CommonTrackerPatternID: commonID,
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: pattern,
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
DisplayName: pattern,
|
||||
MaxAgeSeconds: &maxAge,
|
||||
Source: &source,
|
||||
CreatedAt: old,
|
||||
UpdatedAt: updatedAt,
|
||||
}
|
||||
|
||||
require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error {
|
||||
return tp.Insert(ctx, tx, fx.scope)
|
||||
}))
|
||||
|
||||
return tp
|
||||
}
|
||||
|
||||
commonPattern := coredata.CommonTrackerPattern{
|
||||
ID: gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType),
|
||||
TrackerType: coredata.TrackerTypeCookie,
|
||||
Pattern: "mapped_catalog_" + gid.New(gid.NilTenant, coredata.CommonTrackerPatternEntityType).String(),
|
||||
MatchType: coredata.TrackerPatternMatchTypeExact,
|
||||
Confidence: 0.5,
|
||||
CreatedAt: old,
|
||||
UpdatedAt: old,
|
||||
}
|
||||
insertCommonTrackerPattern(t, ctx, client, commonPattern)
|
||||
|
||||
stale := newPattern("stale_unfinished", old, nil)
|
||||
fresh := newPattern("fresh_unfinished", now, nil)
|
||||
completed := newPattern("completed_mapping", old, &commonPattern.ID)
|
||||
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return coredata.ResetStaleMappings(ctx, conn, 10*time.Minute)
|
||||
}))
|
||||
|
||||
load := func(id gid.GID) coredata.TrackerPattern {
|
||||
var reloaded coredata.TrackerPattern
|
||||
require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error {
|
||||
return reloaded.LoadByID(ctx, conn, fx.scope, id)
|
||||
}))
|
||||
|
||||
return reloaded
|
||||
}
|
||||
|
||||
assert.NotNil(t, load(stale.ID).MappingRequestedAt, "claimed-but-unfinished idle row must be re-armed")
|
||||
assert.Nil(t, load(fresh.ID).MappingRequestedAt, "recently claimed row must not be re-armed before the window elapses")
|
||||
assert.Nil(t, load(completed.ID).MappingRequestedAt, "completed mapping (catalog row assigned) must never be re-armed")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user