Files
probo/pkg/cookiebanner/common_pattern_enrichment_worker.go
Émile Ré ef6cead482 Unify enrichment tracking and outcome-based status
Make common_tracker_patterns and common_third_parties share one
enrichment-tracking model and fix the misleading proboctl status.

Both tables now carry the enrichment JSONB provenance payload, an
enrichment_attempts counter, and a last_enrichment_attempt_at clock.
On common_tracker_patterns the enriched_at done-flag is renamed to
last_enrichment_attempt_at and stamped at claim time, so it is truthful
to "attempt" rather than "success". A row is considered to have been
through the workflow when it carries an enrichment payload, not when a
timestamp is set, which lets stale recovery key off the payload being
absent with budget remaining, exactly like common_third_parties.

The claim path reads the attempt counter and timestamp back via
RETURNING so the in-memory receiver matches the database clock instead
of a separate app-side time.Now.

The enricher builds a per-field provenance payload (description and
third-party outcomes plus the mapping attribution) and persists it via
UpdateEnrichment, named to mirror the common-third-party sibling. The
common pattern enrichment worker gains a max-attempts ceiling so a
permanently failing row stops looping.

proboctl now shows "enriched" only when every field the last run
recorded an outcome for resolved a value, otherwise "partial (X/Y)",
replacing the misleading "enriched (no description)" label.

Signed-off-by: Émile Ré <emile@probo.com>
2026-06-16 17:44:57 +02:00

133 lines
4.0 KiB
Go

// Copyright (c) 2026 Probo Inc <hello@probo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package cookiebanner
import (
"context"
"errors"
"fmt"
"time"
"go.gearno.de/kit/log"
"go.gearno.de/kit/pg"
"go.gearno.de/kit/worker"
"go.probo.inc/probo/pkg/coredata"
)
const (
defaultEnrichmentStaleAfter = 10 * time.Minute
// defaultEnrichmentMaxAttempts caps how many times a row is retried
// before stale recovery leaves it alone, so a permanently failing row
// does not loop forever.
defaultEnrichmentMaxAttempts = 3
)
// commonPatternEnrichmentHandler is the queue poller for common tracker
// pattern enrichment. It owns only the claim/dequeue and stale-recovery
// mechanics; the enrichment work itself lives in CommonPatternEnricher so
// it can also run synchronously from operator tooling.
type commonPatternEnrichmentHandler struct {
pg *pg.Client
logger *log.Logger
enricher *CommonPatternEnricher
staleAfter time.Duration
maxAttempts int
}
// NewCommonPatternEnrichmentWorker builds the worker that fills
// descriptions on common_tracker_patterns using an agent with web
// search, then fans the result out to every linked tracker pattern. It is
// a global system worker: common_tracker_patterns is not tenant-scoped,
// so a single enrichment benefits all tenants. The worker no-ops when no
// LLM client is configured; callers should gate registration on config
// presence.
func NewCommonPatternEnrichmentWorker(
pgClient *pg.Client,
logger *log.Logger,
enrichmentCfg TrackerEnrichmentAgentConfig,
mappingCfg TrackerMappingAgentConfig,
staleAfter time.Duration,
maxAttempts int,
opts ...worker.Option,
) *worker.Worker[coredata.CommonTrackerPattern] {
if staleAfter <= 0 {
staleAfter = defaultEnrichmentStaleAfter
}
if maxAttempts <= 0 {
maxAttempts = defaultEnrichmentMaxAttempts
}
h := &commonPatternEnrichmentHandler{
pg: pgClient,
logger: logger,
enricher: NewCommonPatternEnricher(pgClient, logger, enrichmentCfg, mappingCfg),
staleAfter: staleAfter,
maxAttempts: maxAttempts,
}
return worker.New(
"common-pattern-enrichment-worker",
h,
logger,
opts...,
)
}
func (h *commonPatternEnrichmentHandler) Claim(ctx context.Context) (coredata.CommonTrackerPattern, error) {
var cp coredata.CommonTrackerPattern
if err := h.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
if err := cp.LoadNextForEnrichmentForUpdateSkipLocked(ctx, tx); err != nil {
return err
}
return cp.ClearEnrichmentRequestedAt(ctx, tx)
},
); err != nil {
if errors.Is(err, coredata.ErrResourceNotFound) {
return coredata.CommonTrackerPattern{}, worker.ErrNoTask
}
return coredata.CommonTrackerPattern{}, fmt.Errorf("cannot claim common tracker pattern enrichment task: %w", err)
}
return cp, nil
}
func (h *commonPatternEnrichmentHandler) Process(ctx context.Context, cp coredata.CommonTrackerPattern) error {
if !h.enricher.Enabled() {
return nil
}
return h.enricher.EnrichPattern(ctx, cp)
}
func (h *commonPatternEnrichmentHandler) RecoverStale(ctx context.Context) error {
return h.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
if err := coredata.ResetStaleEnrichments(ctx, conn, h.staleAfter, h.maxAttempts); err != nil {
return fmt.Errorf("cannot reset stale common tracker pattern enrichments: %w", err)
}
return nil
},
)
}