Files
probo/pkg/thirdparty/resolver.go
Émile Ré 229c6b99c6 Add common third party enricher worker
Introduce a poll-based worker that fills the global common_third_parties
catalog (URLs, headquarter address, legal name, certifications, logo)
so each tenant no longer starts from sparse, name-only rows. Enrichment
is requested at row creation by ResolveOrCreateCommonThirdParty; curated
seed rows are not enqueued, to avoid a re-seed storm.

The pipeline uses two specialized agents plus a deterministic logo step.
Agent A (company profile) resolves legal name, headquarter address, and
the canonical website over web search; its website and legal name feed
Agent B and the logo step. Agent B (compliance docs) resolves the legal
document URLs, trust/security/status pages, and certifications using the
browser read-only toolset (gated on ChromeDPAddr) plus web search. The
logo step restores pkg/webinspect as a pure deterministic package and
stores the discovered icon in S3, linked via logo_file_id.

Each agent returns per-field value/confidence/source_url. The worker
writes a column only when confidence clears a configurable threshold and
the field is not externally owned (seed or human), and always records
full per-field provenance in a new enrichment JSONB column so re-runs
fill only gaps and human edits are never clobbered. New bookkeeping
columns (enrichment_requested_at, enrichment, enrichment_attempts) back
the claim queue and stale recovery; agents run outside transactions and
results persist in one final transaction.

The worker is opt-in: it no-ops unless its agent provider is configured.

Signed-off-by: Émile Ré <emile@probo.com>
2026-06-12 14:39:51 +02:00

113 lines
3.9 KiB
Go

// Copyright (c) 2026 Probo Inc <hello@probo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package thirdparty
import (
"context"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5/pgconn"
"go.gearno.de/kit/log"
"go.gearno.de/kit/pg"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/slug"
)
// ResolveOrCreateCommonThirdParty links a named vendor to the global
// catalog, creating a row when none matches. Dedup is deterministic:
// exact name, then slug, before insert. Callers run inside their own
// transaction and pass the logger explicitly, so it is shared by the
// tracker mapping worker and the common pattern enrichment worker.
//
// It never seeds common_third_party_domains: observed initiator domains
// are a co-occurrence signal, not verified vendor ownership, and the
// global catalog's domain set (used for cross-tenant domain matching) is
// owned by the curated seed instead.
func ResolveOrCreateCommonThirdParty(
ctx context.Context,
tx pg.Tx,
logger *log.Logger,
name string,
category coredata.ThirdPartyCategory,
) (*gid.GID, error) {
var party coredata.CommonThirdParty
if err := party.LoadByName(ctx, tx, name); err == nil {
return &party.ID, nil
} else if !errors.Is(err, coredata.ErrResourceNotFound) {
return nil, fmt.Errorf("cannot load common third party by name: %w", err)
}
partySlug := slug.Make(name)
if partySlug == "" {
return nil, nil
}
if err := party.LoadBySlug(ctx, tx, partySlug); err == nil {
return &party.ID, nil
} else if !errors.Is(err, coredata.ErrResourceNotFound) {
return nil, fmt.Errorf("cannot load common third party by slug: %w", err)
}
now := time.Now()
party = coredata.CommonThirdParty{
ID: gid.New(gid.NilTenant, coredata.CommonThirdPartyEntityType),
Name: name,
Slug: partySlug,
Category: category,
Certifications: []string{},
// Request enrichment at creation: a freshly resolved catalog row
// carries only name/slug/category, so the enrichment worker fills
// the rest (URLs, address, certifications, logo). Curated seed
// rows are inserted via Upsert without this flag, so a full
// re-seed does not trigger an enrichment storm.
EnrichmentRequestedAt: &now,
CreatedAt: now,
UpdatedAt: now,
}
// Insert inside a savepoint so a concurrent transaction that created
// the same slug between our lookup and write does not abort the
// caller's transaction. On the unique-violation race, reload the
// winning row and return it instead of failing.
insertErr := tx.Savepoint(ctx, func(ctx context.Context, sp pg.Tx) error {
return party.Insert(ctx, sp)
})
if insertErr != nil {
if pgErr, ok := errors.AsType[*pgconn.PgError](insertErr); ok &&
pgErr.Code == "23505" &&
pgErr.ConstraintName == "common_third_parties_slug_key" {
if err := party.LoadBySlug(ctx, tx, partySlug); err != nil {
return nil, fmt.Errorf("cannot reload common third party after insert race: %w", err)
}
return &party.ID, nil
}
return nil, fmt.Errorf("cannot create common third party: %w", insertErr)
}
logger.InfoCtx(
ctx,
"created common third party from agent identification",
log.String("name", name),
log.String("category", category.String()),
)
return &party.ID, nil
}