Add common third party enricher worker
Introduce a poll-based worker that fills the global common_third_parties catalog (URLs, headquarter address, legal name, certifications, logo) so each tenant no longer starts from sparse, name-only rows. Enrichment is requested at row creation by ResolveOrCreateCommonThirdParty; curated seed rows are not enqueued, to avoid a re-seed storm. The pipeline uses two specialized agents plus a deterministic logo step. Agent A (company profile) resolves legal name, headquarter address, and the canonical website over web search; its website and legal name feed Agent B and the logo step. Agent B (compliance docs) resolves the legal document URLs, trust/security/status pages, and certifications using the browser read-only toolset (gated on ChromeDPAddr) plus web search. The logo step restores pkg/webinspect as a pure deterministic package and stores the discovered icon in S3, linked via logo_file_id. Each agent returns per-field value/confidence/source_url. The worker writes a column only when confidence clears a configurable threshold and the field is not externally owned (seed or human), and always records full per-field provenance in a new enrichment JSONB column so re-runs fill only gaps and human edits are never clobbered. New bookkeeping columns (enrichment_requested_at, enrichment, enrichment_attempts) back the claim queue and stale recovery; agents run outside transactions and results persist in one final transaction. The worker is opt-in: it no-ops unless its agent provider is configured. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -181,6 +181,15 @@ func New() *Implm {
|
||||
StaleAfter: 1500,
|
||||
MaxConcurrency: 1,
|
||||
},
|
||||
CommonThirdPartyEnrichmentWorker: CommonThirdPartyEnrichmentWorkerConfig{
|
||||
Interval: 10,
|
||||
MaxConcurrency: 1,
|
||||
StaleAfter: 900,
|
||||
AgentTimeout: 90,
|
||||
AgentMaxTurns: 12,
|
||||
ConfidenceThreshold: 0.7,
|
||||
MaxAttempts: 3,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -328,6 +337,11 @@ func (impl *Implm) Run(
|
||||
|
||||
fileManagerService := filemanager.NewService(pgClient, baseURL, s3Client)
|
||||
|
||||
commonThirdPartyEnrichmentCfg, err := impl.buildCommonThirdPartyEnrichmentConfig(l, tp, r, fileManagerService)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var (
|
||||
samlCert *x509.Certificate
|
||||
samlKey *rsa.PrivateKey
|
||||
@@ -819,6 +833,34 @@ func (impl *Implm) Run(
|
||||
)
|
||||
}
|
||||
|
||||
// The common-third-party enrichment worker fills catalog metadata
|
||||
// (URLs, address, certifications, logo) via two agents plus a
|
||||
// deterministic logo step. It needs an LLM client, so it is only
|
||||
// started when its agent config is present.
|
||||
stopCommonThirdPartyEnrichmentWorker := func() {}
|
||||
|
||||
if commonThirdPartyEnrichmentCfg.LLMClient != nil {
|
||||
commonThirdPartyEnrichmentWorker := thirdparty.NewCommonThirdPartyEnrichmentWorker(
|
||||
pgClient,
|
||||
l.Named("common-third-party-enrichment-worker"),
|
||||
commonThirdPartyEnrichmentCfg,
|
||||
worker.WithInterval(time.Duration(impl.cfg.CommonThirdPartyEnrichmentWorker.Interval)*time.Second),
|
||||
worker.WithMaxConcurrency(impl.cfg.CommonThirdPartyEnrichmentWorker.MaxConcurrency),
|
||||
)
|
||||
|
||||
var commonThirdPartyEnrichmentWorkerCtx context.Context
|
||||
|
||||
commonThirdPartyEnrichmentWorkerCtx, stopCommonThirdPartyEnrichmentWorker = context.WithCancel(context.Background())
|
||||
|
||||
wg.Go(
|
||||
func() {
|
||||
if err := commonThirdPartyEnrichmentWorker.Run(commonThirdPartyEnrichmentWorkerCtx); err != nil {
|
||||
cancel(fmt.Errorf("common third party enrichment worker crashed: %w", err))
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
mailingListWorker := mailman.NewMailingListWorker(mailmanService, pgClient, l.Named("mailing-list-worker"))
|
||||
mailingListWorkerCtx, stopMailingListWorker := context.WithCancel(context.Background())
|
||||
|
||||
@@ -910,6 +952,7 @@ func (impl *Implm) Run(
|
||||
stopTrackerPolicyWorker()
|
||||
stopTrackerMappingWorker()
|
||||
stopCommonPatternEnrichmentWorker()
|
||||
stopCommonThirdPartyEnrichmentWorker()
|
||||
stopMailingListWorker()
|
||||
stopVettingWorker()
|
||||
stopEvidenceDescriptionWorker()
|
||||
|
||||
Reference in New Issue
Block a user