From 229c6b99c631209ddaad2b67983a3376b8e6d9fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89mile=20R=C3=A9?= Date: Wed, 10 Jun 2026 13:49:56 +0200 Subject: [PATCH] Add common third party enricher worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce a poll-based worker that fills the global common_third_parties catalog (URLs, headquarter address, legal name, certifications, logo) so each tenant no longer starts from sparse, name-only rows. Enrichment is requested at row creation by ResolveOrCreateCommonThirdParty; curated seed rows are not enqueued, to avoid a re-seed storm. The pipeline uses two specialized agents plus a deterministic logo step. Agent A (company profile) resolves legal name, headquarter address, and the canonical website over web search; its website and legal name feed Agent B and the logo step. Agent B (compliance docs) resolves the legal document URLs, trust/security/status pages, and certifications using the browser read-only toolset (gated on ChromeDPAddr) plus web search. The logo step restores pkg/webinspect as a pure deterministic package and stores the discovered icon in S3, linked via logo_file_id. Each agent returns per-field value/confidence/source_url. The worker writes a column only when confidence clears a configurable threshold and the field is not externally owned (seed or human), and always records full per-field provenance in a new enrichment JSONB column so re-runs fill only gaps and human edits are never clobbered. New bookkeeping columns (enrichment_requested_at, enrichment, enrichment_attempts) back the claim queue and stale recovery; agents run outside transactions and results persist in one final transaction. The worker is opt-in: it no-ops unless its agent provider is configured. Signed-off-by: Émile Ré --- .../charts/probo/templates/deployment.yaml | 46 ++ .../probo/values-production.yaml.example | 20 + contrib/helm/charts/probo/values.yaml | 24 + pkg/bootstrap/builder.go | 18 + pkg/bootstrap/builder_test.go | 29 ++ pkg/coredata/common_third_party.go | 253 ++++++++++ pkg/coredata/migrations/20260609T120000Z.sql | 28 ++ pkg/probod/aliases.go | 5 +- pkg/probod/common_third_party_enrichment.go | 73 +++ pkg/probod/probod.go | 43 ++ pkg/probodconfig/config.go | 5 +- pkg/probodconfig/llm_config.go | 35 +- ...ommon_third_party_company_profile_agent.go | 94 ++++ ...ommon_third_party_compliance_docs_agent.go | 105 ++++ .../common_third_party_enrichment.go | 345 +++++++++++++ .../common_third_party_enrichment_worker.go | 456 ++++++++++++++++++ pkg/thirdparty/common_third_party_logo.go | 207 ++++++++ ...ommon_third_party_company_profile.txt.tmpl | 33 ++ ...ommon_third_party_compliance_docs.txt.tmpl | 51 ++ pkg/thirdparty/resolver.go | 10 +- pkg/webinspect/logo.go | 180 +++++++ pkg/webinspect/logo_test.go | 163 +++++++ pkg/webinspect/parse.go | 117 +++++ 23 files changed, 2325 insertions(+), 15 deletions(-) create mode 100644 pkg/coredata/migrations/20260609T120000Z.sql create mode 100644 pkg/probod/common_third_party_enrichment.go create mode 100644 pkg/thirdparty/common_third_party_company_profile_agent.go create mode 100644 pkg/thirdparty/common_third_party_compliance_docs_agent.go create mode 100644 pkg/thirdparty/common_third_party_enrichment.go create mode 100644 pkg/thirdparty/common_third_party_enrichment_worker.go create mode 100644 pkg/thirdparty/common_third_party_logo.go create mode 100644 pkg/thirdparty/prompts/common_third_party_company_profile.txt.tmpl create mode 100644 pkg/thirdparty/prompts/common_third_party_compliance_docs.txt.tmpl create mode 100644 pkg/webinspect/logo.go create mode 100644 pkg/webinspect/logo_test.go create mode 100644 pkg/webinspect/parse.go diff --git a/contrib/helm/charts/probo/templates/deployment.yaml b/contrib/helm/charts/probo/templates/deployment.yaml index 795fc1bbd..c57ae873c 100644 --- a/contrib/helm/charts/probo/templates/deployment.yaml +++ b/contrib/helm/charts/probo/templates/deployment.yaml @@ -378,6 +378,52 @@ spec: - name: COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS value: {{ .Values.probo.commonPatternEnrichmentWorker.agentMaxTurns | quote }} {{- end }} + # Common Third Party Enrichment Agent + {{- if .Values.probo.commonThirdPartyEnrichment.provider }} + - name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_PROVIDER + value: {{ .Values.probo.commonThirdPartyEnrichment.provider | quote }} + {{- end }} + {{- if .Values.probo.commonThirdPartyEnrichment.modelName }} + - name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MODEL_NAME + value: {{ .Values.probo.commonThirdPartyEnrichment.modelName | quote }} + {{- end }} + {{- if .Values.probo.commonThirdPartyEnrichment.temperature }} + - name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_TEMPERATURE + value: {{ .Values.probo.commonThirdPartyEnrichment.temperature | quote }} + {{- end }} + {{- if .Values.probo.commonThirdPartyEnrichment.maxTokens }} + - name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MAX_TOKENS + value: {{ .Values.probo.commonThirdPartyEnrichment.maxTokens | quote }} + {{- end }} + # Common Third Party Enrichment Worker + {{- if .Values.probo.commonThirdPartyEnrichmentWorker.interval }} + - name: COMMON_THIRD_PARTY_ENRICHMENT_INTERVAL + value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.interval | quote }} + {{- end }} + {{- if .Values.probo.commonThirdPartyEnrichmentWorker.maxConcurrency }} + - name: COMMON_THIRD_PARTY_ENRICHMENT_MAX_CONCURRENCY + value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.maxConcurrency | quote }} + {{- end }} + {{- if .Values.probo.commonThirdPartyEnrichmentWorker.staleAfter }} + - name: COMMON_THIRD_PARTY_ENRICHMENT_STALE_AFTER + value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.staleAfter | quote }} + {{- end }} + {{- if .Values.probo.commonThirdPartyEnrichmentWorker.agentTimeout }} + - name: COMMON_THIRD_PARTY_ENRICHMENT_AGENT_TIMEOUT + value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.agentTimeout | quote }} + {{- end }} + {{- if .Values.probo.commonThirdPartyEnrichmentWorker.agentMaxTurns }} + - name: COMMON_THIRD_PARTY_ENRICHMENT_AGENT_MAX_TURNS + value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.agentMaxTurns | quote }} + {{- end }} + {{- if .Values.probo.commonThirdPartyEnrichmentWorker.confidenceThreshold }} + - name: COMMON_THIRD_PARTY_ENRICHMENT_CONFIDENCE_THRESHOLD + value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.confidenceThreshold | quote }} + {{- end }} + {{- if .Values.probo.commonThirdPartyEnrichmentWorker.maxAttempts }} + - name: COMMON_THIRD_PARTY_ENRICHMENT_MAX_ATTEMPTS + value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.maxAttempts | quote }} + {{- end }} # Custom Domains {{- if .Values.probo.customDomains.enabled }} - name: CUSTOM_DOMAINS_RENEWAL_INTERVAL diff --git a/contrib/helm/charts/probo/values-production.yaml.example b/contrib/helm/charts/probo/values-production.yaml.example index 4de046539..ee5861ecf 100644 --- a/contrib/helm/charts/probo/values-production.yaml.example +++ b/contrib/helm/charts/probo/values-production.yaml.example @@ -229,6 +229,26 @@ probo: # agentTimeout: 45 # agentMaxTurns: 10 + # Common third party enrichment agent (optional, fills common_third_parties + # metadata: URLs, address, certifications, logo). + # commonThirdPartyEnrichment: + # provider: "openai" + # modelName: "gpt-4o" + # temperature: "0.1" + # maxTokens: "8192" + + # Common third party enrichment worker tuning (optional; seconds for + # interval/staleAfter/agentTimeout). confidenceThreshold is the 0-1 floor + # a value must clear before it is written. + # commonThirdPartyEnrichmentWorker: + # interval: 10 + # maxConcurrency: 1 + # staleAfter: 900 + # agentTimeout: 90 + # agentMaxTurns: 12 + # confidenceThreshold: 0.7 + # maxAttempts: 3 + # OpenTelemetry tracing (optional) tracing: enabled: true diff --git a/contrib/helm/charts/probo/values.yaml b/contrib/helm/charts/probo/values.yaml index 27e2d8656..9f6f09048 100644 --- a/contrib/helm/charts/probo/values.yaml +++ b/contrib/helm/charts/probo/values.yaml @@ -331,6 +331,30 @@ probo: agentTimeout: 45 agentMaxTurns: 10 + # Common third party enrichment agent (optional, requires a provider + # key). Powers the company-profile and compliance-docs agents that fill + # common_third_parties metadata. Browser tools for the compliance-docs + # agent use the chromeDpAddr endpoint when set; web search uses + # firecrawl.apiKey when set. + commonThirdPartyEnrichment: + provider: "" + modelName: "" + temperature: "" + maxTokens: "" + + # Common third party enrichment background worker tuning (optional). + # interval, staleAfter, and agentTimeout are in seconds. + # confidenceThreshold is the floor (0-1) a resolved value must clear + # before it is written to its column. + commonThirdPartyEnrichmentWorker: + interval: 10 + maxConcurrency: 1 + staleAfter: 900 + agentTimeout: 90 + agentMaxTurns: 12 + confidenceThreshold: 0.7 + maxAttempts: 3 + # Custom domains configuration (optional) customDomains: enabled: false diff --git a/pkg/bootstrap/builder.go b/pkg/bootstrap/builder.go index 65f93c3a0..a734b1d26 100644 --- a/pkg/bootstrap/builder.go +++ b/pkg/bootstrap/builder.go @@ -242,6 +242,15 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { Temperature: b.getEnvFloatPtr("AGENT_TRACKER_ENRICHMENT_TEMPERATURE"), MaxTokens: new(b.getEnvIntOrDefault("AGENT_TRACKER_ENRICHMENT_MAX_TOKENS", 4096)), }, + CommonThirdPartyEnrichment: probodconfig.LLMAgentConfig{ + Provider: b.getEnvOrDefault("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_PROVIDER", ""), + ModelName: b.getEnvOrDefault("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MODEL_NAME", ""), + // Agent B browses pages and emits a moderate structured + // output; the budget must leave headroom for reasoning + // models whose reasoning tokens count against max_tokens. + Temperature: b.getEnvFloatPtr("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_TEMPERATURE"), + MaxTokens: new(b.getEnvIntOrDefault("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MAX_TOKENS", 8192)), + }, Tools: probodconfig.AgentToolsConfig{ FirecrawlAPIKey: b.getEnv("FIRECRAWL_API_KEY"), }, @@ -292,6 +301,15 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { AgentTimeout: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_AGENT_TIMEOUT", 45), AgentMaxTurns: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS", 10), }, + CommonThirdPartyEnrichmentWorker: probodconfig.CommonThirdPartyEnrichmentWorkerConfig{ + Interval: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_INTERVAL", 10), + MaxConcurrency: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_MAX_CONCURRENCY", 1), + StaleAfter: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_STALE_AFTER", 900), + AgentTimeout: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_AGENT_TIMEOUT", 90), + AgentMaxTurns: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_AGENT_MAX_TURNS", 12), + ConfidenceThreshold: b.getEnvFloatOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_CONFIDENCE_THRESHOLD", 0.7), + MaxAttempts: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_MAX_ATTEMPTS", 3), + }, Branding: b.getEnvBoolOrDefault("BRANDING", true), }, } diff --git a/pkg/bootstrap/builder_test.go b/pkg/bootstrap/builder_test.go index 9808952b3..78496f407 100644 --- a/pkg/bootstrap/builder_test.go +++ b/pkg/bootstrap/builder_test.go @@ -245,6 +245,13 @@ func TestBuilder_Build_Defaults(t *testing.T) { assert.Equal(t, 600, cfg.Probod.CommonPatternEnrichmentWorker.StaleAfter) assert.Equal(t, 45, cfg.Probod.CommonPatternEnrichmentWorker.AgentTimeout) assert.Equal(t, 10, cfg.Probod.CommonPatternEnrichmentWorker.AgentMaxTurns) + assert.Equal(t, 10, cfg.Probod.CommonThirdPartyEnrichmentWorker.Interval) + assert.Equal(t, 1, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxConcurrency) + assert.Equal(t, 900, cfg.Probod.CommonThirdPartyEnrichmentWorker.StaleAfter) + assert.Equal(t, 90, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentTimeout) + assert.Equal(t, 12, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentMaxTurns) + assert.Equal(t, 0.7, cfg.Probod.CommonThirdPartyEnrichmentWorker.ConfidenceThreshold) + assert.Equal(t, 3, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxAttempts) assert.Equal(t, 10, cfg.Probod.ThirdPartyVetting.Interval) assert.Equal(t, 1500, cfg.Probod.ThirdPartyVetting.StaleAfter) assert.Equal(t, 1, cfg.Probod.ThirdPartyVetting.MaxConcurrency) @@ -369,6 +376,17 @@ func TestBuilder_Build_CustomValues(t *testing.T) { env["COMMON_PATTERN_ENRICHMENT_STALE_AFTER"] = "900" env["COMMON_PATTERN_ENRICHMENT_AGENT_TIMEOUT"] = "50" env["COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS"] = "5" + // Common third party enrichment agent + worker tuning override + env["AGENT_COMMON_THIRD_PARTY_ENRICHMENT_PROVIDER"] = "openai" + env["AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MODEL_NAME"] = "gpt-4o" + env["AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MAX_TOKENS"] = "16384" + env["COMMON_THIRD_PARTY_ENRICHMENT_INTERVAL"] = "25" + env["COMMON_THIRD_PARTY_ENRICHMENT_MAX_CONCURRENCY"] = "2" + env["COMMON_THIRD_PARTY_ENRICHMENT_STALE_AFTER"] = "1200" + env["COMMON_THIRD_PARTY_ENRICHMENT_AGENT_TIMEOUT"] = "120" + env["COMMON_THIRD_PARTY_ENRICHMENT_AGENT_MAX_TURNS"] = "8" + env["COMMON_THIRD_PARTY_ENRICHMENT_CONFIDENCE_THRESHOLD"] = "0.85" + env["COMMON_THIRD_PARTY_ENRICHMENT_MAX_ATTEMPTS"] = "5" env["THIRD_PARTY_VETTING_INTERVAL"] = "15" env["THIRD_PARTY_VETTING_STALE_AFTER"] = "1800" env["THIRD_PARTY_VETTING_MAX_CONCURRENCY"] = "2" @@ -490,6 +508,17 @@ func TestBuilder_Build_CustomValues(t *testing.T) { assert.Equal(t, 900, cfg.Probod.CommonPatternEnrichmentWorker.StaleAfter) assert.Equal(t, 50, cfg.Probod.CommonPatternEnrichmentWorker.AgentTimeout) assert.Equal(t, 5, cfg.Probod.CommonPatternEnrichmentWorker.AgentMaxTurns) + assert.Equal(t, "openai", cfg.Probod.Agents.CommonThirdPartyEnrichment.Provider) + assert.Equal(t, "gpt-4o", cfg.Probod.Agents.CommonThirdPartyEnrichment.ModelName) + require.NotNil(t, cfg.Probod.Agents.CommonThirdPartyEnrichment.MaxTokens) + assert.Equal(t, 16384, *cfg.Probod.Agents.CommonThirdPartyEnrichment.MaxTokens) + assert.Equal(t, 25, cfg.Probod.CommonThirdPartyEnrichmentWorker.Interval) + assert.Equal(t, 2, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxConcurrency) + assert.Equal(t, 1200, cfg.Probod.CommonThirdPartyEnrichmentWorker.StaleAfter) + assert.Equal(t, 120, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentTimeout) + assert.Equal(t, 8, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentMaxTurns) + assert.Equal(t, 0.85, cfg.Probod.CommonThirdPartyEnrichmentWorker.ConfidenceThreshold) + assert.Equal(t, 5, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxAttempts) assert.Equal(t, 15, cfg.Probod.ThirdPartyVetting.Interval) assert.Equal(t, 1800, cfg.Probod.ThirdPartyVetting.StaleAfter) assert.Equal(t, 2, cfg.Probod.ThirdPartyVetting.MaxConcurrency) diff --git a/pkg/coredata/common_third_party.go b/pkg/coredata/common_third_party.go index b078b5525..67d963360 100644 --- a/pkg/coredata/common_third_party.go +++ b/pkg/coredata/common_third_party.go @@ -16,6 +16,7 @@ package coredata import ( "context" + "encoding/json" "errors" "fmt" "maps" @@ -49,6 +50,9 @@ type ( SecurityPageURL *string `db:"security_page_url"` TrustPageURL *string `db:"trust_page_url"` LogoFileID *gid.GID `db:"logo_file_id"` + EnrichmentRequestedAt *time.Time `db:"enrichment_requested_at"` + Enrichment json.RawMessage `db:"enrichment"` + EnrichmentAttempts int `db:"enrichment_attempts"` CreatedAt time.Time `db:"created_at"` UpdatedAt time.Time `db:"updated_at"` } @@ -124,6 +128,9 @@ SELECT security_page_url, trust_page_url, logo_file_id, + enrichment_requested_at, + enrichment, + enrichment_attempts, created_at, updated_at FROM @@ -181,6 +188,9 @@ SELECT security_page_url, trust_page_url, logo_file_id, + enrichment_requested_at, + enrichment, + enrichment_attempts, created_at, updated_at FROM @@ -238,6 +248,9 @@ SELECT security_page_url, trust_page_url, logo_file_id, + enrichment_requested_at, + enrichment, + enrichment_attempts, created_at, updated_at FROM @@ -294,6 +307,9 @@ INSERT INTO common_third_parties ( security_page_url, trust_page_url, logo_file_id, + enrichment_requested_at, + enrichment, + enrichment_attempts, created_at, updated_at ) VALUES ( @@ -316,6 +332,9 @@ INSERT INTO common_third_parties ( @security_page_url, @trust_page_url, @logo_file_id, + @enrichment_requested_at, + @enrichment, + @enrichment_attempts, @created_at, @updated_at ) @@ -341,6 +360,9 @@ INSERT INTO common_third_parties ( "security_page_url": t.SecurityPageURL, "trust_page_url": t.TrustPageURL, "logo_file_id": t.LogoFileID, + "enrichment_requested_at": t.EnrichmentRequestedAt, + "enrichment": t.Enrichment, + "enrichment_attempts": t.EnrichmentAttempts, "created_at": t.CreatedAt, "updated_at": t.UpdatedAt, } @@ -381,6 +403,9 @@ INSERT INTO common_third_parties ( security_page_url, trust_page_url, logo_file_id, + enrichment_requested_at, + enrichment, + enrichment_attempts, created_at, updated_at ) VALUES ( @@ -403,6 +428,9 @@ INSERT INTO common_third_parties ( @security_page_url, @trust_page_url, @logo_file_id, + @enrichment_requested_at, + @enrichment, + @enrichment_attempts, @created_at, @updated_at ) @@ -445,6 +473,9 @@ RETURNING security_page_url, trust_page_url, logo_file_id, + enrichment_requested_at, + enrichment, + enrichment_attempts, created_at, updated_at ` @@ -471,6 +502,9 @@ RETURNING "security_page_url": t.SecurityPageURL, "trust_page_url": t.TrustPageURL, "logo_file_id": t.LogoFileID, + "enrichment_requested_at": t.EnrichmentRequestedAt, + "enrichment": t.Enrichment, + "enrichment_attempts": t.EnrichmentAttempts, "created_at": t.CreatedAt, "updated_at": t.UpdatedAt, } @@ -534,6 +568,9 @@ SELECT security_page_url, trust_page_url, logo_file_id, + enrichment_requested_at, + enrichment, + enrichment_attempts, created_at, updated_at FROM @@ -585,6 +622,9 @@ SELECT security_page_url, trust_page_url, logo_file_id, + enrichment_requested_at, + enrichment, + enrichment_attempts, created_at, updated_at FROM @@ -690,6 +730,9 @@ SELECT security_page_url, trust_page_url, logo_file_id, + enrichment_requested_at, + enrichment, + enrichment_attempts, created_at, updated_at FROM @@ -750,3 +793,213 @@ WHERE return count, nil } + +// LoadNextForEnrichmentForUpdateSkipLocked claims the oldest row queued +// for enrichment. The global catalog is not tenant-scoped, so the claim +// is intentionally cross-tenant: the enrichment worker is a system +// worker that drains the queue regardless of tenant. +func (t *CommonThirdParty) LoadNextForEnrichmentForUpdateSkipLocked( + ctx context.Context, + tx pg.Tx, +) error { + q := ` +SELECT + id, + name, + slug, + category, + headquarter_address, + legal_name, + website_url, + privacy_policy_url, + service_level_agreement_url, + service_software_agreement_url, + data_processing_agreement_url, + business_associate_agreement_url, + subprocessors_list_url, + certifications, + status_page_url, + terms_of_service_url, + security_page_url, + trust_page_url, + logo_file_id, + enrichment_requested_at, + enrichment, + enrichment_attempts, + created_at, + updated_at +FROM + common_third_parties +WHERE + enrichment_requested_at IS NOT NULL +ORDER BY + enrichment_requested_at ASC +FOR UPDATE SKIP LOCKED +LIMIT 1; +` + + rows, err := tx.Query(ctx, q) + if err != nil { + return fmt.Errorf("cannot query common third party for enrichment: %w", err) + } + defer rows.Close() + + row, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[CommonThirdParty]) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return ErrResourceNotFound + } + + return fmt.Errorf("cannot collect common third party for enrichment: %w", err) + } + + *t = row + + return nil +} + +// ClearEnrichmentRequestedAt removes the row from the enrichment queue +// and bumps the attempt counter. It bumps updated_at so the +// stale-recovery clock starts at claim time, keeping +// ResetStaleCommonThirdPartyEnrichments from re-arming a row that is +// still being processed. The attempt counter is incremented up front so +// a crash between claim and persist still counts against the retry +// budget. +func (t *CommonThirdParty) ClearEnrichmentRequestedAt( + ctx context.Context, + tx pg.Tx, +) error { + q := ` +UPDATE common_third_parties +SET + enrichment_requested_at = NULL, + enrichment_attempts = enrichment_attempts + 1, + updated_at = NOW() +WHERE id = @id +` + + args := pgx.StrictNamedArgs{"id": t.ID} + + _, err := tx.Exec(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot clear enrichment requested at: %w", err) + } + + t.EnrichmentRequestedAt = nil + t.EnrichmentAttempts++ + + return nil +} + +// UpdateEnrichment persists the enrichment result: the resolved metadata +// fields plus the per-field enrichment provenance JSON. It is a targeted +// partial update that never touches id, slug, category, name, logo, the +// queue column, or the attempt counter (logo is owned by +// UpdateLogoFileID; the queue column and counter are managed by +// ClearEnrichmentRequestedAt). The caller decides which scalar fields to +// write versus leave untouched, then passes the merged receiver here. +func (t CommonThirdParty) UpdateEnrichment( + ctx context.Context, + conn pg.Tx, +) error { + q := ` +UPDATE common_third_parties +SET + headquarter_address = @headquarter_address, + legal_name = @legal_name, + website_url = @website_url, + privacy_policy_url = @privacy_policy_url, + service_level_agreement_url = @service_level_agreement_url, + service_software_agreement_url = @service_software_agreement_url, + data_processing_agreement_url = @data_processing_agreement_url, + business_associate_agreement_url = @business_associate_agreement_url, + subprocessors_list_url = @subprocessors_list_url, + certifications = @certifications, + status_page_url = @status_page_url, + terms_of_service_url = @terms_of_service_url, + security_page_url = @security_page_url, + trust_page_url = @trust_page_url, + enrichment = @enrichment, + updated_at = @updated_at +WHERE + id = @id +` + + args := pgx.StrictNamedArgs{ + "id": t.ID, + "headquarter_address": t.HeadquarterAddress, + "legal_name": t.LegalName, + "website_url": t.WebsiteURL, + "privacy_policy_url": t.PrivacyPolicyURL, + "service_level_agreement_url": t.ServiceLevelAgreementURL, + "service_software_agreement_url": t.ServiceSoftwareAgreementURL, + "data_processing_agreement_url": t.DataProcessingAgreementURL, + "business_associate_agreement_url": t.BusinessAssociateAgreementURL, + "subprocessors_list_url": t.SubprocessorsListURL, + "certifications": t.Certifications, + "status_page_url": t.StatusPageURL, + "terms_of_service_url": t.TermsOfServiceURL, + "security_page_url": t.SecurityPageURL, + "trust_page_url": t.TrustPageURL, + "enrichment": t.Enrichment, + "updated_at": t.UpdatedAt, + } + + result, err := conn.Exec(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot update common third party enrichment: %w", err) + } + + if result.RowsAffected() == 0 { + return ErrResourceNotFound + } + + return nil +} + +// ResetStaleCommonThirdPartyEnrichments re-arms enrichment_requested_at +// on rows whose enrichment was claimed but never completed and have been +// idle longer than staleAfter, so a crashed or timed-out run is retried. +// +// A claimed row has enrichment_attempts > 0 (Claim increments it) and a +// completed row has a non-null enrichment payload (Process always writes +// it, even on a no-result run), so the sweep targets rows that were +// claimed but carry no enrichment yet. Curated rows that were never +// enqueued keep enrichment_attempts = 0 and are left untouched. The +// max-attempts ceiling stops permanently failing rows from looping +// forever. +// +// Like the claim query, this sweep is intentionally cross-tenant: the +// enrichment worker is a system worker that drains the queue regardless +// of tenant. +func ResetStaleCommonThirdPartyEnrichments( + ctx context.Context, + conn pg.Querier, + staleAfter time.Duration, + maxAttempts int, +) error { + q := ` +UPDATE common_third_parties +SET + enrichment_requested_at = NOW(), + updated_at = NOW() +WHERE + enrichment_requested_at IS NULL + AND enrichment IS NULL + AND enrichment_attempts > 0 + AND enrichment_attempts < @max_attempts + AND updated_at < @stale_before +` + + args := pgx.StrictNamedArgs{ + "max_attempts": maxAttempts, + "stale_before": time.Now().Add(-staleAfter), + } + + _, err := conn.Exec(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot reset stale common third party enrichments: %w", err) + } + + return nil +} diff --git a/pkg/coredata/migrations/20260609T120000Z.sql b/pkg/coredata/migrations/20260609T120000Z.sql new file mode 100644 index 000000000..6e86b427a --- /dev/null +++ b/pkg/coredata/migrations/20260609T120000Z.sql @@ -0,0 +1,28 @@ +-- Copyright (c) 2026 Probo Inc . +-- +-- Permission to use, copy, modify, and/or distribute this software for any +-- purpose with or without fee is hereby granted, provided that the above +-- copyright notice and this permission notice appear in all copies. +-- +-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +-- PERFORMANCE OF THIS SOFTWARE. + +ALTER TABLE + common_third_parties +ADD + COLUMN enrichment_requested_at TIMESTAMP WITH TIME ZONE, +ADD + COLUMN enrichment JSONB, +ADD + COLUMN enrichment_attempts INTEGER NOT NULL DEFAULT 0; + +-- Partial index backs the enrichment worker's claim query, which polls +-- only rows currently queued for enrichment. +CREATE INDEX common_third_parties_enrichment_requested_at_idx + ON common_third_parties (enrichment_requested_at) + WHERE enrichment_requested_at IS NOT NULL; diff --git a/pkg/probod/aliases.go b/pkg/probod/aliases.go index 22910c272..ae28af29d 100644 --- a/pkg/probod/aliases.go +++ b/pkg/probod/aliases.go @@ -43,8 +43,9 @@ type ( ThirdPartyVettingWorkerConfig = probodconfig.ThirdPartyVettingWorkerConfig AgentsConfig = probodconfig.AgentsConfig - TrackerMappingWorkerConfig = probodconfig.TrackerMappingWorkerConfig - CommonPatternEnrichmentWorkerConfig = probodconfig.CommonPatternEnrichmentWorkerConfig + TrackerMappingWorkerConfig = probodconfig.TrackerMappingWorkerConfig + CommonPatternEnrichmentWorkerConfig = probodconfig.CommonPatternEnrichmentWorkerConfig + CommonThirdPartyEnrichmentWorkerConfig = probodconfig.CommonThirdPartyEnrichmentWorkerConfig MailerConfig = probodconfig.MailerConfig SMTPConfig = probodconfig.SMTPConfig diff --git a/pkg/probod/common_third_party_enrichment.go b/pkg/probod/common_third_party_enrichment.go new file mode 100644 index 000000000..dbdaea71b --- /dev/null +++ b/pkg/probod/common_third_party_enrichment.go @@ -0,0 +1,73 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package probod + +import ( + "fmt" + "time" + + "github.com/prometheus/client_golang/prometheus" + "go.gearno.de/kit/log" + "go.opentelemetry.io/otel/trace" + "go.probo.inc/probo/pkg/filemanager" + "go.probo.inc/probo/pkg/thirdparty" +) + +// buildCommonThirdPartyEnrichmentConfig wires the common-third-party +// enrichment worker config: the LLM client for its two agents plus the +// worker tuning, browser endpoint, and logo-storage dependencies. It is +// opt-in: a deployment that does not set +// `llm.common-third-party-enrichment.provider` gets a zero config (nil +// LLM client), so the worker runs as a no-op and the caller skips +// registration. +func (impl *Implm) buildCommonThirdPartyEnrichmentConfig( + l *log.Logger, + tp trace.TracerProvider, + r prometheus.Registerer, + fileManager *filemanager.Service, +) (thirdparty.EnrichmentConfig, error) { + if impl.cfg.Agents.CommonThirdPartyEnrichment.Provider == "" { + return thirdparty.EnrichmentConfig{}, nil + } + + agentCfg, llmClient, err := impl.resolveAgentClient( + "common-third-party-enrichment", + impl.cfg.Agents.CommonThirdPartyEnrichment, + l, + tp, + r, + ) + if err != nil { + return thirdparty.EnrichmentConfig{}, fmt.Errorf("cannot resolve common third party enrichment agent client: %w", err) + } + + workerCfg := impl.cfg.CommonThirdPartyEnrichmentWorker + + return thirdparty.EnrichmentConfig{ + LLMClient: llmClient, + Model: agentCfg.ModelName, + MaxTokens: agentCfg.MaxTokens, + Temperature: agentCfg.Temperature, + FirecrawlAPIKey: impl.cfg.Agents.Tools.FirecrawlAPIKey, + ChromeAddr: impl.cfg.ChromeDPAddr, + AgentTimeout: time.Duration(workerCfg.AgentTimeout) * time.Second, + MaxTurns: workerCfg.AgentMaxTurns, + ConfidenceThreshold: workerCfg.ConfidenceThreshold, + StaleAfter: time.Duration(workerCfg.StaleAfter) * time.Second, + MaxAttempts: workerCfg.MaxAttempts, + FileManager: fileManager, + Bucket: impl.cfg.AWS.Bucket, + }, nil +} diff --git a/pkg/probod/probod.go b/pkg/probod/probod.go index 401fda026..a3f9655c3 100644 --- a/pkg/probod/probod.go +++ b/pkg/probod/probod.go @@ -181,6 +181,15 @@ func New() *Implm { StaleAfter: 1500, MaxConcurrency: 1, }, + CommonThirdPartyEnrichmentWorker: CommonThirdPartyEnrichmentWorkerConfig{ + Interval: 10, + MaxConcurrency: 1, + StaleAfter: 900, + AgentTimeout: 90, + AgentMaxTurns: 12, + ConfidenceThreshold: 0.7, + MaxAttempts: 3, + }, }, } } @@ -328,6 +337,11 @@ func (impl *Implm) Run( fileManagerService := filemanager.NewService(pgClient, baseURL, s3Client) + commonThirdPartyEnrichmentCfg, err := impl.buildCommonThirdPartyEnrichmentConfig(l, tp, r, fileManagerService) + if err != nil { + return err + } + var ( samlCert *x509.Certificate samlKey *rsa.PrivateKey @@ -819,6 +833,34 @@ func (impl *Implm) Run( ) } + // The common-third-party enrichment worker fills catalog metadata + // (URLs, address, certifications, logo) via two agents plus a + // deterministic logo step. It needs an LLM client, so it is only + // started when its agent config is present. + stopCommonThirdPartyEnrichmentWorker := func() {} + + if commonThirdPartyEnrichmentCfg.LLMClient != nil { + commonThirdPartyEnrichmentWorker := thirdparty.NewCommonThirdPartyEnrichmentWorker( + pgClient, + l.Named("common-third-party-enrichment-worker"), + commonThirdPartyEnrichmentCfg, + worker.WithInterval(time.Duration(impl.cfg.CommonThirdPartyEnrichmentWorker.Interval)*time.Second), + worker.WithMaxConcurrency(impl.cfg.CommonThirdPartyEnrichmentWorker.MaxConcurrency), + ) + + var commonThirdPartyEnrichmentWorkerCtx context.Context + + commonThirdPartyEnrichmentWorkerCtx, stopCommonThirdPartyEnrichmentWorker = context.WithCancel(context.Background()) + + wg.Go( + func() { + if err := commonThirdPartyEnrichmentWorker.Run(commonThirdPartyEnrichmentWorkerCtx); err != nil { + cancel(fmt.Errorf("common third party enrichment worker crashed: %w", err)) + } + }, + ) + } + mailingListWorker := mailman.NewMailingListWorker(mailmanService, pgClient, l.Named("mailing-list-worker")) mailingListWorkerCtx, stopMailingListWorker := context.WithCancel(context.Background()) @@ -910,6 +952,7 @@ func (impl *Implm) Run( stopTrackerPolicyWorker() stopTrackerMappingWorker() stopCommonPatternEnrichmentWorker() + stopCommonThirdPartyEnrichmentWorker() stopMailingListWorker() stopVettingWorker() stopEvidenceDescriptionWorker() diff --git a/pkg/probodconfig/config.go b/pkg/probodconfig/config.go index 02cd09769..2c15a982e 100644 --- a/pkg/probodconfig/config.go +++ b/pkg/probodconfig/config.go @@ -62,8 +62,9 @@ type ( EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"` ThirdPartyVetting ThirdPartyVettingWorkerConfig `json:"third-party-vetting-worker"` - TrackerMappingWorker TrackerMappingWorkerConfig `json:"tracker-mapping-worker"` - CommonPatternEnrichmentWorker CommonPatternEnrichmentWorkerConfig `json:"common-pattern-enrichment-worker"` + TrackerMappingWorker TrackerMappingWorkerConfig `json:"tracker-mapping-worker"` + CommonPatternEnrichmentWorker CommonPatternEnrichmentWorkerConfig `json:"common-pattern-enrichment-worker"` + CommonThirdPartyEnrichmentWorker CommonThirdPartyEnrichmentWorkerConfig `json:"common-third-party-enrichment-worker"` ChromeDPAddr string `json:"chrome-dp-addr"` CustomDomains CustomDomainsConfig `json:"custom-domains"` diff --git a/pkg/probodconfig/llm_config.go b/pkg/probodconfig/llm_config.go index 589c042a6..f81d43bab 100644 --- a/pkg/probodconfig/llm_config.go +++ b/pkg/probodconfig/llm_config.go @@ -76,6 +76,22 @@ type ( AgentMaxTurns int `json:"agent-max-turns"` } + // CommonThirdPartyEnrichmentWorkerConfig holds worker-side tuning for + // the common-third-party enrichment background worker. LLM parameters + // for its agents live under AgentsConfig.CommonThirdPartyEnrichment. + // ConfidenceThreshold is the floor a resolved value must clear before + // it is written to its column; MaxAttempts caps stale-recovery + // retries. + CommonThirdPartyEnrichmentWorkerConfig struct { + Interval int `json:"interval"` // seconds between polls + MaxConcurrency int `json:"max-concurrency"` + StaleAfter int `json:"stale-after"` // seconds before a claim is recycled + AgentTimeout int `json:"agent-timeout"` // seconds, single agent run + AgentMaxTurns int `json:"agent-max-turns"` + ConfidenceThreshold float64 `json:"confidence-threshold"` + MaxAttempts int `json:"max-attempts"` + } + // AgentToolsConfig holds API keys and settings for external tools // that agents can use (web search, scraping, etc.). AgentToolsConfig struct { @@ -86,15 +102,16 @@ type ( // settings. Default is used as a fallback when an agent-specific field // is zero-valued. AgentsConfig struct { - Providers map[string]LLMProviderConfig `json:"providers"` - Default LLMAgentConfig `json:"defaults"` - Probo LLMAgentConfig `json:"probo"` - EvidenceDescriber LLMAgentConfig `json:"evidence-describer"` - ThirdPartyVetter LLMAgentConfig `json:"third-party-vetter"` - ThirdPartyDisambiguation LLMAgentConfig `json:"third-party-disambiguation"` - TrackerMapping LLMAgentConfig `json:"tracker-mapping"` - TrackerEnrichment LLMAgentConfig `json:"tracker-enrichment"` - Tools AgentToolsConfig `json:"tools"` + Providers map[string]LLMProviderConfig `json:"providers"` + Default LLMAgentConfig `json:"defaults"` + Probo LLMAgentConfig `json:"probo"` + EvidenceDescriber LLMAgentConfig `json:"evidence-describer"` + ThirdPartyVetter LLMAgentConfig `json:"third-party-vetter"` + ThirdPartyDisambiguation LLMAgentConfig `json:"third-party-disambiguation"` + TrackerMapping LLMAgentConfig `json:"tracker-mapping"` + TrackerEnrichment LLMAgentConfig `json:"tracker-enrichment"` + CommonThirdPartyEnrichment LLMAgentConfig `json:"common-third-party-enrichment"` + Tools AgentToolsConfig `json:"tools"` } ) diff --git a/pkg/thirdparty/common_third_party_company_profile_agent.go b/pkg/thirdparty/common_third_party_company_profile_agent.go new file mode 100644 index 000000000..a04dc4f17 --- /dev/null +++ b/pkg/thirdparty/common_third_party_company_profile_agent.go @@ -0,0 +1,94 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package thirdparty + +import ( + _ "embed" + "fmt" + "strings" + + "go.gearno.de/kit/log" + "go.probo.inc/probo/pkg/agent" + "go.probo.inc/probo/pkg/agent/tools/search" + "go.probo.inc/probo/pkg/coredata" +) + +//go:embed prompts/common_third_party_company_profile.txt.tmpl +var companyProfilePrompt string + +// CompanyProfileResult is the structured output of the company-profile +// agent (Agent A): the vendor's identity facts. website_url is the key +// signal the compliance-docs agent and the logo step depend on, so it is +// resolved here first. +type CompanyProfileResult struct { + LegalName EnrichedField `json:"legal_name" jsonschema:"The vendor's full legal company name including the entity suffix (e.g. 'Acme Technologies, Inc.')."` + HeadquarterAddress EnrichedField `json:"headquarter_address" jsonschema:"The vendor's headquarters postal address (street, city, region, country)."` + WebsiteURL EnrichedField `json:"website_url" jsonschema:"The vendor's canonical primary marketing website URL (https scheme, no tracking query parameters, no trailing path)."` +} + +func buildCompanyProfileAgent( + cfg EnrichmentConfig, + logger *log.Logger, +) *agent.Agent { + var tools []agent.Tool + + if cfg.FirecrawlAPIKey != "" { + tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlAPIKey)) + } + + outputType, err := agent.NewOutputType[CompanyProfileResult]("common_third_party_company_profile") + if err != nil { + panic(fmt.Sprintf("thirdparty: cannot build company profile output type: %s", err)) + } + + opts := []agent.Option{ + agent.WithInstructions(companyProfilePrompt), + agent.WithModel(cfg.Model), + agent.WithOutputType(outputType), + agent.WithMaxTurns(resolveEnrichmentMaxTurns(cfg.MaxTurns)), + agent.WithMaxTokens(resolveEnrichmentMaxTokens(cfg.MaxTokens)), + agent.WithLogger(logger), + } + + if len(tools) > 0 { + opts = append(opts, agent.WithTools(tools...)) + } + + if cfg.Temperature != nil { + opts = append(opts, agent.WithTemperature(*cfg.Temperature)) + } + + return agent.New("common-third-party-company-profile", cfg.LLMClient, opts...) +} + +// buildCompanyProfilePrompt renders the per-row input for Agent A. Any +// values already on the row are passed as hints so the agent confirms or +// corrects them rather than starting cold. +func buildCompanyProfilePrompt(party coredata.CommonThirdParty) string { + var b strings.Builder + + fmt.Fprintf(&b, "Research this company and return its profile.\n\n") + fmt.Fprintf(&b, " %s \n", party.Name) + + if party.WebsiteURL != nil && strings.TrimSpace(*party.WebsiteURL) != "" { + fmt.Fprintf(&b, " %s \n", *party.WebsiteURL) + } + + if party.LegalName != nil && strings.TrimSpace(*party.LegalName) != "" { + fmt.Fprintf(&b, " %s \n", *party.LegalName) + } + + return b.String() +} diff --git a/pkg/thirdparty/common_third_party_compliance_docs_agent.go b/pkg/thirdparty/common_third_party_compliance_docs_agent.go new file mode 100644 index 000000000..932e58e5a --- /dev/null +++ b/pkg/thirdparty/common_third_party_compliance_docs_agent.go @@ -0,0 +1,105 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package thirdparty + +import ( + _ "embed" + "fmt" + "strings" + + "go.gearno.de/kit/log" + "go.probo.inc/probo/pkg/agent" + "go.probo.inc/probo/pkg/agent/tools/search" +) + +//go:embed prompts/common_third_party_compliance_docs.txt.tmpl +var complianceDocsPrompt string + +// ComplianceDocsResult is the structured output of the compliance-docs +// agent (Agent B): the legal-document URLs, trust/security/status pages, +// and certifications. These all live in the same source ecosystem (the +// vendor footer and trust portal), so one agent resolves them together. +type ComplianceDocsResult struct { + PrivacyPolicyURL EnrichedField `json:"privacy_policy_url" jsonschema:"URL of the vendor's privacy policy."` + TermsOfServiceURL EnrichedField `json:"terms_of_service_url" jsonschema:"URL of the vendor's terms of service / terms of use."` + ServiceLevelAgreementURL EnrichedField `json:"service_level_agreement_url" jsonschema:"URL of the vendor's public service level agreement (SLA). Often gated behind sales; return empty when not public."` + ServiceSoftwareAgreementURL EnrichedField `json:"service_software_agreement_url" jsonschema:"URL of the vendor's master software/subscription agreement (MSA). Often gated or identical to the terms of service; return empty when not public."` + DataProcessingAgreementURL EnrichedField `json:"data_processing_agreement_url" jsonschema:"URL of the vendor's data processing agreement (DPA). Often a PDF; return empty when only available on request."` + BusinessAssociateAgreementURL EnrichedField `json:"business_associate_agreement_url" jsonschema:"URL of the vendor's HIPAA business associate agreement (BAA). Almost always gated behind sales; return empty when not public."` + SubprocessorsListURL EnrichedField `json:"subprocessors_list_url" jsonschema:"URL of the vendor's sub-processors list page."` + StatusPageURL EnrichedField `json:"status_page_url" jsonschema:"URL of the vendor's uptime/status page (e.g. status.vendor.com)."` + SecurityPageURL EnrichedField `json:"security_page_url" jsonschema:"URL of the vendor's security page or security overview."` + TrustPageURL EnrichedField `json:"trust_page_url" jsonschema:"URL of the vendor's trust center / trust portal (e.g. Vanta, SafeBase, Drata hosted)."` + Certifications CertificationsField `json:"certifications" jsonschema:"Certifications and compliance frameworks the vendor publicly claims."` +} + +// buildComplianceDocsAgent builds Agent B. extraTools carries the browser +// read-only toolset when a headless Chrome endpoint is configured; it is +// empty otherwise, in which case the agent relies on web_search alone. +func buildComplianceDocsAgent( + cfg EnrichmentConfig, + logger *log.Logger, + extraTools []agent.Tool, +) *agent.Agent { + tools := append([]agent.Tool{}, extraTools...) + + if cfg.FirecrawlAPIKey != "" { + tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlAPIKey)) + } + + outputType, err := agent.NewOutputType[ComplianceDocsResult]("common_third_party_compliance_docs") + if err != nil { + panic(fmt.Sprintf("thirdparty: cannot build compliance docs output type: %s", err)) + } + + opts := []agent.Option{ + agent.WithInstructions(complianceDocsPrompt), + agent.WithModel(cfg.Model), + agent.WithOutputType(outputType), + agent.WithMaxTurns(resolveEnrichmentMaxTurns(cfg.MaxTurns)), + agent.WithMaxTokens(resolveEnrichmentMaxTokens(cfg.MaxTokens)), + agent.WithLogger(logger), + } + + if len(tools) > 0 { + opts = append(opts, agent.WithTools(tools...)) + } + + if cfg.Temperature != nil { + opts = append(opts, agent.WithTemperature(*cfg.Temperature)) + } + + return agent.New("common-third-party-compliance-docs", cfg.LLMClient, opts...) +} + +// buildComplianceDocsPrompt renders the per-row input for Agent B, +// seeding it with the vendor name and the website/legal name resolved by +// Agent A so it can scope its search to the vendor's own domain. +func buildComplianceDocsPrompt(name, websiteURL, legalName string) string { + var b strings.Builder + + fmt.Fprintf(&b, "Find the compliance documents and trust pages for this vendor.\n\n") + fmt.Fprintf(&b, " %s \n", name) + + if w := strings.TrimSpace(websiteURL); w != "" { + fmt.Fprintf(&b, " %s \n", w) + } + + if l := strings.TrimSpace(legalName); l != "" { + fmt.Fprintf(&b, " %s \n", l) + } + + return b.String() +} diff --git a/pkg/thirdparty/common_third_party_enrichment.go b/pkg/thirdparty/common_third_party_enrichment.go new file mode 100644 index 000000000..0e1ca3d8b --- /dev/null +++ b/pkg/thirdparty/common_third_party_enrichment.go @@ -0,0 +1,345 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package thirdparty + +import ( + "encoding/json" + "strings" + "time" + + "go.probo.inc/probo/pkg/coredata" +) + +// Provenance sources and per-field statuses recorded in the enrichment +// JSON. The "source" distinguishes values written by this enricher from +// values owned externally (curated seed data or a human edit), which the +// enricher must never overwrite. +const ( + enrichmentSourceEnrichment = "enrichment" + enrichmentSourceExternal = "external" + + enrichmentFieldStatusFound = "found" + enrichmentFieldStatusNotFound = "not_found" + enrichmentFieldStatusLowConfidence = "low_confidence" + enrichmentFieldStatusExternal = "exists_external" + + // Run-level status recorded at the top of the enrichment payload. + enrichmentStatusDone = "done" + enrichmentStatusPartial = "partial" + enrichmentStatusFailed = "failed" +) + +type ( + // EnrichedField is the per-field unit the enrichment agents return: + // the resolved value plus a self-assessed confidence and the source + // URL where it was verified. The worker applies a confidence + // threshold before writing the value to its column. + EnrichedField struct { + Value string `json:"value" jsonschema:"The resolved value, or an empty string when not confidently found. Never guess."` + Confidence float64 `json:"confidence" jsonschema:"Confidence from 0.0 to 1.0 that the value is correct. Use 0 when the value was not found."` + SourceURL string `json:"source_url" jsonschema:"The URL where this value was verified, or an empty string."` + } + + // CertificationsField is the list-valued counterpart of EnrichedField + // used for the certifications array. + CertificationsField struct { + Values []string `json:"values" jsonschema:"Certification or compliance framework names the vendor publicly claims (e.g. 'SOC 2 Type II', 'ISO 27001', 'HIPAA'). Empty when none are found."` + Confidence float64 `json:"confidence" jsonschema:"Confidence from 0.0 to 1.0 in the certifications list. Use 0 when none are found."` + SourceURL string `json:"source_url" jsonschema:"The URL where the certifications were found (trust or security page), or an empty string."` + } + + // EnrichmentFieldMeta is the per-field provenance recorded in the + // common_third_parties.enrichment JSON column. + EnrichmentFieldMeta struct { + Confidence float64 `json:"confidence"` + SourceURL string `json:"source_url,omitempty"` + Status string `json:"status"` + Source string `json:"source"` + UpdatedAt time.Time `json:"updated_at"` + } + + // EnrichmentMetadata is the full payload stored in the enrichment + // JSON column: run-level bookkeeping plus per-field provenance keyed + // by the column name. + EnrichmentMetadata struct { + Model string `json:"model,omitempty"` + AttemptedAt time.Time `json:"attempted_at"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + Fields map[string]EnrichmentFieldMeta `json:"fields"` + } +) + +// scalarField describes one *string column the enricher can fill, +// pairing the agent's resolved value with accessors into the receiver. +type scalarField struct { + name string + get func(*coredata.CommonThirdParty) *string + set func(*coredata.CommonThirdParty, *string) + result EnrichedField +} + +// parseEnrichmentFields extracts the prior per-field provenance from a +// row's enrichment payload. A missing or malformed payload yields an +// empty map, which the merge treats as "no field is enrichment-owned". +func parseEnrichmentFields(raw json.RawMessage) map[string]EnrichmentFieldMeta { + if len(raw) == 0 { + return map[string]EnrichmentFieldMeta{} + } + + var meta EnrichmentMetadata + if err := json.Unmarshal(raw, &meta); err != nil { + return map[string]EnrichmentFieldMeta{} + } + + if meta.Fields == nil { + return map[string]EnrichmentFieldMeta{} + } + + return meta.Fields +} + +// applyScalarField merges one resolved scalar field into party, honoring +// the confidence threshold and prior provenance, and records the +// resulting per-field metadata. A value already present that this +// enricher did not write (curated seed data or a human edit) is left +// untouched. The column is written only when the value clears the +// threshold; below it the column keeps its prior value and the metadata +// records why nothing was written. +func applyScalarField( + party *coredata.CommonThirdParty, + meta map[string]EnrichmentFieldMeta, + prior map[string]EnrichmentFieldMeta, + field scalarField, + threshold float64, + now time.Time, +) { + value := strings.TrimSpace(field.result.Value) + sourceURL := strings.TrimSpace(field.result.SourceURL) + + existing := field.get(party) + hasExisting := existing != nil && strings.TrimSpace(*existing) != "" + + priorMeta, hadPrior := prior[field.name] + enrichmentOwned := hadPrior && priorMeta.Source == enrichmentSourceEnrichment + + if hasExisting && !enrichmentOwned { + meta[field.name] = EnrichmentFieldMeta{ + Status: enrichmentFieldStatusExternal, + Source: enrichmentSourceExternal, + UpdatedAt: now, + } + + return + } + + if value != "" && field.result.Confidence >= threshold { + v := value + field.set(party, &v) + meta[field.name] = EnrichmentFieldMeta{ + Confidence: field.result.Confidence, + SourceURL: sourceURL, + Status: enrichmentFieldStatusFound, + Source: enrichmentSourceEnrichment, + UpdatedAt: now, + } + + return + } + + status := enrichmentFieldStatusNotFound + if value != "" { + status = enrichmentFieldStatusLowConfidence + } + + meta[field.name] = EnrichmentFieldMeta{ + Confidence: field.result.Confidence, + SourceURL: sourceURL, + Status: status, + Source: enrichmentSourceEnrichment, + UpdatedAt: now, + } +} + +// applyCertifications is the list-valued counterpart of +// applyScalarField for the certifications column. +func applyCertifications( + party *coredata.CommonThirdParty, + meta map[string]EnrichmentFieldMeta, + prior map[string]EnrichmentFieldMeta, + result CertificationsField, + threshold float64, + now time.Time, +) { + const name = "certifications" + + values := normalizeCertifications(result.Values) + sourceURL := strings.TrimSpace(result.SourceURL) + + hasExisting := len(party.Certifications) > 0 + + priorMeta, hadPrior := prior[name] + enrichmentOwned := hadPrior && priorMeta.Source == enrichmentSourceEnrichment + + if hasExisting && !enrichmentOwned { + meta[name] = EnrichmentFieldMeta{ + Status: enrichmentFieldStatusExternal, + Source: enrichmentSourceExternal, + UpdatedAt: now, + } + + return + } + + if len(values) > 0 && result.Confidence >= threshold { + party.Certifications = values + meta[name] = EnrichmentFieldMeta{ + Confidence: result.Confidence, + SourceURL: sourceURL, + Status: enrichmentFieldStatusFound, + Source: enrichmentSourceEnrichment, + UpdatedAt: now, + } + + return + } + + status := enrichmentFieldStatusNotFound + if len(values) > 0 { + status = enrichmentFieldStatusLowConfidence + } + + meta[name] = EnrichmentFieldMeta{ + Confidence: result.Confidence, + SourceURL: sourceURL, + Status: status, + Source: enrichmentSourceEnrichment, + UpdatedAt: now, + } +} + +// normalizeCertifications trims, drops blanks, and de-duplicates the +// certification names returned by the agent, preserving order. +func normalizeCertifications(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + + for _, v := range values { + v = strings.TrimSpace(v) + if v == "" { + continue + } + + key := strings.ToLower(v) + if _, ok := seen[key]; ok { + continue + } + + seen[key] = struct{}{} + out = append(out, v) + } + + return out +} + +// scalarFields returns the descriptor list pairing each *string column +// with the resolved value from the two agents. website_url comes from +// the company-profile agent; the document and page URLs come from the +// compliance-docs agent. +func scalarFields( + company CompanyProfileResult, + compliance ComplianceDocsResult, +) []scalarField { + return []scalarField{ + { + name: "legal_name", + get: func(p *coredata.CommonThirdParty) *string { return p.LegalName }, + set: func(p *coredata.CommonThirdParty, v *string) { p.LegalName = v }, + result: company.LegalName, + }, + { + name: "headquarter_address", + get: func(p *coredata.CommonThirdParty) *string { return p.HeadquarterAddress }, + set: func(p *coredata.CommonThirdParty, v *string) { p.HeadquarterAddress = v }, + result: company.HeadquarterAddress, + }, + { + name: "website_url", + get: func(p *coredata.CommonThirdParty) *string { return p.WebsiteURL }, + set: func(p *coredata.CommonThirdParty, v *string) { p.WebsiteURL = v }, + result: company.WebsiteURL, + }, + { + name: "privacy_policy_url", + get: func(p *coredata.CommonThirdParty) *string { return p.PrivacyPolicyURL }, + set: func(p *coredata.CommonThirdParty, v *string) { p.PrivacyPolicyURL = v }, + result: compliance.PrivacyPolicyURL, + }, + { + name: "terms_of_service_url", + get: func(p *coredata.CommonThirdParty) *string { return p.TermsOfServiceURL }, + set: func(p *coredata.CommonThirdParty, v *string) { p.TermsOfServiceURL = v }, + result: compliance.TermsOfServiceURL, + }, + { + name: "service_level_agreement_url", + get: func(p *coredata.CommonThirdParty) *string { return p.ServiceLevelAgreementURL }, + set: func(p *coredata.CommonThirdParty, v *string) { p.ServiceLevelAgreementURL = v }, + result: compliance.ServiceLevelAgreementURL, + }, + { + name: "service_software_agreement_url", + get: func(p *coredata.CommonThirdParty) *string { return p.ServiceSoftwareAgreementURL }, + set: func(p *coredata.CommonThirdParty, v *string) { p.ServiceSoftwareAgreementURL = v }, + result: compliance.ServiceSoftwareAgreementURL, + }, + { + name: "data_processing_agreement_url", + get: func(p *coredata.CommonThirdParty) *string { return p.DataProcessingAgreementURL }, + set: func(p *coredata.CommonThirdParty, v *string) { p.DataProcessingAgreementURL = v }, + result: compliance.DataProcessingAgreementURL, + }, + { + name: "business_associate_agreement_url", + get: func(p *coredata.CommonThirdParty) *string { return p.BusinessAssociateAgreementURL }, + set: func(p *coredata.CommonThirdParty, v *string) { p.BusinessAssociateAgreementURL = v }, + result: compliance.BusinessAssociateAgreementURL, + }, + { + name: "subprocessors_list_url", + get: func(p *coredata.CommonThirdParty) *string { return p.SubprocessorsListURL }, + set: func(p *coredata.CommonThirdParty, v *string) { p.SubprocessorsListURL = v }, + result: compliance.SubprocessorsListURL, + }, + { + name: "status_page_url", + get: func(p *coredata.CommonThirdParty) *string { return p.StatusPageURL }, + set: func(p *coredata.CommonThirdParty, v *string) { p.StatusPageURL = v }, + result: compliance.StatusPageURL, + }, + { + name: "security_page_url", + get: func(p *coredata.CommonThirdParty) *string { return p.SecurityPageURL }, + set: func(p *coredata.CommonThirdParty, v *string) { p.SecurityPageURL = v }, + result: compliance.SecurityPageURL, + }, + { + name: "trust_page_url", + get: func(p *coredata.CommonThirdParty) *string { return p.TrustPageURL }, + set: func(p *coredata.CommonThirdParty, v *string) { p.TrustPageURL = v }, + result: compliance.TrustPageURL, + }, + } +} diff --git a/pkg/thirdparty/common_third_party_enrichment_worker.go b/pkg/thirdparty/common_third_party_enrichment_worker.go new file mode 100644 index 000000000..1760c3cf0 --- /dev/null +++ b/pkg/thirdparty/common_third_party_enrichment_worker.go @@ -0,0 +1,456 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package thirdparty + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "go.gearno.de/kit/httpclient" + "go.gearno.de/kit/log" + "go.gearno.de/kit/pg" + "go.gearno.de/kit/worker" + "go.probo.inc/probo/pkg/agent" + "go.probo.inc/probo/pkg/agent/tools/browser" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/filemanager" + "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/llm" +) + +const ( + // defaultEnrichmentAgentTimeout caps a single enrichment agent run. + // It is generous because Agent B browses several pages on top of the + // LLM round-trips. + defaultEnrichmentAgentTimeout = 90 * time.Second + + // defaultEnrichmentMaxTurns bounds an agent's reasoning loop (LLM + // call plus tool round-trips). Agent B may navigate the site footer + // and trust portal across several turns before synthesizing. + defaultEnrichmentMaxTurns = 12 + + // defaultEnrichmentMaxTokens caps agent output. The structured + // output is moderate; the budget leaves headroom for reasoning + // models whose reasoning tokens count against max_tokens. + defaultEnrichmentMaxTokens = 8192 + + // defaultEnrichmentConfidenceThreshold is the floor a resolved value + // must clear before it is written to its column. Values below it are + // recorded in the enrichment metadata but not promoted. + defaultEnrichmentConfidenceThreshold = 0.7 + + // defaultEnrichmentStaleAfter is the idle window after which a + // claimed-but-unfinished enrichment is re-armed. + defaultEnrichmentStaleAfter = 15 * time.Minute + + // defaultEnrichmentMaxAttempts caps how many times a row is retried + // before stale recovery leaves it alone, so a permanently failing + // row does not loop forever. + defaultEnrichmentMaxAttempts = 3 + + enrichmentLogoUserAgent = "Probo-Enricher/1.0" +) + +// EnrichmentConfig configures the common-third-party enrichment worker +// and the two agents it runs. The worker no-ops when LLMClient is nil; +// callers gate registration on config presence. Browser tools for Agent +// B are enabled only when ChromeAddr is set; otherwise it relies on +// web_search alone. Logo storage is enabled only when FileManager and +// Bucket are both set. +type EnrichmentConfig struct { + LLMClient *llm.Client + Model string + MaxTokens *int + Temperature *float64 + FirecrawlAPIKey string + ChromeAddr string + AgentTimeout time.Duration + MaxTurns int + ConfidenceThreshold float64 + StaleAfter time.Duration + MaxAttempts int + + FileManager *filemanager.Service + Bucket string +} + +func (c EnrichmentConfig) withDefaults() EnrichmentConfig { + if c.AgentTimeout <= 0 { + c.AgentTimeout = defaultEnrichmentAgentTimeout + } + + if c.MaxTurns < 1 { + c.MaxTurns = defaultEnrichmentMaxTurns + } + + if c.ConfidenceThreshold <= 0 { + c.ConfidenceThreshold = defaultEnrichmentConfidenceThreshold + } + + if c.StaleAfter <= 0 { + c.StaleAfter = defaultEnrichmentStaleAfter + } + + if c.MaxAttempts < 1 { + c.MaxAttempts = defaultEnrichmentMaxAttempts + } + + return c +} + +func resolveEnrichmentMaxTurns(configured int) int { + if configured > 0 { + return configured + } + + return defaultEnrichmentMaxTurns +} + +func resolveEnrichmentMaxTokens(configured *int) int { + if configured != nil && *configured > 0 { + return *configured + } + + return defaultEnrichmentMaxTokens +} + +type enrichmentHandler struct { + pg *pg.Client + logger *log.Logger + cfg EnrichmentConfig + companyAgent *agent.Agent + httpClient *http.Client +} + +// NewCommonThirdPartyEnrichmentWorker builds the worker that enriches +// global common_third_parties rows. It is a system worker: the catalog +// is not tenant-scoped, so a single enrichment benefits all tenants. The +// worker no-ops when no LLM client is configured. +func NewCommonThirdPartyEnrichmentWorker( + pgClient *pg.Client, + logger *log.Logger, + cfg EnrichmentConfig, + opts ...worker.Option, +) *worker.Worker[coredata.CommonThirdParty] { + cfg = cfg.withDefaults() + + h := &enrichmentHandler{ + pg: pgClient, + logger: logger, + cfg: cfg, + httpClient: newEnrichmentHTTPClient(), + } + + // Agent A has no browser, so it is built once and reused. Agent B is + // built per Process because it needs a per-run browser bound to the + // process context. + if cfg.LLMClient != nil { + h.companyAgent = buildCompanyProfileAgent(cfg, logger) + } + + return worker.New( + "common-third-party-enrichment-worker", + h, + logger, + opts..., + ) +} + +func (h *enrichmentHandler) Claim(ctx context.Context) (coredata.CommonThirdParty, error) { + var party coredata.CommonThirdParty + + if err := h.pg.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + if err := party.LoadNextForEnrichmentForUpdateSkipLocked(ctx, tx); err != nil { + return err + } + + return party.ClearEnrichmentRequestedAt(ctx, tx) + }, + ); err != nil { + if errors.Is(err, coredata.ErrResourceNotFound) { + return coredata.CommonThirdParty{}, worker.ErrNoTask + } + + return coredata.CommonThirdParty{}, fmt.Errorf("cannot claim common third party enrichment task: %w", err) + } + + return party, nil +} + +// Process runs the enrichment pipeline for one catalog row: Agent A +// (company profile) first, then Agent B (compliance docs) and the +// deterministic logo step, all outside any transaction. The merged +// result and per-field provenance are persisted in a single final +// transaction. Process always writes an enrichment payload, even on a +// no-result run, so stale recovery does not re-queue the row. +func (h *enrichmentHandler) Process(ctx context.Context, party coredata.CommonThirdParty) error { + if h.companyAgent == nil { + return nil + } + + now := time.Now() + prior := parseEnrichmentFields(party.Enrichment) + + var ( + runErrors []string + anySuccess bool + ) + + // Agent A first: it resolves website_url, which Agent B and the logo + // step depend on. + company, err := h.runCompanyProfile(ctx, party) + if err != nil { + h.logger.WarnCtx(ctx, "company profile agent failed", log.Error(err), log.String("common_third_party_id", party.ID.String())) + runErrors = append(runErrors, "company_profile: "+err.Error()) + } else { + anySuccess = true + } + + website := effectiveWebsiteURL(party, company, h.cfg.ConfidenceThreshold) + legalName := effectiveLegalName(party, company, h.cfg.ConfidenceThreshold) + + // Agent B: compliance documents and trust pages. + compliance, err := h.runComplianceDocs(ctx, party.Name, website, legalName) + if err != nil { + h.logger.WarnCtx(ctx, "compliance docs agent failed", log.Error(err), log.String("common_third_party_id", party.ID.String())) + runErrors = append(runErrors, "compliance_docs: "+err.Error()) + } else { + anySuccess = true + } + + // Deterministic logo step (no LLM). Uploads to S3 outside the final + // transaction; the File row is inserted below. + logoFile := h.prepareLogo(ctx, party, website) + + meta := make(map[string]EnrichmentFieldMeta) + + for _, field := range scalarFields(company, compliance) { + applyScalarField(&party, meta, prior, field, h.cfg.ConfidenceThreshold, now) + } + + applyCertifications(&party, meta, prior, compliance.Certifications, h.cfg.ConfidenceThreshold, now) + + status := enrichmentStatusDone + switch { + case !anySuccess: + status = enrichmentStatusFailed + case len(runErrors) > 0: + status = enrichmentStatusPartial + } + + payload := EnrichmentMetadata{ + Model: h.cfg.Model, + AttemptedAt: now, + Status: status, + Error: strings.Join(runErrors, "; "), + Fields: meta, + } + + raw, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("cannot marshal enrichment metadata: %w", err) + } + + party.Enrichment = raw + party.UpdatedAt = now + + return h.pg.WithTx( + ctx, + func(ctx context.Context, tx pg.Tx) error { + if logoFile != nil { + if err := logoFile.Insert(ctx, tx, coredata.NewScope(gid.NilTenant)); err != nil { + return fmt.Errorf("cannot insert common third party logo file: %w", err) + } + + party.LogoFileID = &logoFile.ID + + if err := party.UpdateLogoFileID(ctx, tx); err != nil { + return fmt.Errorf("cannot update common third party logo: %w", err) + } + } + + if err := party.UpdateEnrichment(ctx, tx); err != nil { + return fmt.Errorf("cannot persist common third party enrichment: %w", err) + } + + h.logger.InfoCtx( + ctx, + "enriched common third party", + log.String("common_third_party_id", party.ID.String()), + log.String("name", party.Name), + log.String("status", status), + log.Bool("logo_stored", logoFile != nil), + ) + + return nil + }, + ) +} + +// RecoverStale re-arms enrichment for rows whose run was claimed but +// never finished. Claim clears enrichment_requested_at up front, so a +// crash between phases would otherwise strand the row. +func (h *enrichmentHandler) RecoverStale(ctx context.Context) error { + return h.pg.WithConn( + ctx, + func(ctx context.Context, conn pg.Querier) error { + if err := coredata.ResetStaleCommonThirdPartyEnrichments(ctx, conn, h.cfg.StaleAfter, h.cfg.MaxAttempts); err != nil { + return fmt.Errorf("cannot reset stale common third party enrichments: %w", err) + } + + return nil + }, + ) +} + +func (h *enrichmentHandler) runCompanyProfile( + ctx context.Context, + party coredata.CommonThirdParty, +) (CompanyProfileResult, error) { + prompt := buildCompanyProfilePrompt(party) + + agentCtx, cancel := context.WithTimeout(ctx, h.cfg.AgentTimeout) + defer cancel() + + result, err := agent.RunTyped[CompanyProfileResult]( + agentCtx, + h.companyAgent, + []llm.Message{ + { + Role: llm.RoleUser, + Parts: []llm.Part{llm.TextPart{Text: prompt}}, + }, + }, + ) + if err != nil { + return CompanyProfileResult{}, fmt.Errorf("company profile agent run failed: %w", err) + } + + return result.Output, nil +} + +// runComplianceDocs builds Agent B with a per-run browser when a Chrome +// endpoint is configured, then runs it. The browser is closed when the +// run returns. The browser is intentionally not pinned to the vendor +// domain so the agent can follow links to hosted trust portals (Vanta, +// SafeBase, etc.); SSRF protection still blocks non-public hosts. +func (h *enrichmentHandler) runComplianceDocs( + ctx context.Context, + name string, + website string, + legalName string, +) (ComplianceDocsResult, error) { + var browserTools []agent.Tool + + if h.cfg.ChromeAddr != "" { + webBrowser := browser.NewBrowser(ctx, h.cfg.ChromeAddr) + defer webBrowser.Close() + + browserTools = browser.NewReadOnlyToolset(webBrowser).Tools() + } + + complianceAgent := buildComplianceDocsAgent(h.cfg, h.logger, browserTools) + + prompt := buildComplianceDocsPrompt(name, website, legalName) + + agentCtx, cancel := context.WithTimeout(ctx, h.cfg.AgentTimeout) + defer cancel() + + result, err := agent.RunTyped[ComplianceDocsResult]( + agentCtx, + complianceAgent, + []llm.Message{ + { + Role: llm.RoleUser, + Parts: []llm.Part{llm.TextPart{Text: prompt}}, + }, + }, + ) + if err != nil { + return ComplianceDocsResult{}, fmt.Errorf("compliance docs agent run failed: %w", err) + } + + return result.Output, nil +} + +// effectiveWebsiteURL is the website passed to Agent B and the logo step. +// A curated value already on the row wins (seed data and human edits are +// trusted); otherwise Agent A's value is used when it clears the +// confidence threshold. +func effectiveWebsiteURL( + party coredata.CommonThirdParty, + company CompanyProfileResult, + threshold float64, +) string { + if party.WebsiteURL != nil { + if v := strings.TrimSpace(*party.WebsiteURL); v != "" { + return v + } + } + + if v := strings.TrimSpace(company.WebsiteURL.Value); v != "" && company.WebsiteURL.Confidence >= threshold { + return v + } + + return "" +} + +// effectiveLegalName is the legal name hint passed to Agent B, resolved +// the same way as effectiveWebsiteURL. +func effectiveLegalName( + party coredata.CommonThirdParty, + company CompanyProfileResult, + threshold float64, +) string { + if party.LegalName != nil { + if v := strings.TrimSpace(*party.LegalName); v != "" { + return v + } + } + + if v := strings.TrimSpace(company.LegalName.Value); v != "" && company.LegalName.Confidence >= threshold { + return v + } + + return "" +} + +type userAgentRoundTripper struct { + next http.RoundTripper +} + +func (t *userAgentRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + r2 := r.Clone(r.Context()) + r2.Header.Set("User-Agent", enrichmentLogoUserAgent) + + return t.next.RoundTrip(r2) +} + +// newEnrichmentHTTPClient builds the SSRF-protected client used by the +// deterministic logo step. +func newEnrichmentHTTPClient() *http.Client { + client := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection()) + client.Timeout = 20 * time.Second + client.Transport = &userAgentRoundTripper{next: client.Transport} + + return client +} diff --git a/pkg/thirdparty/common_third_party_logo.go b/pkg/thirdparty/common_third_party_logo.go new file mode 100644 index 000000000..f34aca267 --- /dev/null +++ b/pkg/thirdparty/common_third_party_logo.go @@ -0,0 +1,207 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package thirdparty + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "go.gearno.de/crypto/uuid" + "go.gearno.de/kit/log" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/gid" + "go.probo.inc/probo/pkg/webinspect" +) + +// maxLogoSize caps a downloaded logo image. Logos are small; the cap +// guards against an oversized or malicious response. +const maxLogoSize = 5 << 20 // 5 MiB + +// prepareLogo discovers and uploads a vendor logo to S3, returning a +// fully-populated (but not yet inserted) File record for the caller to +// persist in its transaction. It is deterministic and best-effort: any +// failure logs and returns (nil, nil) so a missing logo never fails the +// enrichment run. +// +// It no-ops when the row already has a logo, when logo storage is not +// configured, or when no website is known. The S3 upload happens here, +// outside any transaction; the caller inserts the File row and links it +// via UpdateLogoFileID. +func (h *enrichmentHandler) prepareLogo( + ctx context.Context, + party coredata.CommonThirdParty, + websiteURL string, +) *coredata.File { + if party.LogoFileID != nil { + return nil + } + + if h.cfg.FileManager == nil || h.cfg.Bucket == "" { + return nil + } + + website := strings.TrimSpace(websiteURL) + if website == "" { + return nil + } + + data, contentType, err := fetchCommonThirdPartyLogo(ctx, h.httpClient, website) + if err != nil { + h.logger.InfoCtx( + ctx, + "could not fetch common third party logo", + log.String("common_third_party_id", party.ID.String()), + log.Error(err), + ) + + return nil + } + + objectKey, err := uuid.NewV7() + if err != nil { + h.logger.WarnCtx(ctx, "cannot generate logo object key", log.Error(err)) + + return nil + } + + now := time.Now() + fileRecord := &coredata.File{ + ID: gid.New(gid.NilTenant, coredata.FileEntityType), + OrganizationID: gid.Nil, + BucketName: h.cfg.Bucket, + MimeType: contentType, + FileName: party.Name + "-logo" + webinspect.ExtensionForMIME(contentType), + FileKey: objectKey.String(), + FileSize: int64(len(data)), + Visibility: coredata.FileVisibilityPublic, + CreatedAt: now, + UpdatedAt: now, + } + + size, err := h.cfg.FileManager.PutFile( + ctx, + fileRecord, + bytes.NewReader(data), + map[string]string{ + "type": "common-third-party-logo", + "common-third-party-id": party.ID.String(), + }, + ) + if err != nil { + h.logger.WarnCtx( + ctx, + "cannot upload common third party logo", + log.String("common_third_party_id", party.ID.String()), + log.Error(err), + ) + + return nil + } + + fileRecord.FileSize = size + + return fileRecord +} + +// fetchCommonThirdPartyLogo finds the best logo for a website and +// downloads it. It first parses the page's for icon links +// (webinspect), then falls back to well-known icon paths on the same +// host. The supplied client must enforce SSRF protection. +func fetchCommonThirdPartyLogo( + ctx context.Context, + client *http.Client, + websiteURL string, +) (data []byte, contentType string, err error) { + candidates := make([]string, 0, 3) + + pageInfo, parseErr := webinspect.Parse(ctx, client, websiteURL) + if parseErr == nil { + if logoURL, logoErr := webinspect.FindLogoURL(pageInfo); logoErr == nil { + candidates = append(candidates, logoURL) + } + } + + if parsed, parseURLErr := url.Parse(websiteURL); parseURLErr == nil && parsed.Host != "" { + base := url.URL{Scheme: parsed.Scheme, Host: parsed.Host} + if base.Scheme == "" { + base.Scheme = "https" + } + + candidates = append( + candidates, + base.ResolveReference(&url.URL{Path: "/apple-touch-icon.png"}).String(), + base.ResolveReference(&url.URL{Path: "/favicon.ico"}).String(), + ) + } + + for _, candidate := range candidates { + data, contentType, err = downloadImage(ctx, client, candidate) + if err == nil { + return data, contentType, nil + } + } + + return nil, "", fmt.Errorf("cannot fetch logo for %s", websiteURL) +} + +// downloadImage fetches a single candidate URL and returns its bytes when +// the response is a non-empty image within the size cap. +func downloadImage( + ctx context.Context, + client *http.Client, + rawURL string, +) ([]byte, string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return nil, "", fmt.Errorf("cannot create logo request: %w", err) + } + + resp, err := client.Do(req) + if err != nil { + return nil, "", fmt.Errorf("cannot fetch logo: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, "", fmt.Errorf("cannot fetch logo: status %d", resp.StatusCode) + } + + contentType := resp.Header.Get("Content-Type") + if idx := strings.Index(contentType, ";"); idx != -1 { + contentType = contentType[:idx] + } + contentType = strings.TrimSpace(contentType) + + if !strings.HasPrefix(contentType, "image/") { + return nil, "", fmt.Errorf("logo response is not an image: %q", contentType) + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, maxLogoSize)) + if err != nil { + return nil, "", fmt.Errorf("cannot read logo body: %w", err) + } + + if len(body) == 0 { + return nil, "", fmt.Errorf("logo response is empty") + } + + return body, contentType, nil +} diff --git a/pkg/thirdparty/prompts/common_third_party_company_profile.txt.tmpl b/pkg/thirdparty/prompts/common_third_party_company_profile.txt.tmpl new file mode 100644 index 000000000..8a84864bd --- /dev/null +++ b/pkg/thirdparty/prompts/common_third_party_company_profile.txt.tmpl @@ -0,0 +1,33 @@ +You are a research agent that builds a factual company profile for a software +vendor or service provider. You are given the vendor's name (and sometimes a +known website). Return only verifiable identity facts. + +Resolve these fields: + +- legal_name: the full legal entity name, including the suffix (Inc., Ltd., + GmbH, S.A.S., etc.). Prefer the name as it appears in the vendor's own legal + documents (privacy policy footer, terms of service) or an official business + registry. +- headquarter_address: the postal address of the company's headquarters. +- website_url: the canonical primary marketing website. Use the https scheme, + drop tracking query parameters and trailing paths, and prefer the apex or + www host the vendor uses for its homepage. + +Method: + +- Use the web_search tool when it is available to confirm facts. Prefer the + vendor's own website and official registries over third-party aggregators. +- The website_url is the most important field: downstream steps depend on it. + Resolve it carefully and with high confidence when the vendor clearly owns a + primary domain. + +Rules: + +- Never guess. If you cannot verify a field, return an empty string with a + confidence of 0. +- confidence is your own 0.0-1.0 estimate that the value is correct. Reserve + values above 0.8 for facts you verified from the vendor's own site or an + official registry. +- source_url is the page where you verified the value. Leave it empty when the + value was not found. +- Do not include commentary; return only the structured fields. diff --git a/pkg/thirdparty/prompts/common_third_party_compliance_docs.txt.tmpl b/pkg/thirdparty/prompts/common_third_party_compliance_docs.txt.tmpl new file mode 100644 index 000000000..c8073481d --- /dev/null +++ b/pkg/thirdparty/prompts/common_third_party_compliance_docs.txt.tmpl @@ -0,0 +1,51 @@ +You are a research agent that locates a software vendor's public compliance +documents and trust pages. You are given the vendor's name and, usually, its +website. Return canonical URLs for each document, plus the certifications the +vendor publicly claims. + +Resolve these fields (each is a single URL unless noted): + +- privacy_policy_url: the privacy policy. +- terms_of_service_url: the terms of service / terms of use. +- service_level_agreement_url: the public SLA. Frequently gated behind sales. +- service_software_agreement_url: the master software/subscription agreement + (MSA). Often gated, or the same document as the terms of service. +- data_processing_agreement_url: the DPA. Often a downloadable PDF; sometimes + only available on request. +- business_associate_agreement_url: the HIPAA BAA. Almost always gated behind + sales or an enterprise plan. +- subprocessors_list_url: the sub-processors list page. +- status_page_url: the uptime/status page (commonly status. or a + hosted statuspage.io / instatus / better-uptime page). +- security_page_url: the security overview page. +- trust_page_url: the trust center / trust portal. Many vendors host this on + Vanta, SafeBase, Drata, Conveyor, or a /trust path. +- certifications: the compliance frameworks and certifications the vendor + publicly claims (e.g. SOC 2 Type II, ISO 27001, ISO 27701, PCI DSS, HIPAA, + GDPR, FedRAMP). Read these from the trust or security page. + +Method: + +- Start from the vendor's when provided. Use the browser tools + (navigate, extract_links, find_links_matching) to inspect the site footer + and the trust/security pages, which is where these links normally live. +- Use web_search to fill gaps with site-scoped queries (for example + "site: data processing agreement"). Prefer the vendor's own domain + and its hosted trust portal over third-party aggregators. +- Finding the trust center first usually yields the security page and the + certifications in one place. + +Rules: + +- Return the most specific canonical URL. Prefer a direct document/page URL + over a generic legal-index page. +- Never guess or fabricate a URL. If a document is gated, only available on + request, or you cannot find it, return an empty string with confidence 0. + Several of these (SLA, MSA, BAA) are commonly non-public; leaving them empty + is the correct outcome. +- confidence is your own 0.0-1.0 estimate that the URL is correct and current. + Reserve values above 0.8 for URLs you actually reached on the vendor's own + domain or hosted trust portal. +- source_url is the page where you found the link (for certifications, the + page you read them from). Leave it empty when nothing was found. +- Do not include commentary; return only the structured fields. diff --git a/pkg/thirdparty/resolver.go b/pkg/thirdparty/resolver.go index 0325aba9c..aa81d1338 100644 --- a/pkg/thirdparty/resolver.go +++ b/pkg/thirdparty/resolver.go @@ -70,8 +70,14 @@ func ResolveOrCreateCommonThirdParty( Slug: partySlug, Category: category, Certifications: []string{}, - CreatedAt: now, - UpdatedAt: now, + // Request enrichment at creation: a freshly resolved catalog row + // carries only name/slug/category, so the enrichment worker fills + // the rest (URLs, address, certifications, logo). Curated seed + // rows are inserted via Upsert without this flag, so a full + // re-seed does not trigger an enrichment storm. + EnrichmentRequestedAt: &now, + CreatedAt: now, + UpdatedAt: now, } // Insert inside a savepoint so a concurrent transaction that created diff --git a/pkg/webinspect/logo.go b/pkg/webinspect/logo.go new file mode 100644 index 000000000..2bba7c162 --- /dev/null +++ b/pkg/webinspect/logo.go @@ -0,0 +1,180 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package webinspect + +import ( + "fmt" + "strconv" + "strings" + + "golang.org/x/net/html" +) + +func FindLogoURL(info *PageInfo) (string, error) { + head := findElement(info.Root, "head") + if head == nil { + return "", fmt.Errorf("cannot find logo: no head element") + } + + var ( + svgIcon string + appleTouchIcon string + appleTouchSize int + largestIcon string + largestSize int + msTileImage string + ) + + for _, n := range findAllIn(head, "link") { + rel := strings.ToLower(attrVal(n, "rel")) + href := attrVal(n, "href") + if href == "" { + continue + } + + switch { + case strings.Contains(rel, "icon") && !strings.Contains(rel, "apple-touch-icon") && attrVal(n, "type") == "image/svg+xml": + svgIcon = href + case strings.Contains(rel, "apple-touch-icon"): + size := parseSizeAttr(attrVal(n, "sizes")) + if appleTouchIcon == "" || size > appleTouchSize { + appleTouchIcon = href + appleTouchSize = size + } + case strings.Contains(rel, "icon") && !strings.Contains(rel, "apple-touch-icon"): + size := parseSizeAttr(attrVal(n, "sizes")) + if largestIcon == "" || size > largestSize { + largestIcon = href + largestSize = size + } + } + } + + for _, n := range findAllIn(head, "meta") { + name := strings.ToLower(attrVal(n, "name")) + content := attrVal(n, "content") + if name == "msapplication-tileimage" && content != "" { + msTileImage = content + } + } + + candidates := []string{ + svgIcon, + appleTouchIcon, + largestIcon, + msTileImage, + } + + for _, href := range candidates { + if href != "" { + return info.ResolveHref(href), nil + } + } + + return "", fmt.Errorf("cannot find logo") +} + +func parseSizeAttr(sizes string) int { + if sizes == "" || strings.EqualFold(sizes, "any") { + return 0 + } + + best := 0 + for token := range strings.FieldsSeq(sizes) { + token = strings.ToLower(token) + parts := strings.SplitN(token, "x", 2) + if len(parts) != 2 { + continue + } + + w, err := strconv.Atoi(parts[0]) + if err != nil { + continue + } + + if w > best { + best = w + } + } + + return best +} + +func ExtensionForMIME(contentType string) string { + ct := strings.ToLower(contentType) + if idx := strings.Index(ct, ";"); idx != -1 { + ct = ct[:idx] + } + ct = strings.TrimSpace(ct) + + switch ct { + case "image/svg+xml": + return ".svg" + case "image/png": + return ".png" + case "image/jpeg": + return ".jpg" + case "image/gif": + return ".gif" + case "image/webp": + return ".webp" + case "image/x-icon", "image/vnd.microsoft.icon": + return ".ico" + default: + return ".png" + } +} + +// HeadLinks returns all nodes from whose rel attribute +// contains the given value (case-insensitive partial match). +func (p *PageInfo) HeadLinks(rel string) []*html.Node { + head := findElement(p.Root, "head") + if head == nil { + return nil + } + + rel = strings.ToLower(rel) + var matches []*html.Node + + for _, n := range findAllIn(head, "link") { + if strings.Contains(strings.ToLower(attrVal(n, "rel")), rel) { + matches = append(matches, n) + } + } + + return matches +} + +// HeadMeta returns the content attribute of the first tag in +// whose name attribute matches (case-insensitive). +func (p *PageInfo) HeadMeta(name string) (string, bool) { + head := findElement(p.Root, "head") + if head == nil { + return "", false + } + + name = strings.ToLower(name) + + for _, n := range findAllIn(head, "meta") { + if strings.EqualFold(attrVal(n, "name"), name) { + content := attrVal(n, "content") + if content != "" { + return content, true + } + } + } + + return "", false +} diff --git a/pkg/webinspect/logo_test.go b/pkg/webinspect/logo_test.go new file mode 100644 index 000000000..77b06d8b3 --- /dev/null +++ b/pkg/webinspect/logo_test.go @@ -0,0 +1,163 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package webinspect_test + +import ( + "net/url" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/webinspect" +) + +func parseTestHTML(t *testing.T, rawURL string, body string) *webinspect.PageInfo { + t.Helper() + + u, err := url.Parse(rawURL) + require.NoError(t, err) + + info, err := webinspect.ParseHTML(u, strings.NewReader(body)) + require.NoError(t, err) + + return info +} + +func TestFindLogoURL_SVGPreferred(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://example.com", ` + + + + `) + + got, err := webinspect.FindLogoURL(info) + require.NoError(t, err) + assert.Equal(t, "https://example.com/favicon.svg", got) +} + +func TestFindLogoURL_AppleTouchIconSecond(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://example.com", ` + + + `) + + got, err := webinspect.FindLogoURL(info) + require.NoError(t, err) + assert.Equal(t, "https://example.com/apple-touch-icon.png", got) +} + +func TestFindLogoURL_AppleTouchIconLargest(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://example.com", ` + + + + `) + + got, err := webinspect.FindLogoURL(info) + require.NoError(t, err) + assert.Equal(t, "https://example.com/touch-180.png", got) +} + +func TestFindLogoURL_LargestIcon(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://example.com", ` + + + + `) + + got, err := webinspect.FindLogoURL(info) + require.NoError(t, err) + assert.Equal(t, "https://example.com/icon-192.png", got) +} + +func TestFindLogoURL_MsTileImage(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://example.com", ` + + `) + + got, err := webinspect.FindLogoURL(info) + require.NoError(t, err) + assert.Equal(t, "https://example.com/mstile-144.png", got) +} + +func TestFindLogoURL_RelativeHrefResolved(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://cdn.example.com/app/", ` + + `) + + got, err := webinspect.FindLogoURL(info) + require.NoError(t, err) + assert.Equal(t, "https://cdn.example.com/assets/logo.svg", got) +} + +func TestFindLogoURL_NoHead(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://example.com", `

no head

`) + + _, err := webinspect.FindLogoURL(info) + assert.Error(t, err) +} + +func TestFindLogoURL_NothingFound(t *testing.T) { + t.Parallel() + + info := parseTestHTML(t, "https://example.com", ``) + + _, err := webinspect.FindLogoURL(info) + assert.Error(t, err) +} + +func TestExtensionForMIME(t *testing.T) { + t.Parallel() + + tests := []struct { + contentType string + expected string + }{ + {"image/svg+xml", ".svg"}, + {"image/png", ".png"}, + {"image/jpeg", ".jpg"}, + {"image/gif", ".gif"}, + {"image/webp", ".webp"}, + {"image/x-icon", ".ico"}, + {"image/vnd.microsoft.icon", ".ico"}, + {"image/png; charset=utf-8", ".png"}, + {"application/octet-stream", ".png"}, + } + + for _, tt := range tests { + t.Run( + tt.contentType, + func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.expected, webinspect.ExtensionForMIME(tt.contentType)) + }, + ) + } +} diff --git a/pkg/webinspect/parse.go b/pkg/webinspect/parse.go new file mode 100644 index 000000000..7ae20659f --- /dev/null +++ b/pkg/webinspect/parse.go @@ -0,0 +1,117 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +// Package webinspect parses a web page's static HTML to extract metadata +// from its (icons/logos, meta tags, link relations). It is a pure, +// deterministic helper: callers supply an http.Client (e.g. one with SSRF +// protection) so the package itself makes no policy decisions about which +// hosts are reachable. The logo discovery is reused by the common +// third-party enricher to populate logo_file_id without an LLM. +package webinspect + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + + "golang.org/x/net/html" +) + +type PageInfo struct { + URL *url.URL + Root *html.Node +} + +func Parse(ctx context.Context, client *http.Client, websiteURL string) (*PageInfo, error) { + parsed, err := url.Parse(websiteURL) + if err != nil { + return nil, fmt.Errorf("cannot parse website URL: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, websiteURL, nil) + if err != nil { + return nil, fmt.Errorf("cannot create request: %w", err) + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("cannot fetch page: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("cannot fetch page: status %d", resp.StatusCode) + } + + const maxHTMLSize = 10 << 20 // 10 MiB + return ParseHTML(parsed, io.LimitReader(resp.Body, maxHTMLSize)) +} + +func ParseHTML(baseURL *url.URL, r io.Reader) (*PageInfo, error) { + root, err := html.Parse(r) + if err != nil { + return nil, fmt.Errorf("cannot parse HTML: %w", err) + } + + return &PageInfo{URL: baseURL, Root: root}, nil +} + +func (p *PageInfo) ResolveHref(href string) string { + ref, err := url.Parse(href) + if err != nil { + return href + } + + return p.URL.ResolveReference(ref).String() +} + +func findElement(n *html.Node, tag string) *html.Node { + if n.Type == html.ElementNode && n.Data == tag { + return n + } + + for c := n.FirstChild; c != nil; c = c.NextSibling { + if found := findElement(c, tag); found != nil { + return found + } + } + + return nil +} + +func findAllIn(parent *html.Node, tag string) []*html.Node { + var nodes []*html.Node + + for c := parent.FirstChild; c != nil; c = c.NextSibling { + if c.Type == html.ElementNode && c.Data == tag { + nodes = append(nodes, c) + } + + nodes = append(nodes, findAllIn(c, tag)...) + } + + return nodes +} + +func attrVal(n *html.Node, key string) string { + for _, a := range n.Attr { + if a.Key == key { + return a.Val + } + } + + return "" +}