Add common third party enricher worker
Introduce a poll-based worker that fills the global common_third_parties catalog (URLs, headquarter address, legal name, certifications, logo) so each tenant no longer starts from sparse, name-only rows. Enrichment is requested at row creation by ResolveOrCreateCommonThirdParty; curated seed rows are not enqueued, to avoid a re-seed storm. The pipeline uses two specialized agents plus a deterministic logo step. Agent A (company profile) resolves legal name, headquarter address, and the canonical website over web search; its website and legal name feed Agent B and the logo step. Agent B (compliance docs) resolves the legal document URLs, trust/security/status pages, and certifications using the browser read-only toolset (gated on ChromeDPAddr) plus web search. The logo step restores pkg/webinspect as a pure deterministic package and stores the discovered icon in S3, linked via logo_file_id. Each agent returns per-field value/confidence/source_url. The worker writes a column only when confidence clears a configurable threshold and the field is not externally owned (seed or human), and always records full per-field provenance in a new enrichment JSONB column so re-runs fill only gaps and human edits are never clobbered. New bookkeeping columns (enrichment_requested_at, enrichment, enrichment_attempts) back the claim queue and stale recovery; agents run outside transactions and results persist in one final transaction. The worker is opt-in: it no-ops unless its agent provider is configured. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -378,6 +378,52 @@ spec:
|
||||
- name: COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS
|
||||
value: {{ .Values.probo.commonPatternEnrichmentWorker.agentMaxTurns | quote }}
|
||||
{{- end }}
|
||||
# Common Third Party Enrichment Agent
|
||||
{{- if .Values.probo.commonThirdPartyEnrichment.provider }}
|
||||
- name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_PROVIDER
|
||||
value: {{ .Values.probo.commonThirdPartyEnrichment.provider | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.probo.commonThirdPartyEnrichment.modelName }}
|
||||
- name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MODEL_NAME
|
||||
value: {{ .Values.probo.commonThirdPartyEnrichment.modelName | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.probo.commonThirdPartyEnrichment.temperature }}
|
||||
- name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_TEMPERATURE
|
||||
value: {{ .Values.probo.commonThirdPartyEnrichment.temperature | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.probo.commonThirdPartyEnrichment.maxTokens }}
|
||||
- name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MAX_TOKENS
|
||||
value: {{ .Values.probo.commonThirdPartyEnrichment.maxTokens | quote }}
|
||||
{{- end }}
|
||||
# Common Third Party Enrichment Worker
|
||||
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.interval }}
|
||||
- name: COMMON_THIRD_PARTY_ENRICHMENT_INTERVAL
|
||||
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.interval | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.maxConcurrency }}
|
||||
- name: COMMON_THIRD_PARTY_ENRICHMENT_MAX_CONCURRENCY
|
||||
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.maxConcurrency | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.staleAfter }}
|
||||
- name: COMMON_THIRD_PARTY_ENRICHMENT_STALE_AFTER
|
||||
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.staleAfter | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.agentTimeout }}
|
||||
- name: COMMON_THIRD_PARTY_ENRICHMENT_AGENT_TIMEOUT
|
||||
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.agentTimeout | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.agentMaxTurns }}
|
||||
- name: COMMON_THIRD_PARTY_ENRICHMENT_AGENT_MAX_TURNS
|
||||
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.agentMaxTurns | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.confidenceThreshold }}
|
||||
- name: COMMON_THIRD_PARTY_ENRICHMENT_CONFIDENCE_THRESHOLD
|
||||
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.confidenceThreshold | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.maxAttempts }}
|
||||
- name: COMMON_THIRD_PARTY_ENRICHMENT_MAX_ATTEMPTS
|
||||
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.maxAttempts | quote }}
|
||||
{{- end }}
|
||||
# Custom Domains
|
||||
{{- if .Values.probo.customDomains.enabled }}
|
||||
- name: CUSTOM_DOMAINS_RENEWAL_INTERVAL
|
||||
|
||||
@@ -229,6 +229,26 @@ probo:
|
||||
# agentTimeout: 45
|
||||
# agentMaxTurns: 10
|
||||
|
||||
# Common third party enrichment agent (optional, fills common_third_parties
|
||||
# metadata: URLs, address, certifications, logo).
|
||||
# commonThirdPartyEnrichment:
|
||||
# provider: "openai"
|
||||
# modelName: "gpt-4o"
|
||||
# temperature: "0.1"
|
||||
# maxTokens: "8192"
|
||||
|
||||
# Common third party enrichment worker tuning (optional; seconds for
|
||||
# interval/staleAfter/agentTimeout). confidenceThreshold is the 0-1 floor
|
||||
# a value must clear before it is written.
|
||||
# commonThirdPartyEnrichmentWorker:
|
||||
# interval: 10
|
||||
# maxConcurrency: 1
|
||||
# staleAfter: 900
|
||||
# agentTimeout: 90
|
||||
# agentMaxTurns: 12
|
||||
# confidenceThreshold: 0.7
|
||||
# maxAttempts: 3
|
||||
|
||||
# OpenTelemetry tracing (optional)
|
||||
tracing:
|
||||
enabled: true
|
||||
|
||||
@@ -331,6 +331,30 @@ probo:
|
||||
agentTimeout: 45
|
||||
agentMaxTurns: 10
|
||||
|
||||
# Common third party enrichment agent (optional, requires a provider
|
||||
# key). Powers the company-profile and compliance-docs agents that fill
|
||||
# common_third_parties metadata. Browser tools for the compliance-docs
|
||||
# agent use the chromeDpAddr endpoint when set; web search uses
|
||||
# firecrawl.apiKey when set.
|
||||
commonThirdPartyEnrichment:
|
||||
provider: ""
|
||||
modelName: ""
|
||||
temperature: ""
|
||||
maxTokens: ""
|
||||
|
||||
# Common third party enrichment background worker tuning (optional).
|
||||
# interval, staleAfter, and agentTimeout are in seconds.
|
||||
# confidenceThreshold is the floor (0-1) a resolved value must clear
|
||||
# before it is written to its column.
|
||||
commonThirdPartyEnrichmentWorker:
|
||||
interval: 10
|
||||
maxConcurrency: 1
|
||||
staleAfter: 900
|
||||
agentTimeout: 90
|
||||
agentMaxTurns: 12
|
||||
confidenceThreshold: 0.7
|
||||
maxAttempts: 3
|
||||
|
||||
# Custom domains configuration (optional)
|
||||
customDomains:
|
||||
enabled: false
|
||||
|
||||
@@ -242,6 +242,15 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
|
||||
Temperature: b.getEnvFloatPtr("AGENT_TRACKER_ENRICHMENT_TEMPERATURE"),
|
||||
MaxTokens: new(b.getEnvIntOrDefault("AGENT_TRACKER_ENRICHMENT_MAX_TOKENS", 4096)),
|
||||
},
|
||||
CommonThirdPartyEnrichment: probodconfig.LLMAgentConfig{
|
||||
Provider: b.getEnvOrDefault("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_PROVIDER", ""),
|
||||
ModelName: b.getEnvOrDefault("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MODEL_NAME", ""),
|
||||
// Agent B browses pages and emits a moderate structured
|
||||
// output; the budget must leave headroom for reasoning
|
||||
// models whose reasoning tokens count against max_tokens.
|
||||
Temperature: b.getEnvFloatPtr("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_TEMPERATURE"),
|
||||
MaxTokens: new(b.getEnvIntOrDefault("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MAX_TOKENS", 8192)),
|
||||
},
|
||||
Tools: probodconfig.AgentToolsConfig{
|
||||
FirecrawlAPIKey: b.getEnv("FIRECRAWL_API_KEY"),
|
||||
},
|
||||
@@ -292,6 +301,15 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
|
||||
AgentTimeout: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_AGENT_TIMEOUT", 45),
|
||||
AgentMaxTurns: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS", 10),
|
||||
},
|
||||
CommonThirdPartyEnrichmentWorker: probodconfig.CommonThirdPartyEnrichmentWorkerConfig{
|
||||
Interval: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_INTERVAL", 10),
|
||||
MaxConcurrency: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_MAX_CONCURRENCY", 1),
|
||||
StaleAfter: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_STALE_AFTER", 900),
|
||||
AgentTimeout: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_AGENT_TIMEOUT", 90),
|
||||
AgentMaxTurns: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_AGENT_MAX_TURNS", 12),
|
||||
ConfidenceThreshold: b.getEnvFloatOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_CONFIDENCE_THRESHOLD", 0.7),
|
||||
MaxAttempts: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_MAX_ATTEMPTS", 3),
|
||||
},
|
||||
Branding: b.getEnvBoolOrDefault("BRANDING", true),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -245,6 +245,13 @@ func TestBuilder_Build_Defaults(t *testing.T) {
|
||||
assert.Equal(t, 600, cfg.Probod.CommonPatternEnrichmentWorker.StaleAfter)
|
||||
assert.Equal(t, 45, cfg.Probod.CommonPatternEnrichmentWorker.AgentTimeout)
|
||||
assert.Equal(t, 10, cfg.Probod.CommonPatternEnrichmentWorker.AgentMaxTurns)
|
||||
assert.Equal(t, 10, cfg.Probod.CommonThirdPartyEnrichmentWorker.Interval)
|
||||
assert.Equal(t, 1, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxConcurrency)
|
||||
assert.Equal(t, 900, cfg.Probod.CommonThirdPartyEnrichmentWorker.StaleAfter)
|
||||
assert.Equal(t, 90, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentTimeout)
|
||||
assert.Equal(t, 12, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentMaxTurns)
|
||||
assert.Equal(t, 0.7, cfg.Probod.CommonThirdPartyEnrichmentWorker.ConfidenceThreshold)
|
||||
assert.Equal(t, 3, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxAttempts)
|
||||
assert.Equal(t, 10, cfg.Probod.ThirdPartyVetting.Interval)
|
||||
assert.Equal(t, 1500, cfg.Probod.ThirdPartyVetting.StaleAfter)
|
||||
assert.Equal(t, 1, cfg.Probod.ThirdPartyVetting.MaxConcurrency)
|
||||
@@ -369,6 +376,17 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
|
||||
env["COMMON_PATTERN_ENRICHMENT_STALE_AFTER"] = "900"
|
||||
env["COMMON_PATTERN_ENRICHMENT_AGENT_TIMEOUT"] = "50"
|
||||
env["COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS"] = "5"
|
||||
// Common third party enrichment agent + worker tuning override
|
||||
env["AGENT_COMMON_THIRD_PARTY_ENRICHMENT_PROVIDER"] = "openai"
|
||||
env["AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MODEL_NAME"] = "gpt-4o"
|
||||
env["AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MAX_TOKENS"] = "16384"
|
||||
env["COMMON_THIRD_PARTY_ENRICHMENT_INTERVAL"] = "25"
|
||||
env["COMMON_THIRD_PARTY_ENRICHMENT_MAX_CONCURRENCY"] = "2"
|
||||
env["COMMON_THIRD_PARTY_ENRICHMENT_STALE_AFTER"] = "1200"
|
||||
env["COMMON_THIRD_PARTY_ENRICHMENT_AGENT_TIMEOUT"] = "120"
|
||||
env["COMMON_THIRD_PARTY_ENRICHMENT_AGENT_MAX_TURNS"] = "8"
|
||||
env["COMMON_THIRD_PARTY_ENRICHMENT_CONFIDENCE_THRESHOLD"] = "0.85"
|
||||
env["COMMON_THIRD_PARTY_ENRICHMENT_MAX_ATTEMPTS"] = "5"
|
||||
env["THIRD_PARTY_VETTING_INTERVAL"] = "15"
|
||||
env["THIRD_PARTY_VETTING_STALE_AFTER"] = "1800"
|
||||
env["THIRD_PARTY_VETTING_MAX_CONCURRENCY"] = "2"
|
||||
@@ -490,6 +508,17 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
|
||||
assert.Equal(t, 900, cfg.Probod.CommonPatternEnrichmentWorker.StaleAfter)
|
||||
assert.Equal(t, 50, cfg.Probod.CommonPatternEnrichmentWorker.AgentTimeout)
|
||||
assert.Equal(t, 5, cfg.Probod.CommonPatternEnrichmentWorker.AgentMaxTurns)
|
||||
assert.Equal(t, "openai", cfg.Probod.Agents.CommonThirdPartyEnrichment.Provider)
|
||||
assert.Equal(t, "gpt-4o", cfg.Probod.Agents.CommonThirdPartyEnrichment.ModelName)
|
||||
require.NotNil(t, cfg.Probod.Agents.CommonThirdPartyEnrichment.MaxTokens)
|
||||
assert.Equal(t, 16384, *cfg.Probod.Agents.CommonThirdPartyEnrichment.MaxTokens)
|
||||
assert.Equal(t, 25, cfg.Probod.CommonThirdPartyEnrichmentWorker.Interval)
|
||||
assert.Equal(t, 2, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxConcurrency)
|
||||
assert.Equal(t, 1200, cfg.Probod.CommonThirdPartyEnrichmentWorker.StaleAfter)
|
||||
assert.Equal(t, 120, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentTimeout)
|
||||
assert.Equal(t, 8, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentMaxTurns)
|
||||
assert.Equal(t, 0.85, cfg.Probod.CommonThirdPartyEnrichmentWorker.ConfidenceThreshold)
|
||||
assert.Equal(t, 5, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxAttempts)
|
||||
assert.Equal(t, 15, cfg.Probod.ThirdPartyVetting.Interval)
|
||||
assert.Equal(t, 1800, cfg.Probod.ThirdPartyVetting.StaleAfter)
|
||||
assert.Equal(t, 2, cfg.Probod.ThirdPartyVetting.MaxConcurrency)
|
||||
|
||||
@@ -16,6 +16,7 @@ package coredata
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
@@ -49,6 +50,9 @@ type (
|
||||
SecurityPageURL *string `db:"security_page_url"`
|
||||
TrustPageURL *string `db:"trust_page_url"`
|
||||
LogoFileID *gid.GID `db:"logo_file_id"`
|
||||
EnrichmentRequestedAt *time.Time `db:"enrichment_requested_at"`
|
||||
Enrichment json.RawMessage `db:"enrichment"`
|
||||
EnrichmentAttempts int `db:"enrichment_attempts"`
|
||||
CreatedAt time.Time `db:"created_at"`
|
||||
UpdatedAt time.Time `db:"updated_at"`
|
||||
}
|
||||
@@ -124,6 +128,9 @@ SELECT
|
||||
security_page_url,
|
||||
trust_page_url,
|
||||
logo_file_id,
|
||||
enrichment_requested_at,
|
||||
enrichment,
|
||||
enrichment_attempts,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM
|
||||
@@ -181,6 +188,9 @@ SELECT
|
||||
security_page_url,
|
||||
trust_page_url,
|
||||
logo_file_id,
|
||||
enrichment_requested_at,
|
||||
enrichment,
|
||||
enrichment_attempts,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM
|
||||
@@ -238,6 +248,9 @@ SELECT
|
||||
security_page_url,
|
||||
trust_page_url,
|
||||
logo_file_id,
|
||||
enrichment_requested_at,
|
||||
enrichment,
|
||||
enrichment_attempts,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM
|
||||
@@ -294,6 +307,9 @@ INSERT INTO common_third_parties (
|
||||
security_page_url,
|
||||
trust_page_url,
|
||||
logo_file_id,
|
||||
enrichment_requested_at,
|
||||
enrichment,
|
||||
enrichment_attempts,
|
||||
created_at,
|
||||
updated_at
|
||||
) VALUES (
|
||||
@@ -316,6 +332,9 @@ INSERT INTO common_third_parties (
|
||||
@security_page_url,
|
||||
@trust_page_url,
|
||||
@logo_file_id,
|
||||
@enrichment_requested_at,
|
||||
@enrichment,
|
||||
@enrichment_attempts,
|
||||
@created_at,
|
||||
@updated_at
|
||||
)
|
||||
@@ -341,6 +360,9 @@ INSERT INTO common_third_parties (
|
||||
"security_page_url": t.SecurityPageURL,
|
||||
"trust_page_url": t.TrustPageURL,
|
||||
"logo_file_id": t.LogoFileID,
|
||||
"enrichment_requested_at": t.EnrichmentRequestedAt,
|
||||
"enrichment": t.Enrichment,
|
||||
"enrichment_attempts": t.EnrichmentAttempts,
|
||||
"created_at": t.CreatedAt,
|
||||
"updated_at": t.UpdatedAt,
|
||||
}
|
||||
@@ -381,6 +403,9 @@ INSERT INTO common_third_parties (
|
||||
security_page_url,
|
||||
trust_page_url,
|
||||
logo_file_id,
|
||||
enrichment_requested_at,
|
||||
enrichment,
|
||||
enrichment_attempts,
|
||||
created_at,
|
||||
updated_at
|
||||
) VALUES (
|
||||
@@ -403,6 +428,9 @@ INSERT INTO common_third_parties (
|
||||
@security_page_url,
|
||||
@trust_page_url,
|
||||
@logo_file_id,
|
||||
@enrichment_requested_at,
|
||||
@enrichment,
|
||||
@enrichment_attempts,
|
||||
@created_at,
|
||||
@updated_at
|
||||
)
|
||||
@@ -445,6 +473,9 @@ RETURNING
|
||||
security_page_url,
|
||||
trust_page_url,
|
||||
logo_file_id,
|
||||
enrichment_requested_at,
|
||||
enrichment,
|
||||
enrichment_attempts,
|
||||
created_at,
|
||||
updated_at
|
||||
`
|
||||
@@ -471,6 +502,9 @@ RETURNING
|
||||
"security_page_url": t.SecurityPageURL,
|
||||
"trust_page_url": t.TrustPageURL,
|
||||
"logo_file_id": t.LogoFileID,
|
||||
"enrichment_requested_at": t.EnrichmentRequestedAt,
|
||||
"enrichment": t.Enrichment,
|
||||
"enrichment_attempts": t.EnrichmentAttempts,
|
||||
"created_at": t.CreatedAt,
|
||||
"updated_at": t.UpdatedAt,
|
||||
}
|
||||
@@ -534,6 +568,9 @@ SELECT
|
||||
security_page_url,
|
||||
trust_page_url,
|
||||
logo_file_id,
|
||||
enrichment_requested_at,
|
||||
enrichment,
|
||||
enrichment_attempts,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM
|
||||
@@ -585,6 +622,9 @@ SELECT
|
||||
security_page_url,
|
||||
trust_page_url,
|
||||
logo_file_id,
|
||||
enrichment_requested_at,
|
||||
enrichment,
|
||||
enrichment_attempts,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM
|
||||
@@ -690,6 +730,9 @@ SELECT
|
||||
security_page_url,
|
||||
trust_page_url,
|
||||
logo_file_id,
|
||||
enrichment_requested_at,
|
||||
enrichment,
|
||||
enrichment_attempts,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM
|
||||
@@ -750,3 +793,213 @@ WHERE
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// LoadNextForEnrichmentForUpdateSkipLocked claims the oldest row queued
|
||||
// for enrichment. The global catalog is not tenant-scoped, so the claim
|
||||
// is intentionally cross-tenant: the enrichment worker is a system
|
||||
// worker that drains the queue regardless of tenant.
|
||||
func (t *CommonThirdParty) LoadNextForEnrichmentForUpdateSkipLocked(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
) error {
|
||||
q := `
|
||||
SELECT
|
||||
id,
|
||||
name,
|
||||
slug,
|
||||
category,
|
||||
headquarter_address,
|
||||
legal_name,
|
||||
website_url,
|
||||
privacy_policy_url,
|
||||
service_level_agreement_url,
|
||||
service_software_agreement_url,
|
||||
data_processing_agreement_url,
|
||||
business_associate_agreement_url,
|
||||
subprocessors_list_url,
|
||||
certifications,
|
||||
status_page_url,
|
||||
terms_of_service_url,
|
||||
security_page_url,
|
||||
trust_page_url,
|
||||
logo_file_id,
|
||||
enrichment_requested_at,
|
||||
enrichment,
|
||||
enrichment_attempts,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM
|
||||
common_third_parties
|
||||
WHERE
|
||||
enrichment_requested_at IS NOT NULL
|
||||
ORDER BY
|
||||
enrichment_requested_at ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1;
|
||||
`
|
||||
|
||||
rows, err := tx.Query(ctx, q)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot query common third party for enrichment: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
row, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[CommonThirdParty])
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrResourceNotFound
|
||||
}
|
||||
|
||||
return fmt.Errorf("cannot collect common third party for enrichment: %w", err)
|
||||
}
|
||||
|
||||
*t = row
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ClearEnrichmentRequestedAt removes the row from the enrichment queue
|
||||
// and bumps the attempt counter. It bumps updated_at so the
|
||||
// stale-recovery clock starts at claim time, keeping
|
||||
// ResetStaleCommonThirdPartyEnrichments from re-arming a row that is
|
||||
// still being processed. The attempt counter is incremented up front so
|
||||
// a crash between claim and persist still counts against the retry
|
||||
// budget.
|
||||
func (t *CommonThirdParty) ClearEnrichmentRequestedAt(
|
||||
ctx context.Context,
|
||||
tx pg.Tx,
|
||||
) error {
|
||||
q := `
|
||||
UPDATE common_third_parties
|
||||
SET
|
||||
enrichment_requested_at = NULL,
|
||||
enrichment_attempts = enrichment_attempts + 1,
|
||||
updated_at = NOW()
|
||||
WHERE id = @id
|
||||
`
|
||||
|
||||
args := pgx.StrictNamedArgs{"id": t.ID}
|
||||
|
||||
_, err := tx.Exec(ctx, q, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot clear enrichment requested at: %w", err)
|
||||
}
|
||||
|
||||
t.EnrichmentRequestedAt = nil
|
||||
t.EnrichmentAttempts++
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateEnrichment persists the enrichment result: the resolved metadata
|
||||
// fields plus the per-field enrichment provenance JSON. It is a targeted
|
||||
// partial update that never touches id, slug, category, name, logo, the
|
||||
// queue column, or the attempt counter (logo is owned by
|
||||
// UpdateLogoFileID; the queue column and counter are managed by
|
||||
// ClearEnrichmentRequestedAt). The caller decides which scalar fields to
|
||||
// write versus leave untouched, then passes the merged receiver here.
|
||||
func (t CommonThirdParty) UpdateEnrichment(
|
||||
ctx context.Context,
|
||||
conn pg.Tx,
|
||||
) error {
|
||||
q := `
|
||||
UPDATE common_third_parties
|
||||
SET
|
||||
headquarter_address = @headquarter_address,
|
||||
legal_name = @legal_name,
|
||||
website_url = @website_url,
|
||||
privacy_policy_url = @privacy_policy_url,
|
||||
service_level_agreement_url = @service_level_agreement_url,
|
||||
service_software_agreement_url = @service_software_agreement_url,
|
||||
data_processing_agreement_url = @data_processing_agreement_url,
|
||||
business_associate_agreement_url = @business_associate_agreement_url,
|
||||
subprocessors_list_url = @subprocessors_list_url,
|
||||
certifications = @certifications,
|
||||
status_page_url = @status_page_url,
|
||||
terms_of_service_url = @terms_of_service_url,
|
||||
security_page_url = @security_page_url,
|
||||
trust_page_url = @trust_page_url,
|
||||
enrichment = @enrichment,
|
||||
updated_at = @updated_at
|
||||
WHERE
|
||||
id = @id
|
||||
`
|
||||
|
||||
args := pgx.StrictNamedArgs{
|
||||
"id": t.ID,
|
||||
"headquarter_address": t.HeadquarterAddress,
|
||||
"legal_name": t.LegalName,
|
||||
"website_url": t.WebsiteURL,
|
||||
"privacy_policy_url": t.PrivacyPolicyURL,
|
||||
"service_level_agreement_url": t.ServiceLevelAgreementURL,
|
||||
"service_software_agreement_url": t.ServiceSoftwareAgreementURL,
|
||||
"data_processing_agreement_url": t.DataProcessingAgreementURL,
|
||||
"business_associate_agreement_url": t.BusinessAssociateAgreementURL,
|
||||
"subprocessors_list_url": t.SubprocessorsListURL,
|
||||
"certifications": t.Certifications,
|
||||
"status_page_url": t.StatusPageURL,
|
||||
"terms_of_service_url": t.TermsOfServiceURL,
|
||||
"security_page_url": t.SecurityPageURL,
|
||||
"trust_page_url": t.TrustPageURL,
|
||||
"enrichment": t.Enrichment,
|
||||
"updated_at": t.UpdatedAt,
|
||||
}
|
||||
|
||||
result, err := conn.Exec(ctx, q, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot update common third party enrichment: %w", err)
|
||||
}
|
||||
|
||||
if result.RowsAffected() == 0 {
|
||||
return ErrResourceNotFound
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResetStaleCommonThirdPartyEnrichments re-arms enrichment_requested_at
|
||||
// on rows whose enrichment was claimed but never completed and have been
|
||||
// idle longer than staleAfter, so a crashed or timed-out run is retried.
|
||||
//
|
||||
// A claimed row has enrichment_attempts > 0 (Claim increments it) and a
|
||||
// completed row has a non-null enrichment payload (Process always writes
|
||||
// it, even on a no-result run), so the sweep targets rows that were
|
||||
// claimed but carry no enrichment yet. Curated rows that were never
|
||||
// enqueued keep enrichment_attempts = 0 and are left untouched. The
|
||||
// max-attempts ceiling stops permanently failing rows from looping
|
||||
// forever.
|
||||
//
|
||||
// Like the claim query, this sweep is intentionally cross-tenant: the
|
||||
// enrichment worker is a system worker that drains the queue regardless
|
||||
// of tenant.
|
||||
func ResetStaleCommonThirdPartyEnrichments(
|
||||
ctx context.Context,
|
||||
conn pg.Querier,
|
||||
staleAfter time.Duration,
|
||||
maxAttempts int,
|
||||
) error {
|
||||
q := `
|
||||
UPDATE common_third_parties
|
||||
SET
|
||||
enrichment_requested_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE
|
||||
enrichment_requested_at IS NULL
|
||||
AND enrichment IS NULL
|
||||
AND enrichment_attempts > 0
|
||||
AND enrichment_attempts < @max_attempts
|
||||
AND updated_at < @stale_before
|
||||
`
|
||||
|
||||
args := pgx.StrictNamedArgs{
|
||||
"max_attempts": maxAttempts,
|
||||
"stale_before": time.Now().Add(-staleAfter),
|
||||
}
|
||||
|
||||
_, err := conn.Exec(ctx, q, args)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reset stale common third party enrichments: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
28
pkg/coredata/migrations/20260609T120000Z.sql
Normal file
28
pkg/coredata/migrations/20260609T120000Z.sql
Normal file
@@ -0,0 +1,28 @@
|
||||
-- Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
--
|
||||
-- Permission to use, copy, modify, and/or distribute this software for any
|
||||
-- purpose with or without fee is hereby granted, provided that the above
|
||||
-- copyright notice and this permission notice appear in all copies.
|
||||
--
|
||||
-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
-- PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
ALTER TABLE
|
||||
common_third_parties
|
||||
ADD
|
||||
COLUMN enrichment_requested_at TIMESTAMP WITH TIME ZONE,
|
||||
ADD
|
||||
COLUMN enrichment JSONB,
|
||||
ADD
|
||||
COLUMN enrichment_attempts INTEGER NOT NULL DEFAULT 0;
|
||||
|
||||
-- Partial index backs the enrichment worker's claim query, which polls
|
||||
-- only rows currently queued for enrichment.
|
||||
CREATE INDEX common_third_parties_enrichment_requested_at_idx
|
||||
ON common_third_parties (enrichment_requested_at)
|
||||
WHERE enrichment_requested_at IS NOT NULL;
|
||||
@@ -43,8 +43,9 @@ type (
|
||||
ThirdPartyVettingWorkerConfig = probodconfig.ThirdPartyVettingWorkerConfig
|
||||
AgentsConfig = probodconfig.AgentsConfig
|
||||
|
||||
TrackerMappingWorkerConfig = probodconfig.TrackerMappingWorkerConfig
|
||||
CommonPatternEnrichmentWorkerConfig = probodconfig.CommonPatternEnrichmentWorkerConfig
|
||||
TrackerMappingWorkerConfig = probodconfig.TrackerMappingWorkerConfig
|
||||
CommonPatternEnrichmentWorkerConfig = probodconfig.CommonPatternEnrichmentWorkerConfig
|
||||
CommonThirdPartyEnrichmentWorkerConfig = probodconfig.CommonThirdPartyEnrichmentWorkerConfig
|
||||
|
||||
MailerConfig = probodconfig.MailerConfig
|
||||
SMTPConfig = probodconfig.SMTPConfig
|
||||
|
||||
73
pkg/probod/common_third_party_enrichment.go
Normal file
73
pkg/probod/common_third_party_enrichment.go
Normal file
@@ -0,0 +1,73 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package probod
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.gearno.de/kit/log"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"go.probo.inc/probo/pkg/filemanager"
|
||||
"go.probo.inc/probo/pkg/thirdparty"
|
||||
)
|
||||
|
||||
// buildCommonThirdPartyEnrichmentConfig wires the common-third-party
|
||||
// enrichment worker config: the LLM client for its two agents plus the
|
||||
// worker tuning, browser endpoint, and logo-storage dependencies. It is
|
||||
// opt-in: a deployment that does not set
|
||||
// `llm.common-third-party-enrichment.provider` gets a zero config (nil
|
||||
// LLM client), so the worker runs as a no-op and the caller skips
|
||||
// registration.
|
||||
func (impl *Implm) buildCommonThirdPartyEnrichmentConfig(
|
||||
l *log.Logger,
|
||||
tp trace.TracerProvider,
|
||||
r prometheus.Registerer,
|
||||
fileManager *filemanager.Service,
|
||||
) (thirdparty.EnrichmentConfig, error) {
|
||||
if impl.cfg.Agents.CommonThirdPartyEnrichment.Provider == "" {
|
||||
return thirdparty.EnrichmentConfig{}, nil
|
||||
}
|
||||
|
||||
agentCfg, llmClient, err := impl.resolveAgentClient(
|
||||
"common-third-party-enrichment",
|
||||
impl.cfg.Agents.CommonThirdPartyEnrichment,
|
||||
l,
|
||||
tp,
|
||||
r,
|
||||
)
|
||||
if err != nil {
|
||||
return thirdparty.EnrichmentConfig{}, fmt.Errorf("cannot resolve common third party enrichment agent client: %w", err)
|
||||
}
|
||||
|
||||
workerCfg := impl.cfg.CommonThirdPartyEnrichmentWorker
|
||||
|
||||
return thirdparty.EnrichmentConfig{
|
||||
LLMClient: llmClient,
|
||||
Model: agentCfg.ModelName,
|
||||
MaxTokens: agentCfg.MaxTokens,
|
||||
Temperature: agentCfg.Temperature,
|
||||
FirecrawlAPIKey: impl.cfg.Agents.Tools.FirecrawlAPIKey,
|
||||
ChromeAddr: impl.cfg.ChromeDPAddr,
|
||||
AgentTimeout: time.Duration(workerCfg.AgentTimeout) * time.Second,
|
||||
MaxTurns: workerCfg.AgentMaxTurns,
|
||||
ConfidenceThreshold: workerCfg.ConfidenceThreshold,
|
||||
StaleAfter: time.Duration(workerCfg.StaleAfter) * time.Second,
|
||||
MaxAttempts: workerCfg.MaxAttempts,
|
||||
FileManager: fileManager,
|
||||
Bucket: impl.cfg.AWS.Bucket,
|
||||
}, nil
|
||||
}
|
||||
@@ -181,6 +181,15 @@ func New() *Implm {
|
||||
StaleAfter: 1500,
|
||||
MaxConcurrency: 1,
|
||||
},
|
||||
CommonThirdPartyEnrichmentWorker: CommonThirdPartyEnrichmentWorkerConfig{
|
||||
Interval: 10,
|
||||
MaxConcurrency: 1,
|
||||
StaleAfter: 900,
|
||||
AgentTimeout: 90,
|
||||
AgentMaxTurns: 12,
|
||||
ConfidenceThreshold: 0.7,
|
||||
MaxAttempts: 3,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -328,6 +337,11 @@ func (impl *Implm) Run(
|
||||
|
||||
fileManagerService := filemanager.NewService(pgClient, baseURL, s3Client)
|
||||
|
||||
commonThirdPartyEnrichmentCfg, err := impl.buildCommonThirdPartyEnrichmentConfig(l, tp, r, fileManagerService)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var (
|
||||
samlCert *x509.Certificate
|
||||
samlKey *rsa.PrivateKey
|
||||
@@ -819,6 +833,34 @@ func (impl *Implm) Run(
|
||||
)
|
||||
}
|
||||
|
||||
// The common-third-party enrichment worker fills catalog metadata
|
||||
// (URLs, address, certifications, logo) via two agents plus a
|
||||
// deterministic logo step. It needs an LLM client, so it is only
|
||||
// started when its agent config is present.
|
||||
stopCommonThirdPartyEnrichmentWorker := func() {}
|
||||
|
||||
if commonThirdPartyEnrichmentCfg.LLMClient != nil {
|
||||
commonThirdPartyEnrichmentWorker := thirdparty.NewCommonThirdPartyEnrichmentWorker(
|
||||
pgClient,
|
||||
l.Named("common-third-party-enrichment-worker"),
|
||||
commonThirdPartyEnrichmentCfg,
|
||||
worker.WithInterval(time.Duration(impl.cfg.CommonThirdPartyEnrichmentWorker.Interval)*time.Second),
|
||||
worker.WithMaxConcurrency(impl.cfg.CommonThirdPartyEnrichmentWorker.MaxConcurrency),
|
||||
)
|
||||
|
||||
var commonThirdPartyEnrichmentWorkerCtx context.Context
|
||||
|
||||
commonThirdPartyEnrichmentWorkerCtx, stopCommonThirdPartyEnrichmentWorker = context.WithCancel(context.Background())
|
||||
|
||||
wg.Go(
|
||||
func() {
|
||||
if err := commonThirdPartyEnrichmentWorker.Run(commonThirdPartyEnrichmentWorkerCtx); err != nil {
|
||||
cancel(fmt.Errorf("common third party enrichment worker crashed: %w", err))
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
mailingListWorker := mailman.NewMailingListWorker(mailmanService, pgClient, l.Named("mailing-list-worker"))
|
||||
mailingListWorkerCtx, stopMailingListWorker := context.WithCancel(context.Background())
|
||||
|
||||
@@ -910,6 +952,7 @@ func (impl *Implm) Run(
|
||||
stopTrackerPolicyWorker()
|
||||
stopTrackerMappingWorker()
|
||||
stopCommonPatternEnrichmentWorker()
|
||||
stopCommonThirdPartyEnrichmentWorker()
|
||||
stopMailingListWorker()
|
||||
stopVettingWorker()
|
||||
stopEvidenceDescriptionWorker()
|
||||
|
||||
@@ -62,8 +62,9 @@ type (
|
||||
EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"`
|
||||
ThirdPartyVetting ThirdPartyVettingWorkerConfig `json:"third-party-vetting-worker"`
|
||||
|
||||
TrackerMappingWorker TrackerMappingWorkerConfig `json:"tracker-mapping-worker"`
|
||||
CommonPatternEnrichmentWorker CommonPatternEnrichmentWorkerConfig `json:"common-pattern-enrichment-worker"`
|
||||
TrackerMappingWorker TrackerMappingWorkerConfig `json:"tracker-mapping-worker"`
|
||||
CommonPatternEnrichmentWorker CommonPatternEnrichmentWorkerConfig `json:"common-pattern-enrichment-worker"`
|
||||
CommonThirdPartyEnrichmentWorker CommonThirdPartyEnrichmentWorkerConfig `json:"common-third-party-enrichment-worker"`
|
||||
|
||||
ChromeDPAddr string `json:"chrome-dp-addr"`
|
||||
CustomDomains CustomDomainsConfig `json:"custom-domains"`
|
||||
|
||||
@@ -76,6 +76,22 @@ type (
|
||||
AgentMaxTurns int `json:"agent-max-turns"`
|
||||
}
|
||||
|
||||
// CommonThirdPartyEnrichmentWorkerConfig holds worker-side tuning for
|
||||
// the common-third-party enrichment background worker. LLM parameters
|
||||
// for its agents live under AgentsConfig.CommonThirdPartyEnrichment.
|
||||
// ConfidenceThreshold is the floor a resolved value must clear before
|
||||
// it is written to its column; MaxAttempts caps stale-recovery
|
||||
// retries.
|
||||
CommonThirdPartyEnrichmentWorkerConfig struct {
|
||||
Interval int `json:"interval"` // seconds between polls
|
||||
MaxConcurrency int `json:"max-concurrency"`
|
||||
StaleAfter int `json:"stale-after"` // seconds before a claim is recycled
|
||||
AgentTimeout int `json:"agent-timeout"` // seconds, single agent run
|
||||
AgentMaxTurns int `json:"agent-max-turns"`
|
||||
ConfidenceThreshold float64 `json:"confidence-threshold"`
|
||||
MaxAttempts int `json:"max-attempts"`
|
||||
}
|
||||
|
||||
// AgentToolsConfig holds API keys and settings for external tools
|
||||
// that agents can use (web search, scraping, etc.).
|
||||
AgentToolsConfig struct {
|
||||
@@ -86,15 +102,16 @@ type (
|
||||
// settings. Default is used as a fallback when an agent-specific field
|
||||
// is zero-valued.
|
||||
AgentsConfig struct {
|
||||
Providers map[string]LLMProviderConfig `json:"providers"`
|
||||
Default LLMAgentConfig `json:"defaults"`
|
||||
Probo LLMAgentConfig `json:"probo"`
|
||||
EvidenceDescriber LLMAgentConfig `json:"evidence-describer"`
|
||||
ThirdPartyVetter LLMAgentConfig `json:"third-party-vetter"`
|
||||
ThirdPartyDisambiguation LLMAgentConfig `json:"third-party-disambiguation"`
|
||||
TrackerMapping LLMAgentConfig `json:"tracker-mapping"`
|
||||
TrackerEnrichment LLMAgentConfig `json:"tracker-enrichment"`
|
||||
Tools AgentToolsConfig `json:"tools"`
|
||||
Providers map[string]LLMProviderConfig `json:"providers"`
|
||||
Default LLMAgentConfig `json:"defaults"`
|
||||
Probo LLMAgentConfig `json:"probo"`
|
||||
EvidenceDescriber LLMAgentConfig `json:"evidence-describer"`
|
||||
ThirdPartyVetter LLMAgentConfig `json:"third-party-vetter"`
|
||||
ThirdPartyDisambiguation LLMAgentConfig `json:"third-party-disambiguation"`
|
||||
TrackerMapping LLMAgentConfig `json:"tracker-mapping"`
|
||||
TrackerEnrichment LLMAgentConfig `json:"tracker-enrichment"`
|
||||
CommonThirdPartyEnrichment LLMAgentConfig `json:"common-third-party-enrichment"`
|
||||
Tools AgentToolsConfig `json:"tools"`
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
94
pkg/thirdparty/common_third_party_company_profile_agent.go
vendored
Normal file
94
pkg/thirdparty/common_third_party_company_profile_agent.go
vendored
Normal file
@@ -0,0 +1,94 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package thirdparty
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.gearno.de/kit/log"
|
||||
"go.probo.inc/probo/pkg/agent"
|
||||
"go.probo.inc/probo/pkg/agent/tools/search"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
)
|
||||
|
||||
//go:embed prompts/common_third_party_company_profile.txt.tmpl
|
||||
var companyProfilePrompt string
|
||||
|
||||
// CompanyProfileResult is the structured output of the company-profile
|
||||
// agent (Agent A): the vendor's identity facts. website_url is the key
|
||||
// signal the compliance-docs agent and the logo step depend on, so it is
|
||||
// resolved here first.
|
||||
type CompanyProfileResult struct {
|
||||
LegalName EnrichedField `json:"legal_name" jsonschema:"The vendor's full legal company name including the entity suffix (e.g. 'Acme Technologies, Inc.')."`
|
||||
HeadquarterAddress EnrichedField `json:"headquarter_address" jsonschema:"The vendor's headquarters postal address (street, city, region, country)."`
|
||||
WebsiteURL EnrichedField `json:"website_url" jsonschema:"The vendor's canonical primary marketing website URL (https scheme, no tracking query parameters, no trailing path)."`
|
||||
}
|
||||
|
||||
func buildCompanyProfileAgent(
|
||||
cfg EnrichmentConfig,
|
||||
logger *log.Logger,
|
||||
) *agent.Agent {
|
||||
var tools []agent.Tool
|
||||
|
||||
if cfg.FirecrawlAPIKey != "" {
|
||||
tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlAPIKey))
|
||||
}
|
||||
|
||||
outputType, err := agent.NewOutputType[CompanyProfileResult]("common_third_party_company_profile")
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("thirdparty: cannot build company profile output type: %s", err))
|
||||
}
|
||||
|
||||
opts := []agent.Option{
|
||||
agent.WithInstructions(companyProfilePrompt),
|
||||
agent.WithModel(cfg.Model),
|
||||
agent.WithOutputType(outputType),
|
||||
agent.WithMaxTurns(resolveEnrichmentMaxTurns(cfg.MaxTurns)),
|
||||
agent.WithMaxTokens(resolveEnrichmentMaxTokens(cfg.MaxTokens)),
|
||||
agent.WithLogger(logger),
|
||||
}
|
||||
|
||||
if len(tools) > 0 {
|
||||
opts = append(opts, agent.WithTools(tools...))
|
||||
}
|
||||
|
||||
if cfg.Temperature != nil {
|
||||
opts = append(opts, agent.WithTemperature(*cfg.Temperature))
|
||||
}
|
||||
|
||||
return agent.New("common-third-party-company-profile", cfg.LLMClient, opts...)
|
||||
}
|
||||
|
||||
// buildCompanyProfilePrompt renders the per-row input for Agent A. Any
|
||||
// values already on the row are passed as hints so the agent confirms or
|
||||
// corrects them rather than starting cold.
|
||||
func buildCompanyProfilePrompt(party coredata.CommonThirdParty) string {
|
||||
var b strings.Builder
|
||||
|
||||
fmt.Fprintf(&b, "Research this company and return its profile.\n\n")
|
||||
fmt.Fprintf(&b, "<name> %s </name>\n", party.Name)
|
||||
|
||||
if party.WebsiteURL != nil && strings.TrimSpace(*party.WebsiteURL) != "" {
|
||||
fmt.Fprintf(&b, "<known_website> %s </known_website>\n", *party.WebsiteURL)
|
||||
}
|
||||
|
||||
if party.LegalName != nil && strings.TrimSpace(*party.LegalName) != "" {
|
||||
fmt.Fprintf(&b, "<known_legal_name> %s </known_legal_name>\n", *party.LegalName)
|
||||
}
|
||||
|
||||
return b.String()
|
||||
}
|
||||
105
pkg/thirdparty/common_third_party_compliance_docs_agent.go
vendored
Normal file
105
pkg/thirdparty/common_third_party_compliance_docs_agent.go
vendored
Normal file
@@ -0,0 +1,105 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package thirdparty
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.gearno.de/kit/log"
|
||||
"go.probo.inc/probo/pkg/agent"
|
||||
"go.probo.inc/probo/pkg/agent/tools/search"
|
||||
)
|
||||
|
||||
//go:embed prompts/common_third_party_compliance_docs.txt.tmpl
|
||||
var complianceDocsPrompt string
|
||||
|
||||
// ComplianceDocsResult is the structured output of the compliance-docs
|
||||
// agent (Agent B): the legal-document URLs, trust/security/status pages,
|
||||
// and certifications. These all live in the same source ecosystem (the
|
||||
// vendor footer and trust portal), so one agent resolves them together.
|
||||
type ComplianceDocsResult struct {
|
||||
PrivacyPolicyURL EnrichedField `json:"privacy_policy_url" jsonschema:"URL of the vendor's privacy policy."`
|
||||
TermsOfServiceURL EnrichedField `json:"terms_of_service_url" jsonschema:"URL of the vendor's terms of service / terms of use."`
|
||||
ServiceLevelAgreementURL EnrichedField `json:"service_level_agreement_url" jsonschema:"URL of the vendor's public service level agreement (SLA). Often gated behind sales; return empty when not public."`
|
||||
ServiceSoftwareAgreementURL EnrichedField `json:"service_software_agreement_url" jsonschema:"URL of the vendor's master software/subscription agreement (MSA). Often gated or identical to the terms of service; return empty when not public."`
|
||||
DataProcessingAgreementURL EnrichedField `json:"data_processing_agreement_url" jsonschema:"URL of the vendor's data processing agreement (DPA). Often a PDF; return empty when only available on request."`
|
||||
BusinessAssociateAgreementURL EnrichedField `json:"business_associate_agreement_url" jsonschema:"URL of the vendor's HIPAA business associate agreement (BAA). Almost always gated behind sales; return empty when not public."`
|
||||
SubprocessorsListURL EnrichedField `json:"subprocessors_list_url" jsonschema:"URL of the vendor's sub-processors list page."`
|
||||
StatusPageURL EnrichedField `json:"status_page_url" jsonschema:"URL of the vendor's uptime/status page (e.g. status.vendor.com)."`
|
||||
SecurityPageURL EnrichedField `json:"security_page_url" jsonschema:"URL of the vendor's security page or security overview."`
|
||||
TrustPageURL EnrichedField `json:"trust_page_url" jsonschema:"URL of the vendor's trust center / trust portal (e.g. Vanta, SafeBase, Drata hosted)."`
|
||||
Certifications CertificationsField `json:"certifications" jsonschema:"Certifications and compliance frameworks the vendor publicly claims."`
|
||||
}
|
||||
|
||||
// buildComplianceDocsAgent builds Agent B. extraTools carries the browser
|
||||
// read-only toolset when a headless Chrome endpoint is configured; it is
|
||||
// empty otherwise, in which case the agent relies on web_search alone.
|
||||
func buildComplianceDocsAgent(
|
||||
cfg EnrichmentConfig,
|
||||
logger *log.Logger,
|
||||
extraTools []agent.Tool,
|
||||
) *agent.Agent {
|
||||
tools := append([]agent.Tool{}, extraTools...)
|
||||
|
||||
if cfg.FirecrawlAPIKey != "" {
|
||||
tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlAPIKey))
|
||||
}
|
||||
|
||||
outputType, err := agent.NewOutputType[ComplianceDocsResult]("common_third_party_compliance_docs")
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("thirdparty: cannot build compliance docs output type: %s", err))
|
||||
}
|
||||
|
||||
opts := []agent.Option{
|
||||
agent.WithInstructions(complianceDocsPrompt),
|
||||
agent.WithModel(cfg.Model),
|
||||
agent.WithOutputType(outputType),
|
||||
agent.WithMaxTurns(resolveEnrichmentMaxTurns(cfg.MaxTurns)),
|
||||
agent.WithMaxTokens(resolveEnrichmentMaxTokens(cfg.MaxTokens)),
|
||||
agent.WithLogger(logger),
|
||||
}
|
||||
|
||||
if len(tools) > 0 {
|
||||
opts = append(opts, agent.WithTools(tools...))
|
||||
}
|
||||
|
||||
if cfg.Temperature != nil {
|
||||
opts = append(opts, agent.WithTemperature(*cfg.Temperature))
|
||||
}
|
||||
|
||||
return agent.New("common-third-party-compliance-docs", cfg.LLMClient, opts...)
|
||||
}
|
||||
|
||||
// buildComplianceDocsPrompt renders the per-row input for Agent B,
|
||||
// seeding it with the vendor name and the website/legal name resolved by
|
||||
// Agent A so it can scope its search to the vendor's own domain.
|
||||
func buildComplianceDocsPrompt(name, websiteURL, legalName string) string {
|
||||
var b strings.Builder
|
||||
|
||||
fmt.Fprintf(&b, "Find the compliance documents and trust pages for this vendor.\n\n")
|
||||
fmt.Fprintf(&b, "<name> %s </name>\n", name)
|
||||
|
||||
if w := strings.TrimSpace(websiteURL); w != "" {
|
||||
fmt.Fprintf(&b, "<website> %s </website>\n", w)
|
||||
}
|
||||
|
||||
if l := strings.TrimSpace(legalName); l != "" {
|
||||
fmt.Fprintf(&b, "<legal_name> %s </legal_name>\n", l)
|
||||
}
|
||||
|
||||
return b.String()
|
||||
}
|
||||
345
pkg/thirdparty/common_third_party_enrichment.go
vendored
Normal file
345
pkg/thirdparty/common_third_party_enrichment.go
vendored
Normal file
@@ -0,0 +1,345 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package thirdparty
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
)
|
||||
|
||||
// Provenance sources and per-field statuses recorded in the enrichment
|
||||
// JSON. The "source" distinguishes values written by this enricher from
|
||||
// values owned externally (curated seed data or a human edit), which the
|
||||
// enricher must never overwrite.
|
||||
const (
|
||||
enrichmentSourceEnrichment = "enrichment"
|
||||
enrichmentSourceExternal = "external"
|
||||
|
||||
enrichmentFieldStatusFound = "found"
|
||||
enrichmentFieldStatusNotFound = "not_found"
|
||||
enrichmentFieldStatusLowConfidence = "low_confidence"
|
||||
enrichmentFieldStatusExternal = "exists_external"
|
||||
|
||||
// Run-level status recorded at the top of the enrichment payload.
|
||||
enrichmentStatusDone = "done"
|
||||
enrichmentStatusPartial = "partial"
|
||||
enrichmentStatusFailed = "failed"
|
||||
)
|
||||
|
||||
type (
|
||||
// EnrichedField is the per-field unit the enrichment agents return:
|
||||
// the resolved value plus a self-assessed confidence and the source
|
||||
// URL where it was verified. The worker applies a confidence
|
||||
// threshold before writing the value to its column.
|
||||
EnrichedField struct {
|
||||
Value string `json:"value" jsonschema:"The resolved value, or an empty string when not confidently found. Never guess."`
|
||||
Confidence float64 `json:"confidence" jsonschema:"Confidence from 0.0 to 1.0 that the value is correct. Use 0 when the value was not found."`
|
||||
SourceURL string `json:"source_url" jsonschema:"The URL where this value was verified, or an empty string."`
|
||||
}
|
||||
|
||||
// CertificationsField is the list-valued counterpart of EnrichedField
|
||||
// used for the certifications array.
|
||||
CertificationsField struct {
|
||||
Values []string `json:"values" jsonschema:"Certification or compliance framework names the vendor publicly claims (e.g. 'SOC 2 Type II', 'ISO 27001', 'HIPAA'). Empty when none are found."`
|
||||
Confidence float64 `json:"confidence" jsonschema:"Confidence from 0.0 to 1.0 in the certifications list. Use 0 when none are found."`
|
||||
SourceURL string `json:"source_url" jsonschema:"The URL where the certifications were found (trust or security page), or an empty string."`
|
||||
}
|
||||
|
||||
// EnrichmentFieldMeta is the per-field provenance recorded in the
|
||||
// common_third_parties.enrichment JSON column.
|
||||
EnrichmentFieldMeta struct {
|
||||
Confidence float64 `json:"confidence"`
|
||||
SourceURL string `json:"source_url,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Source string `json:"source"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// EnrichmentMetadata is the full payload stored in the enrichment
|
||||
// JSON column: run-level bookkeeping plus per-field provenance keyed
|
||||
// by the column name.
|
||||
EnrichmentMetadata struct {
|
||||
Model string `json:"model,omitempty"`
|
||||
AttemptedAt time.Time `json:"attempted_at"`
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Fields map[string]EnrichmentFieldMeta `json:"fields"`
|
||||
}
|
||||
)
|
||||
|
||||
// scalarField describes one *string column the enricher can fill,
|
||||
// pairing the agent's resolved value with accessors into the receiver.
|
||||
type scalarField struct {
|
||||
name string
|
||||
get func(*coredata.CommonThirdParty) *string
|
||||
set func(*coredata.CommonThirdParty, *string)
|
||||
result EnrichedField
|
||||
}
|
||||
|
||||
// parseEnrichmentFields extracts the prior per-field provenance from a
|
||||
// row's enrichment payload. A missing or malformed payload yields an
|
||||
// empty map, which the merge treats as "no field is enrichment-owned".
|
||||
func parseEnrichmentFields(raw json.RawMessage) map[string]EnrichmentFieldMeta {
|
||||
if len(raw) == 0 {
|
||||
return map[string]EnrichmentFieldMeta{}
|
||||
}
|
||||
|
||||
var meta EnrichmentMetadata
|
||||
if err := json.Unmarshal(raw, &meta); err != nil {
|
||||
return map[string]EnrichmentFieldMeta{}
|
||||
}
|
||||
|
||||
if meta.Fields == nil {
|
||||
return map[string]EnrichmentFieldMeta{}
|
||||
}
|
||||
|
||||
return meta.Fields
|
||||
}
|
||||
|
||||
// applyScalarField merges one resolved scalar field into party, honoring
|
||||
// the confidence threshold and prior provenance, and records the
|
||||
// resulting per-field metadata. A value already present that this
|
||||
// enricher did not write (curated seed data or a human edit) is left
|
||||
// untouched. The column is written only when the value clears the
|
||||
// threshold; below it the column keeps its prior value and the metadata
|
||||
// records why nothing was written.
|
||||
func applyScalarField(
|
||||
party *coredata.CommonThirdParty,
|
||||
meta map[string]EnrichmentFieldMeta,
|
||||
prior map[string]EnrichmentFieldMeta,
|
||||
field scalarField,
|
||||
threshold float64,
|
||||
now time.Time,
|
||||
) {
|
||||
value := strings.TrimSpace(field.result.Value)
|
||||
sourceURL := strings.TrimSpace(field.result.SourceURL)
|
||||
|
||||
existing := field.get(party)
|
||||
hasExisting := existing != nil && strings.TrimSpace(*existing) != ""
|
||||
|
||||
priorMeta, hadPrior := prior[field.name]
|
||||
enrichmentOwned := hadPrior && priorMeta.Source == enrichmentSourceEnrichment
|
||||
|
||||
if hasExisting && !enrichmentOwned {
|
||||
meta[field.name] = EnrichmentFieldMeta{
|
||||
Status: enrichmentFieldStatusExternal,
|
||||
Source: enrichmentSourceExternal,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if value != "" && field.result.Confidence >= threshold {
|
||||
v := value
|
||||
field.set(party, &v)
|
||||
meta[field.name] = EnrichmentFieldMeta{
|
||||
Confidence: field.result.Confidence,
|
||||
SourceURL: sourceURL,
|
||||
Status: enrichmentFieldStatusFound,
|
||||
Source: enrichmentSourceEnrichment,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
status := enrichmentFieldStatusNotFound
|
||||
if value != "" {
|
||||
status = enrichmentFieldStatusLowConfidence
|
||||
}
|
||||
|
||||
meta[field.name] = EnrichmentFieldMeta{
|
||||
Confidence: field.result.Confidence,
|
||||
SourceURL: sourceURL,
|
||||
Status: status,
|
||||
Source: enrichmentSourceEnrichment,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
}
|
||||
|
||||
// applyCertifications is the list-valued counterpart of
|
||||
// applyScalarField for the certifications column.
|
||||
func applyCertifications(
|
||||
party *coredata.CommonThirdParty,
|
||||
meta map[string]EnrichmentFieldMeta,
|
||||
prior map[string]EnrichmentFieldMeta,
|
||||
result CertificationsField,
|
||||
threshold float64,
|
||||
now time.Time,
|
||||
) {
|
||||
const name = "certifications"
|
||||
|
||||
values := normalizeCertifications(result.Values)
|
||||
sourceURL := strings.TrimSpace(result.SourceURL)
|
||||
|
||||
hasExisting := len(party.Certifications) > 0
|
||||
|
||||
priorMeta, hadPrior := prior[name]
|
||||
enrichmentOwned := hadPrior && priorMeta.Source == enrichmentSourceEnrichment
|
||||
|
||||
if hasExisting && !enrichmentOwned {
|
||||
meta[name] = EnrichmentFieldMeta{
|
||||
Status: enrichmentFieldStatusExternal,
|
||||
Source: enrichmentSourceExternal,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if len(values) > 0 && result.Confidence >= threshold {
|
||||
party.Certifications = values
|
||||
meta[name] = EnrichmentFieldMeta{
|
||||
Confidence: result.Confidence,
|
||||
SourceURL: sourceURL,
|
||||
Status: enrichmentFieldStatusFound,
|
||||
Source: enrichmentSourceEnrichment,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
status := enrichmentFieldStatusNotFound
|
||||
if len(values) > 0 {
|
||||
status = enrichmentFieldStatusLowConfidence
|
||||
}
|
||||
|
||||
meta[name] = EnrichmentFieldMeta{
|
||||
Confidence: result.Confidence,
|
||||
SourceURL: sourceURL,
|
||||
Status: status,
|
||||
Source: enrichmentSourceEnrichment,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeCertifications trims, drops blanks, and de-duplicates the
|
||||
// certification names returned by the agent, preserving order.
|
||||
func normalizeCertifications(values []string) []string {
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
out := make([]string, 0, len(values))
|
||||
|
||||
for _, v := range values {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
key := strings.ToLower(v)
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
seen[key] = struct{}{}
|
||||
out = append(out, v)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// scalarFields returns the descriptor list pairing each *string column
|
||||
// with the resolved value from the two agents. website_url comes from
|
||||
// the company-profile agent; the document and page URLs come from the
|
||||
// compliance-docs agent.
|
||||
func scalarFields(
|
||||
company CompanyProfileResult,
|
||||
compliance ComplianceDocsResult,
|
||||
) []scalarField {
|
||||
return []scalarField{
|
||||
{
|
||||
name: "legal_name",
|
||||
get: func(p *coredata.CommonThirdParty) *string { return p.LegalName },
|
||||
set: func(p *coredata.CommonThirdParty, v *string) { p.LegalName = v },
|
||||
result: company.LegalName,
|
||||
},
|
||||
{
|
||||
name: "headquarter_address",
|
||||
get: func(p *coredata.CommonThirdParty) *string { return p.HeadquarterAddress },
|
||||
set: func(p *coredata.CommonThirdParty, v *string) { p.HeadquarterAddress = v },
|
||||
result: company.HeadquarterAddress,
|
||||
},
|
||||
{
|
||||
name: "website_url",
|
||||
get: func(p *coredata.CommonThirdParty) *string { return p.WebsiteURL },
|
||||
set: func(p *coredata.CommonThirdParty, v *string) { p.WebsiteURL = v },
|
||||
result: company.WebsiteURL,
|
||||
},
|
||||
{
|
||||
name: "privacy_policy_url",
|
||||
get: func(p *coredata.CommonThirdParty) *string { return p.PrivacyPolicyURL },
|
||||
set: func(p *coredata.CommonThirdParty, v *string) { p.PrivacyPolicyURL = v },
|
||||
result: compliance.PrivacyPolicyURL,
|
||||
},
|
||||
{
|
||||
name: "terms_of_service_url",
|
||||
get: func(p *coredata.CommonThirdParty) *string { return p.TermsOfServiceURL },
|
||||
set: func(p *coredata.CommonThirdParty, v *string) { p.TermsOfServiceURL = v },
|
||||
result: compliance.TermsOfServiceURL,
|
||||
},
|
||||
{
|
||||
name: "service_level_agreement_url",
|
||||
get: func(p *coredata.CommonThirdParty) *string { return p.ServiceLevelAgreementURL },
|
||||
set: func(p *coredata.CommonThirdParty, v *string) { p.ServiceLevelAgreementURL = v },
|
||||
result: compliance.ServiceLevelAgreementURL,
|
||||
},
|
||||
{
|
||||
name: "service_software_agreement_url",
|
||||
get: func(p *coredata.CommonThirdParty) *string { return p.ServiceSoftwareAgreementURL },
|
||||
set: func(p *coredata.CommonThirdParty, v *string) { p.ServiceSoftwareAgreementURL = v },
|
||||
result: compliance.ServiceSoftwareAgreementURL,
|
||||
},
|
||||
{
|
||||
name: "data_processing_agreement_url",
|
||||
get: func(p *coredata.CommonThirdParty) *string { return p.DataProcessingAgreementURL },
|
||||
set: func(p *coredata.CommonThirdParty, v *string) { p.DataProcessingAgreementURL = v },
|
||||
result: compliance.DataProcessingAgreementURL,
|
||||
},
|
||||
{
|
||||
name: "business_associate_agreement_url",
|
||||
get: func(p *coredata.CommonThirdParty) *string { return p.BusinessAssociateAgreementURL },
|
||||
set: func(p *coredata.CommonThirdParty, v *string) { p.BusinessAssociateAgreementURL = v },
|
||||
result: compliance.BusinessAssociateAgreementURL,
|
||||
},
|
||||
{
|
||||
name: "subprocessors_list_url",
|
||||
get: func(p *coredata.CommonThirdParty) *string { return p.SubprocessorsListURL },
|
||||
set: func(p *coredata.CommonThirdParty, v *string) { p.SubprocessorsListURL = v },
|
||||
result: compliance.SubprocessorsListURL,
|
||||
},
|
||||
{
|
||||
name: "status_page_url",
|
||||
get: func(p *coredata.CommonThirdParty) *string { return p.StatusPageURL },
|
||||
set: func(p *coredata.CommonThirdParty, v *string) { p.StatusPageURL = v },
|
||||
result: compliance.StatusPageURL,
|
||||
},
|
||||
{
|
||||
name: "security_page_url",
|
||||
get: func(p *coredata.CommonThirdParty) *string { return p.SecurityPageURL },
|
||||
set: func(p *coredata.CommonThirdParty, v *string) { p.SecurityPageURL = v },
|
||||
result: compliance.SecurityPageURL,
|
||||
},
|
||||
{
|
||||
name: "trust_page_url",
|
||||
get: func(p *coredata.CommonThirdParty) *string { return p.TrustPageURL },
|
||||
set: func(p *coredata.CommonThirdParty, v *string) { p.TrustPageURL = v },
|
||||
result: compliance.TrustPageURL,
|
||||
},
|
||||
}
|
||||
}
|
||||
456
pkg/thirdparty/common_third_party_enrichment_worker.go
vendored
Normal file
456
pkg/thirdparty/common_third_party_enrichment_worker.go
vendored
Normal file
@@ -0,0 +1,456 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package thirdparty
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.gearno.de/kit/httpclient"
|
||||
"go.gearno.de/kit/log"
|
||||
"go.gearno.de/kit/pg"
|
||||
"go.gearno.de/kit/worker"
|
||||
"go.probo.inc/probo/pkg/agent"
|
||||
"go.probo.inc/probo/pkg/agent/tools/browser"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/filemanager"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/llm"
|
||||
)
|
||||
|
||||
const (
|
||||
// defaultEnrichmentAgentTimeout caps a single enrichment agent run.
|
||||
// It is generous because Agent B browses several pages on top of the
|
||||
// LLM round-trips.
|
||||
defaultEnrichmentAgentTimeout = 90 * time.Second
|
||||
|
||||
// defaultEnrichmentMaxTurns bounds an agent's reasoning loop (LLM
|
||||
// call plus tool round-trips). Agent B may navigate the site footer
|
||||
// and trust portal across several turns before synthesizing.
|
||||
defaultEnrichmentMaxTurns = 12
|
||||
|
||||
// defaultEnrichmentMaxTokens caps agent output. The structured
|
||||
// output is moderate; the budget leaves headroom for reasoning
|
||||
// models whose reasoning tokens count against max_tokens.
|
||||
defaultEnrichmentMaxTokens = 8192
|
||||
|
||||
// defaultEnrichmentConfidenceThreshold is the floor a resolved value
|
||||
// must clear before it is written to its column. Values below it are
|
||||
// recorded in the enrichment metadata but not promoted.
|
||||
defaultEnrichmentConfidenceThreshold = 0.7
|
||||
|
||||
// defaultEnrichmentStaleAfter is the idle window after which a
|
||||
// claimed-but-unfinished enrichment is re-armed.
|
||||
defaultEnrichmentStaleAfter = 15 * time.Minute
|
||||
|
||||
// defaultEnrichmentMaxAttempts caps how many times a row is retried
|
||||
// before stale recovery leaves it alone, so a permanently failing
|
||||
// row does not loop forever.
|
||||
defaultEnrichmentMaxAttempts = 3
|
||||
|
||||
enrichmentLogoUserAgent = "Probo-Enricher/1.0"
|
||||
)
|
||||
|
||||
// EnrichmentConfig configures the common-third-party enrichment worker
|
||||
// and the two agents it runs. The worker no-ops when LLMClient is nil;
|
||||
// callers gate registration on config presence. Browser tools for Agent
|
||||
// B are enabled only when ChromeAddr is set; otherwise it relies on
|
||||
// web_search alone. Logo storage is enabled only when FileManager and
|
||||
// Bucket are both set.
|
||||
type EnrichmentConfig struct {
|
||||
LLMClient *llm.Client
|
||||
Model string
|
||||
MaxTokens *int
|
||||
Temperature *float64
|
||||
FirecrawlAPIKey string
|
||||
ChromeAddr string
|
||||
AgentTimeout time.Duration
|
||||
MaxTurns int
|
||||
ConfidenceThreshold float64
|
||||
StaleAfter time.Duration
|
||||
MaxAttempts int
|
||||
|
||||
FileManager *filemanager.Service
|
||||
Bucket string
|
||||
}
|
||||
|
||||
func (c EnrichmentConfig) withDefaults() EnrichmentConfig {
|
||||
if c.AgentTimeout <= 0 {
|
||||
c.AgentTimeout = defaultEnrichmentAgentTimeout
|
||||
}
|
||||
|
||||
if c.MaxTurns < 1 {
|
||||
c.MaxTurns = defaultEnrichmentMaxTurns
|
||||
}
|
||||
|
||||
if c.ConfidenceThreshold <= 0 {
|
||||
c.ConfidenceThreshold = defaultEnrichmentConfidenceThreshold
|
||||
}
|
||||
|
||||
if c.StaleAfter <= 0 {
|
||||
c.StaleAfter = defaultEnrichmentStaleAfter
|
||||
}
|
||||
|
||||
if c.MaxAttempts < 1 {
|
||||
c.MaxAttempts = defaultEnrichmentMaxAttempts
|
||||
}
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func resolveEnrichmentMaxTurns(configured int) int {
|
||||
if configured > 0 {
|
||||
return configured
|
||||
}
|
||||
|
||||
return defaultEnrichmentMaxTurns
|
||||
}
|
||||
|
||||
func resolveEnrichmentMaxTokens(configured *int) int {
|
||||
if configured != nil && *configured > 0 {
|
||||
return *configured
|
||||
}
|
||||
|
||||
return defaultEnrichmentMaxTokens
|
||||
}
|
||||
|
||||
type enrichmentHandler struct {
|
||||
pg *pg.Client
|
||||
logger *log.Logger
|
||||
cfg EnrichmentConfig
|
||||
companyAgent *agent.Agent
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewCommonThirdPartyEnrichmentWorker builds the worker that enriches
|
||||
// global common_third_parties rows. It is a system worker: the catalog
|
||||
// is not tenant-scoped, so a single enrichment benefits all tenants. The
|
||||
// worker no-ops when no LLM client is configured.
|
||||
func NewCommonThirdPartyEnrichmentWorker(
|
||||
pgClient *pg.Client,
|
||||
logger *log.Logger,
|
||||
cfg EnrichmentConfig,
|
||||
opts ...worker.Option,
|
||||
) *worker.Worker[coredata.CommonThirdParty] {
|
||||
cfg = cfg.withDefaults()
|
||||
|
||||
h := &enrichmentHandler{
|
||||
pg: pgClient,
|
||||
logger: logger,
|
||||
cfg: cfg,
|
||||
httpClient: newEnrichmentHTTPClient(),
|
||||
}
|
||||
|
||||
// Agent A has no browser, so it is built once and reused. Agent B is
|
||||
// built per Process because it needs a per-run browser bound to the
|
||||
// process context.
|
||||
if cfg.LLMClient != nil {
|
||||
h.companyAgent = buildCompanyProfileAgent(cfg, logger)
|
||||
}
|
||||
|
||||
return worker.New(
|
||||
"common-third-party-enrichment-worker",
|
||||
h,
|
||||
logger,
|
||||
opts...,
|
||||
)
|
||||
}
|
||||
|
||||
func (h *enrichmentHandler) Claim(ctx context.Context) (coredata.CommonThirdParty, error) {
|
||||
var party coredata.CommonThirdParty
|
||||
|
||||
if err := h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
if err := party.LoadNextForEnrichmentForUpdateSkipLocked(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return party.ClearEnrichmentRequestedAt(ctx, tx)
|
||||
},
|
||||
); err != nil {
|
||||
if errors.Is(err, coredata.ErrResourceNotFound) {
|
||||
return coredata.CommonThirdParty{}, worker.ErrNoTask
|
||||
}
|
||||
|
||||
return coredata.CommonThirdParty{}, fmt.Errorf("cannot claim common third party enrichment task: %w", err)
|
||||
}
|
||||
|
||||
return party, nil
|
||||
}
|
||||
|
||||
// Process runs the enrichment pipeline for one catalog row: Agent A
|
||||
// (company profile) first, then Agent B (compliance docs) and the
|
||||
// deterministic logo step, all outside any transaction. The merged
|
||||
// result and per-field provenance are persisted in a single final
|
||||
// transaction. Process always writes an enrichment payload, even on a
|
||||
// no-result run, so stale recovery does not re-queue the row.
|
||||
func (h *enrichmentHandler) Process(ctx context.Context, party coredata.CommonThirdParty) error {
|
||||
if h.companyAgent == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
prior := parseEnrichmentFields(party.Enrichment)
|
||||
|
||||
var (
|
||||
runErrors []string
|
||||
anySuccess bool
|
||||
)
|
||||
|
||||
// Agent A first: it resolves website_url, which Agent B and the logo
|
||||
// step depend on.
|
||||
company, err := h.runCompanyProfile(ctx, party)
|
||||
if err != nil {
|
||||
h.logger.WarnCtx(ctx, "company profile agent failed", log.Error(err), log.String("common_third_party_id", party.ID.String()))
|
||||
runErrors = append(runErrors, "company_profile: "+err.Error())
|
||||
} else {
|
||||
anySuccess = true
|
||||
}
|
||||
|
||||
website := effectiveWebsiteURL(party, company, h.cfg.ConfidenceThreshold)
|
||||
legalName := effectiveLegalName(party, company, h.cfg.ConfidenceThreshold)
|
||||
|
||||
// Agent B: compliance documents and trust pages.
|
||||
compliance, err := h.runComplianceDocs(ctx, party.Name, website, legalName)
|
||||
if err != nil {
|
||||
h.logger.WarnCtx(ctx, "compliance docs agent failed", log.Error(err), log.String("common_third_party_id", party.ID.String()))
|
||||
runErrors = append(runErrors, "compliance_docs: "+err.Error())
|
||||
} else {
|
||||
anySuccess = true
|
||||
}
|
||||
|
||||
// Deterministic logo step (no LLM). Uploads to S3 outside the final
|
||||
// transaction; the File row is inserted below.
|
||||
logoFile := h.prepareLogo(ctx, party, website)
|
||||
|
||||
meta := make(map[string]EnrichmentFieldMeta)
|
||||
|
||||
for _, field := range scalarFields(company, compliance) {
|
||||
applyScalarField(&party, meta, prior, field, h.cfg.ConfidenceThreshold, now)
|
||||
}
|
||||
|
||||
applyCertifications(&party, meta, prior, compliance.Certifications, h.cfg.ConfidenceThreshold, now)
|
||||
|
||||
status := enrichmentStatusDone
|
||||
switch {
|
||||
case !anySuccess:
|
||||
status = enrichmentStatusFailed
|
||||
case len(runErrors) > 0:
|
||||
status = enrichmentStatusPartial
|
||||
}
|
||||
|
||||
payload := EnrichmentMetadata{
|
||||
Model: h.cfg.Model,
|
||||
AttemptedAt: now,
|
||||
Status: status,
|
||||
Error: strings.Join(runErrors, "; "),
|
||||
Fields: meta,
|
||||
}
|
||||
|
||||
raw, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot marshal enrichment metadata: %w", err)
|
||||
}
|
||||
|
||||
party.Enrichment = raw
|
||||
party.UpdatedAt = now
|
||||
|
||||
return h.pg.WithTx(
|
||||
ctx,
|
||||
func(ctx context.Context, tx pg.Tx) error {
|
||||
if logoFile != nil {
|
||||
if err := logoFile.Insert(ctx, tx, coredata.NewScope(gid.NilTenant)); err != nil {
|
||||
return fmt.Errorf("cannot insert common third party logo file: %w", err)
|
||||
}
|
||||
|
||||
party.LogoFileID = &logoFile.ID
|
||||
|
||||
if err := party.UpdateLogoFileID(ctx, tx); err != nil {
|
||||
return fmt.Errorf("cannot update common third party logo: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := party.UpdateEnrichment(ctx, tx); err != nil {
|
||||
return fmt.Errorf("cannot persist common third party enrichment: %w", err)
|
||||
}
|
||||
|
||||
h.logger.InfoCtx(
|
||||
ctx,
|
||||
"enriched common third party",
|
||||
log.String("common_third_party_id", party.ID.String()),
|
||||
log.String("name", party.Name),
|
||||
log.String("status", status),
|
||||
log.Bool("logo_stored", logoFile != nil),
|
||||
)
|
||||
|
||||
return nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// RecoverStale re-arms enrichment for rows whose run was claimed but
|
||||
// never finished. Claim clears enrichment_requested_at up front, so a
|
||||
// crash between phases would otherwise strand the row.
|
||||
func (h *enrichmentHandler) RecoverStale(ctx context.Context) error {
|
||||
return h.pg.WithConn(
|
||||
ctx,
|
||||
func(ctx context.Context, conn pg.Querier) error {
|
||||
if err := coredata.ResetStaleCommonThirdPartyEnrichments(ctx, conn, h.cfg.StaleAfter, h.cfg.MaxAttempts); err != nil {
|
||||
return fmt.Errorf("cannot reset stale common third party enrichments: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (h *enrichmentHandler) runCompanyProfile(
|
||||
ctx context.Context,
|
||||
party coredata.CommonThirdParty,
|
||||
) (CompanyProfileResult, error) {
|
||||
prompt := buildCompanyProfilePrompt(party)
|
||||
|
||||
agentCtx, cancel := context.WithTimeout(ctx, h.cfg.AgentTimeout)
|
||||
defer cancel()
|
||||
|
||||
result, err := agent.RunTyped[CompanyProfileResult](
|
||||
agentCtx,
|
||||
h.companyAgent,
|
||||
[]llm.Message{
|
||||
{
|
||||
Role: llm.RoleUser,
|
||||
Parts: []llm.Part{llm.TextPart{Text: prompt}},
|
||||
},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return CompanyProfileResult{}, fmt.Errorf("company profile agent run failed: %w", err)
|
||||
}
|
||||
|
||||
return result.Output, nil
|
||||
}
|
||||
|
||||
// runComplianceDocs builds Agent B with a per-run browser when a Chrome
|
||||
// endpoint is configured, then runs it. The browser is closed when the
|
||||
// run returns. The browser is intentionally not pinned to the vendor
|
||||
// domain so the agent can follow links to hosted trust portals (Vanta,
|
||||
// SafeBase, etc.); SSRF protection still blocks non-public hosts.
|
||||
func (h *enrichmentHandler) runComplianceDocs(
|
||||
ctx context.Context,
|
||||
name string,
|
||||
website string,
|
||||
legalName string,
|
||||
) (ComplianceDocsResult, error) {
|
||||
var browserTools []agent.Tool
|
||||
|
||||
if h.cfg.ChromeAddr != "" {
|
||||
webBrowser := browser.NewBrowser(ctx, h.cfg.ChromeAddr)
|
||||
defer webBrowser.Close()
|
||||
|
||||
browserTools = browser.NewReadOnlyToolset(webBrowser).Tools()
|
||||
}
|
||||
|
||||
complianceAgent := buildComplianceDocsAgent(h.cfg, h.logger, browserTools)
|
||||
|
||||
prompt := buildComplianceDocsPrompt(name, website, legalName)
|
||||
|
||||
agentCtx, cancel := context.WithTimeout(ctx, h.cfg.AgentTimeout)
|
||||
defer cancel()
|
||||
|
||||
result, err := agent.RunTyped[ComplianceDocsResult](
|
||||
agentCtx,
|
||||
complianceAgent,
|
||||
[]llm.Message{
|
||||
{
|
||||
Role: llm.RoleUser,
|
||||
Parts: []llm.Part{llm.TextPart{Text: prompt}},
|
||||
},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return ComplianceDocsResult{}, fmt.Errorf("compliance docs agent run failed: %w", err)
|
||||
}
|
||||
|
||||
return result.Output, nil
|
||||
}
|
||||
|
||||
// effectiveWebsiteURL is the website passed to Agent B and the logo step.
|
||||
// A curated value already on the row wins (seed data and human edits are
|
||||
// trusted); otherwise Agent A's value is used when it clears the
|
||||
// confidence threshold.
|
||||
func effectiveWebsiteURL(
|
||||
party coredata.CommonThirdParty,
|
||||
company CompanyProfileResult,
|
||||
threshold float64,
|
||||
) string {
|
||||
if party.WebsiteURL != nil {
|
||||
if v := strings.TrimSpace(*party.WebsiteURL); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
if v := strings.TrimSpace(company.WebsiteURL.Value); v != "" && company.WebsiteURL.Confidence >= threshold {
|
||||
return v
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// effectiveLegalName is the legal name hint passed to Agent B, resolved
|
||||
// the same way as effectiveWebsiteURL.
|
||||
func effectiveLegalName(
|
||||
party coredata.CommonThirdParty,
|
||||
company CompanyProfileResult,
|
||||
threshold float64,
|
||||
) string {
|
||||
if party.LegalName != nil {
|
||||
if v := strings.TrimSpace(*party.LegalName); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
if v := strings.TrimSpace(company.LegalName.Value); v != "" && company.LegalName.Confidence >= threshold {
|
||||
return v
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
type userAgentRoundTripper struct {
|
||||
next http.RoundTripper
|
||||
}
|
||||
|
||||
func (t *userAgentRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
r2 := r.Clone(r.Context())
|
||||
r2.Header.Set("User-Agent", enrichmentLogoUserAgent)
|
||||
|
||||
return t.next.RoundTrip(r2)
|
||||
}
|
||||
|
||||
// newEnrichmentHTTPClient builds the SSRF-protected client used by the
|
||||
// deterministic logo step.
|
||||
func newEnrichmentHTTPClient() *http.Client {
|
||||
client := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
|
||||
client.Timeout = 20 * time.Second
|
||||
client.Transport = &userAgentRoundTripper{next: client.Transport}
|
||||
|
||||
return client
|
||||
}
|
||||
207
pkg/thirdparty/common_third_party_logo.go
vendored
Normal file
207
pkg/thirdparty/common_third_party_logo.go
vendored
Normal file
@@ -0,0 +1,207 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package thirdparty
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.gearno.de/crypto/uuid"
|
||||
"go.gearno.de/kit/log"
|
||||
"go.probo.inc/probo/pkg/coredata"
|
||||
"go.probo.inc/probo/pkg/gid"
|
||||
"go.probo.inc/probo/pkg/webinspect"
|
||||
)
|
||||
|
||||
// maxLogoSize caps a downloaded logo image. Logos are small; the cap
|
||||
// guards against an oversized or malicious response.
|
||||
const maxLogoSize = 5 << 20 // 5 MiB
|
||||
|
||||
// prepareLogo discovers and uploads a vendor logo to S3, returning a
|
||||
// fully-populated (but not yet inserted) File record for the caller to
|
||||
// persist in its transaction. It is deterministic and best-effort: any
|
||||
// failure logs and returns (nil, nil) so a missing logo never fails the
|
||||
// enrichment run.
|
||||
//
|
||||
// It no-ops when the row already has a logo, when logo storage is not
|
||||
// configured, or when no website is known. The S3 upload happens here,
|
||||
// outside any transaction; the caller inserts the File row and links it
|
||||
// via UpdateLogoFileID.
|
||||
func (h *enrichmentHandler) prepareLogo(
|
||||
ctx context.Context,
|
||||
party coredata.CommonThirdParty,
|
||||
websiteURL string,
|
||||
) *coredata.File {
|
||||
if party.LogoFileID != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if h.cfg.FileManager == nil || h.cfg.Bucket == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
website := strings.TrimSpace(websiteURL)
|
||||
if website == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
data, contentType, err := fetchCommonThirdPartyLogo(ctx, h.httpClient, website)
|
||||
if err != nil {
|
||||
h.logger.InfoCtx(
|
||||
ctx,
|
||||
"could not fetch common third party logo",
|
||||
log.String("common_third_party_id", party.ID.String()),
|
||||
log.Error(err),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
objectKey, err := uuid.NewV7()
|
||||
if err != nil {
|
||||
h.logger.WarnCtx(ctx, "cannot generate logo object key", log.Error(err))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
fileRecord := &coredata.File{
|
||||
ID: gid.New(gid.NilTenant, coredata.FileEntityType),
|
||||
OrganizationID: gid.Nil,
|
||||
BucketName: h.cfg.Bucket,
|
||||
MimeType: contentType,
|
||||
FileName: party.Name + "-logo" + webinspect.ExtensionForMIME(contentType),
|
||||
FileKey: objectKey.String(),
|
||||
FileSize: int64(len(data)),
|
||||
Visibility: coredata.FileVisibilityPublic,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
size, err := h.cfg.FileManager.PutFile(
|
||||
ctx,
|
||||
fileRecord,
|
||||
bytes.NewReader(data),
|
||||
map[string]string{
|
||||
"type": "common-third-party-logo",
|
||||
"common-third-party-id": party.ID.String(),
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
h.logger.WarnCtx(
|
||||
ctx,
|
||||
"cannot upload common third party logo",
|
||||
log.String("common_third_party_id", party.ID.String()),
|
||||
log.Error(err),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
fileRecord.FileSize = size
|
||||
|
||||
return fileRecord
|
||||
}
|
||||
|
||||
// fetchCommonThirdPartyLogo finds the best logo for a website and
|
||||
// downloads it. It first parses the page's <head> for icon links
|
||||
// (webinspect), then falls back to well-known icon paths on the same
|
||||
// host. The supplied client must enforce SSRF protection.
|
||||
func fetchCommonThirdPartyLogo(
|
||||
ctx context.Context,
|
||||
client *http.Client,
|
||||
websiteURL string,
|
||||
) (data []byte, contentType string, err error) {
|
||||
candidates := make([]string, 0, 3)
|
||||
|
||||
pageInfo, parseErr := webinspect.Parse(ctx, client, websiteURL)
|
||||
if parseErr == nil {
|
||||
if logoURL, logoErr := webinspect.FindLogoURL(pageInfo); logoErr == nil {
|
||||
candidates = append(candidates, logoURL)
|
||||
}
|
||||
}
|
||||
|
||||
if parsed, parseURLErr := url.Parse(websiteURL); parseURLErr == nil && parsed.Host != "" {
|
||||
base := url.URL{Scheme: parsed.Scheme, Host: parsed.Host}
|
||||
if base.Scheme == "" {
|
||||
base.Scheme = "https"
|
||||
}
|
||||
|
||||
candidates = append(
|
||||
candidates,
|
||||
base.ResolveReference(&url.URL{Path: "/apple-touch-icon.png"}).String(),
|
||||
base.ResolveReference(&url.URL{Path: "/favicon.ico"}).String(),
|
||||
)
|
||||
}
|
||||
|
||||
for _, candidate := range candidates {
|
||||
data, contentType, err = downloadImage(ctx, client, candidate)
|
||||
if err == nil {
|
||||
return data, contentType, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, "", fmt.Errorf("cannot fetch logo for %s", websiteURL)
|
||||
}
|
||||
|
||||
// downloadImage fetches a single candidate URL and returns its bytes when
|
||||
// the response is a non-empty image within the size cap.
|
||||
func downloadImage(
|
||||
ctx context.Context,
|
||||
client *http.Client,
|
||||
rawURL string,
|
||||
) ([]byte, string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("cannot create logo request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("cannot fetch logo: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, "", fmt.Errorf("cannot fetch logo: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if idx := strings.Index(contentType, ";"); idx != -1 {
|
||||
contentType = contentType[:idx]
|
||||
}
|
||||
contentType = strings.TrimSpace(contentType)
|
||||
|
||||
if !strings.HasPrefix(contentType, "image/") {
|
||||
return nil, "", fmt.Errorf("logo response is not an image: %q", contentType)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxLogoSize))
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("cannot read logo body: %w", err)
|
||||
}
|
||||
|
||||
if len(body) == 0 {
|
||||
return nil, "", fmt.Errorf("logo response is empty")
|
||||
}
|
||||
|
||||
return body, contentType, nil
|
||||
}
|
||||
33
pkg/thirdparty/prompts/common_third_party_company_profile.txt.tmpl
vendored
Normal file
33
pkg/thirdparty/prompts/common_third_party_company_profile.txt.tmpl
vendored
Normal file
@@ -0,0 +1,33 @@
|
||||
You are a research agent that builds a factual company profile for a software
|
||||
vendor or service provider. You are given the vendor's name (and sometimes a
|
||||
known website). Return only verifiable identity facts.
|
||||
|
||||
Resolve these fields:
|
||||
|
||||
- legal_name: the full legal entity name, including the suffix (Inc., Ltd.,
|
||||
GmbH, S.A.S., etc.). Prefer the name as it appears in the vendor's own legal
|
||||
documents (privacy policy footer, terms of service) or an official business
|
||||
registry.
|
||||
- headquarter_address: the postal address of the company's headquarters.
|
||||
- website_url: the canonical primary marketing website. Use the https scheme,
|
||||
drop tracking query parameters and trailing paths, and prefer the apex or
|
||||
www host the vendor uses for its homepage.
|
||||
|
||||
Method:
|
||||
|
||||
- Use the web_search tool when it is available to confirm facts. Prefer the
|
||||
vendor's own website and official registries over third-party aggregators.
|
||||
- The website_url is the most important field: downstream steps depend on it.
|
||||
Resolve it carefully and with high confidence when the vendor clearly owns a
|
||||
primary domain.
|
||||
|
||||
Rules:
|
||||
|
||||
- Never guess. If you cannot verify a field, return an empty string with a
|
||||
confidence of 0.
|
||||
- confidence is your own 0.0-1.0 estimate that the value is correct. Reserve
|
||||
values above 0.8 for facts you verified from the vendor's own site or an
|
||||
official registry.
|
||||
- source_url is the page where you verified the value. Leave it empty when the
|
||||
value was not found.
|
||||
- Do not include commentary; return only the structured fields.
|
||||
51
pkg/thirdparty/prompts/common_third_party_compliance_docs.txt.tmpl
vendored
Normal file
51
pkg/thirdparty/prompts/common_third_party_compliance_docs.txt.tmpl
vendored
Normal file
@@ -0,0 +1,51 @@
|
||||
You are a research agent that locates a software vendor's public compliance
|
||||
documents and trust pages. You are given the vendor's name and, usually, its
|
||||
website. Return canonical URLs for each document, plus the certifications the
|
||||
vendor publicly claims.
|
||||
|
||||
Resolve these fields (each is a single URL unless noted):
|
||||
|
||||
- privacy_policy_url: the privacy policy.
|
||||
- terms_of_service_url: the terms of service / terms of use.
|
||||
- service_level_agreement_url: the public SLA. Frequently gated behind sales.
|
||||
- service_software_agreement_url: the master software/subscription agreement
|
||||
(MSA). Often gated, or the same document as the terms of service.
|
||||
- data_processing_agreement_url: the DPA. Often a downloadable PDF; sometimes
|
||||
only available on request.
|
||||
- business_associate_agreement_url: the HIPAA BAA. Almost always gated behind
|
||||
sales or an enterprise plan.
|
||||
- subprocessors_list_url: the sub-processors list page.
|
||||
- status_page_url: the uptime/status page (commonly status.<domain> or a
|
||||
hosted statuspage.io / instatus / better-uptime page).
|
||||
- security_page_url: the security overview page.
|
||||
- trust_page_url: the trust center / trust portal. Many vendors host this on
|
||||
Vanta, SafeBase, Drata, Conveyor, or a /trust path.
|
||||
- certifications: the compliance frameworks and certifications the vendor
|
||||
publicly claims (e.g. SOC 2 Type II, ISO 27001, ISO 27701, PCI DSS, HIPAA,
|
||||
GDPR, FedRAMP). Read these from the trust or security page.
|
||||
|
||||
Method:
|
||||
|
||||
- Start from the vendor's <website> when provided. Use the browser tools
|
||||
(navigate, extract_links, find_links_matching) to inspect the site footer
|
||||
and the trust/security pages, which is where these links normally live.
|
||||
- Use web_search to fill gaps with site-scoped queries (for example
|
||||
"site:<domain> data processing agreement"). Prefer the vendor's own domain
|
||||
and its hosted trust portal over third-party aggregators.
|
||||
- Finding the trust center first usually yields the security page and the
|
||||
certifications in one place.
|
||||
|
||||
Rules:
|
||||
|
||||
- Return the most specific canonical URL. Prefer a direct document/page URL
|
||||
over a generic legal-index page.
|
||||
- Never guess or fabricate a URL. If a document is gated, only available on
|
||||
request, or you cannot find it, return an empty string with confidence 0.
|
||||
Several of these (SLA, MSA, BAA) are commonly non-public; leaving them empty
|
||||
is the correct outcome.
|
||||
- confidence is your own 0.0-1.0 estimate that the URL is correct and current.
|
||||
Reserve values above 0.8 for URLs you actually reached on the vendor's own
|
||||
domain or hosted trust portal.
|
||||
- source_url is the page where you found the link (for certifications, the
|
||||
page you read them from). Leave it empty when nothing was found.
|
||||
- Do not include commentary; return only the structured fields.
|
||||
10
pkg/thirdparty/resolver.go
vendored
10
pkg/thirdparty/resolver.go
vendored
@@ -70,8 +70,14 @@ func ResolveOrCreateCommonThirdParty(
|
||||
Slug: partySlug,
|
||||
Category: category,
|
||||
Certifications: []string{},
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
// Request enrichment at creation: a freshly resolved catalog row
|
||||
// carries only name/slug/category, so the enrichment worker fills
|
||||
// the rest (URLs, address, certifications, logo). Curated seed
|
||||
// rows are inserted via Upsert without this flag, so a full
|
||||
// re-seed does not trigger an enrichment storm.
|
||||
EnrichmentRequestedAt: &now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
|
||||
// Insert inside a savepoint so a concurrent transaction that created
|
||||
|
||||
180
pkg/webinspect/logo.go
Normal file
180
pkg/webinspect/logo.go
Normal file
@@ -0,0 +1,180 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package webinspect
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func FindLogoURL(info *PageInfo) (string, error) {
|
||||
head := findElement(info.Root, "head")
|
||||
if head == nil {
|
||||
return "", fmt.Errorf("cannot find logo: no head element")
|
||||
}
|
||||
|
||||
var (
|
||||
svgIcon string
|
||||
appleTouchIcon string
|
||||
appleTouchSize int
|
||||
largestIcon string
|
||||
largestSize int
|
||||
msTileImage string
|
||||
)
|
||||
|
||||
for _, n := range findAllIn(head, "link") {
|
||||
rel := strings.ToLower(attrVal(n, "rel"))
|
||||
href := attrVal(n, "href")
|
||||
if href == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.Contains(rel, "icon") && !strings.Contains(rel, "apple-touch-icon") && attrVal(n, "type") == "image/svg+xml":
|
||||
svgIcon = href
|
||||
case strings.Contains(rel, "apple-touch-icon"):
|
||||
size := parseSizeAttr(attrVal(n, "sizes"))
|
||||
if appleTouchIcon == "" || size > appleTouchSize {
|
||||
appleTouchIcon = href
|
||||
appleTouchSize = size
|
||||
}
|
||||
case strings.Contains(rel, "icon") && !strings.Contains(rel, "apple-touch-icon"):
|
||||
size := parseSizeAttr(attrVal(n, "sizes"))
|
||||
if largestIcon == "" || size > largestSize {
|
||||
largestIcon = href
|
||||
largestSize = size
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, n := range findAllIn(head, "meta") {
|
||||
name := strings.ToLower(attrVal(n, "name"))
|
||||
content := attrVal(n, "content")
|
||||
if name == "msapplication-tileimage" && content != "" {
|
||||
msTileImage = content
|
||||
}
|
||||
}
|
||||
|
||||
candidates := []string{
|
||||
svgIcon,
|
||||
appleTouchIcon,
|
||||
largestIcon,
|
||||
msTileImage,
|
||||
}
|
||||
|
||||
for _, href := range candidates {
|
||||
if href != "" {
|
||||
return info.ResolveHref(href), nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("cannot find logo")
|
||||
}
|
||||
|
||||
func parseSizeAttr(sizes string) int {
|
||||
if sizes == "" || strings.EqualFold(sizes, "any") {
|
||||
return 0
|
||||
}
|
||||
|
||||
best := 0
|
||||
for token := range strings.FieldsSeq(sizes) {
|
||||
token = strings.ToLower(token)
|
||||
parts := strings.SplitN(token, "x", 2)
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
w, err := strconv.Atoi(parts[0])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if w > best {
|
||||
best = w
|
||||
}
|
||||
}
|
||||
|
||||
return best
|
||||
}
|
||||
|
||||
func ExtensionForMIME(contentType string) string {
|
||||
ct := strings.ToLower(contentType)
|
||||
if idx := strings.Index(ct, ";"); idx != -1 {
|
||||
ct = ct[:idx]
|
||||
}
|
||||
ct = strings.TrimSpace(ct)
|
||||
|
||||
switch ct {
|
||||
case "image/svg+xml":
|
||||
return ".svg"
|
||||
case "image/png":
|
||||
return ".png"
|
||||
case "image/jpeg":
|
||||
return ".jpg"
|
||||
case "image/gif":
|
||||
return ".gif"
|
||||
case "image/webp":
|
||||
return ".webp"
|
||||
case "image/x-icon", "image/vnd.microsoft.icon":
|
||||
return ".ico"
|
||||
default:
|
||||
return ".png"
|
||||
}
|
||||
}
|
||||
|
||||
// HeadLinks returns all <link> nodes from <head> whose rel attribute
|
||||
// contains the given value (case-insensitive partial match).
|
||||
func (p *PageInfo) HeadLinks(rel string) []*html.Node {
|
||||
head := findElement(p.Root, "head")
|
||||
if head == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
rel = strings.ToLower(rel)
|
||||
var matches []*html.Node
|
||||
|
||||
for _, n := range findAllIn(head, "link") {
|
||||
if strings.Contains(strings.ToLower(attrVal(n, "rel")), rel) {
|
||||
matches = append(matches, n)
|
||||
}
|
||||
}
|
||||
|
||||
return matches
|
||||
}
|
||||
|
||||
// HeadMeta returns the content attribute of the first <meta> tag in <head>
|
||||
// whose name attribute matches (case-insensitive).
|
||||
func (p *PageInfo) HeadMeta(name string) (string, bool) {
|
||||
head := findElement(p.Root, "head")
|
||||
if head == nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
name = strings.ToLower(name)
|
||||
|
||||
for _, n := range findAllIn(head, "meta") {
|
||||
if strings.EqualFold(attrVal(n, "name"), name) {
|
||||
content := attrVal(n, "content")
|
||||
if content != "" {
|
||||
return content, true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
163
pkg/webinspect/logo_test.go
Normal file
163
pkg/webinspect/logo_test.go
Normal file
@@ -0,0 +1,163 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
package webinspect_test
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.probo.inc/probo/pkg/webinspect"
|
||||
)
|
||||
|
||||
func parseTestHTML(t *testing.T, rawURL string, body string) *webinspect.PageInfo {
|
||||
t.Helper()
|
||||
|
||||
u, err := url.Parse(rawURL)
|
||||
require.NoError(t, err)
|
||||
|
||||
info, err := webinspect.ParseHTML(u, strings.NewReader(body))
|
||||
require.NoError(t, err)
|
||||
|
||||
return info
|
||||
}
|
||||
|
||||
func TestFindLogoURL_SVGPreferred(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
info := parseTestHTML(t, "https://example.com", `<html><head>
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg">
|
||||
<link rel="apple-touch-icon" href="/apple-touch-icon.png">
|
||||
<link rel="icon" sizes="192x192" href="/icon-192.png">
|
||||
</head><body></body></html>`)
|
||||
|
||||
got, err := webinspect.FindLogoURL(info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "https://example.com/favicon.svg", got)
|
||||
}
|
||||
|
||||
func TestFindLogoURL_AppleTouchIconSecond(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
info := parseTestHTML(t, "https://example.com", `<html><head>
|
||||
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
|
||||
<link rel="icon" sizes="32x32" href="/favicon-32.png">
|
||||
</head><body></body></html>`)
|
||||
|
||||
got, err := webinspect.FindLogoURL(info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "https://example.com/apple-touch-icon.png", got)
|
||||
}
|
||||
|
||||
func TestFindLogoURL_AppleTouchIconLargest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
info := parseTestHTML(t, "https://example.com", `<html><head>
|
||||
<link rel="apple-touch-icon" sizes="120x120" href="/touch-120.png">
|
||||
<link rel="apple-touch-icon" sizes="180x180" href="/touch-180.png">
|
||||
<link rel="apple-touch-icon" sizes="152x152" href="/touch-152.png">
|
||||
</head><body></body></html>`)
|
||||
|
||||
got, err := webinspect.FindLogoURL(info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "https://example.com/touch-180.png", got)
|
||||
}
|
||||
|
||||
func TestFindLogoURL_LargestIcon(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
info := parseTestHTML(t, "https://example.com", `<html><head>
|
||||
<link rel="icon" sizes="32x32" href="/icon-32.png">
|
||||
<link rel="icon" sizes="192x192" href="/icon-192.png">
|
||||
<link rel="icon" sizes="16x16" href="/icon-16.png">
|
||||
</head><body></body></html>`)
|
||||
|
||||
got, err := webinspect.FindLogoURL(info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "https://example.com/icon-192.png", got)
|
||||
}
|
||||
|
||||
func TestFindLogoURL_MsTileImage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
info := parseTestHTML(t, "https://example.com", `<html><head>
|
||||
<meta name="msapplication-TileImage" content="/mstile-144.png">
|
||||
</head><body></body></html>`)
|
||||
|
||||
got, err := webinspect.FindLogoURL(info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "https://example.com/mstile-144.png", got)
|
||||
}
|
||||
|
||||
func TestFindLogoURL_RelativeHrefResolved(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
info := parseTestHTML(t, "https://cdn.example.com/app/", `<html><head>
|
||||
<link rel="icon" type="image/svg+xml" href="../assets/logo.svg">
|
||||
</head><body></body></html>`)
|
||||
|
||||
got, err := webinspect.FindLogoURL(info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "https://cdn.example.com/assets/logo.svg", got)
|
||||
}
|
||||
|
||||
func TestFindLogoURL_NoHead(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
info := parseTestHTML(t, "https://example.com", `<html><body><p>no head</p></body></html>`)
|
||||
|
||||
_, err := webinspect.FindLogoURL(info)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestFindLogoURL_NothingFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
info := parseTestHTML(t, "https://example.com", `<html><head></head><body></body></html>`)
|
||||
|
||||
_, err := webinspect.FindLogoURL(info)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestExtensionForMIME(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
contentType string
|
||||
expected string
|
||||
}{
|
||||
{"image/svg+xml", ".svg"},
|
||||
{"image/png", ".png"},
|
||||
{"image/jpeg", ".jpg"},
|
||||
{"image/gif", ".gif"},
|
||||
{"image/webp", ".webp"},
|
||||
{"image/x-icon", ".ico"},
|
||||
{"image/vnd.microsoft.icon", ".ico"},
|
||||
{"image/png; charset=utf-8", ".png"},
|
||||
{"application/octet-stream", ".png"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(
|
||||
tt.contentType,
|
||||
func(t *testing.T) {
|
||||
t.Parallel()
|
||||
assert.Equal(t, tt.expected, webinspect.ExtensionForMIME(tt.contentType))
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
117
pkg/webinspect/parse.go
Normal file
117
pkg/webinspect/parse.go
Normal file
@@ -0,0 +1,117 @@
|
||||
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||
//
|
||||
// Permission to use, copy, modify, and/or distribute this software for any
|
||||
// purpose with or without fee is hereby granted, provided that the above
|
||||
// copyright notice and this permission notice appear in all copies.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||
// PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
// Package webinspect parses a web page's static HTML to extract metadata
|
||||
// from its <head> (icons/logos, meta tags, link relations). It is a pure,
|
||||
// deterministic helper: callers supply an http.Client (e.g. one with SSRF
|
||||
// protection) so the package itself makes no policy decisions about which
|
||||
// hosts are reachable. The logo discovery is reused by the common
|
||||
// third-party enricher to populate logo_file_id without an LLM.
|
||||
package webinspect
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
type PageInfo struct {
|
||||
URL *url.URL
|
||||
Root *html.Node
|
||||
}
|
||||
|
||||
func Parse(ctx context.Context, client *http.Client, websiteURL string) (*PageInfo, error) {
|
||||
parsed, err := url.Parse(websiteURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse website URL: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, websiteURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot fetch page: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("cannot fetch page: status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
const maxHTMLSize = 10 << 20 // 10 MiB
|
||||
return ParseHTML(parsed, io.LimitReader(resp.Body, maxHTMLSize))
|
||||
}
|
||||
|
||||
func ParseHTML(baseURL *url.URL, r io.Reader) (*PageInfo, error) {
|
||||
root, err := html.Parse(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse HTML: %w", err)
|
||||
}
|
||||
|
||||
return &PageInfo{URL: baseURL, Root: root}, nil
|
||||
}
|
||||
|
||||
func (p *PageInfo) ResolveHref(href string) string {
|
||||
ref, err := url.Parse(href)
|
||||
if err != nil {
|
||||
return href
|
||||
}
|
||||
|
||||
return p.URL.ResolveReference(ref).String()
|
||||
}
|
||||
|
||||
func findElement(n *html.Node, tag string) *html.Node {
|
||||
if n.Type == html.ElementNode && n.Data == tag {
|
||||
return n
|
||||
}
|
||||
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if found := findElement(c, tag); found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func findAllIn(parent *html.Node, tag string) []*html.Node {
|
||||
var nodes []*html.Node
|
||||
|
||||
for c := parent.FirstChild; c != nil; c = c.NextSibling {
|
||||
if c.Type == html.ElementNode && c.Data == tag {
|
||||
nodes = append(nodes, c)
|
||||
}
|
||||
|
||||
nodes = append(nodes, findAllIn(c, tag)...)
|
||||
}
|
||||
|
||||
return nodes
|
||||
}
|
||||
|
||||
func attrVal(n *html.Node, key string) string {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == key {
|
||||
return a.Val
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
Reference in New Issue
Block a user