Add common third party enricher worker
Introduce a poll-based worker that fills the global common_third_parties catalog (URLs, headquarter address, legal name, certifications, logo) so each tenant no longer starts from sparse, name-only rows. Enrichment is requested at row creation by ResolveOrCreateCommonThirdParty; curated seed rows are not enqueued, to avoid a re-seed storm. The pipeline uses two specialized agents plus a deterministic logo step. Agent A (company profile) resolves legal name, headquarter address, and the canonical website over web search; its website and legal name feed Agent B and the logo step. Agent B (compliance docs) resolves the legal document URLs, trust/security/status pages, and certifications using the browser read-only toolset (gated on ChromeDPAddr) plus web search. The logo step restores pkg/webinspect as a pure deterministic package and stores the discovered icon in S3, linked via logo_file_id. Each agent returns per-field value/confidence/source_url. The worker writes a column only when confidence clears a configurable threshold and the field is not externally owned (seed or human), and always records full per-field provenance in a new enrichment JSONB column so re-runs fill only gaps and human edits are never clobbered. New bookkeeping columns (enrichment_requested_at, enrichment, enrichment_attempts) back the claim queue and stale recovery; agents run outside transactions and results persist in one final transaction. The worker is opt-in: it no-ops unless its agent provider is configured. Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
@@ -378,6 +378,52 @@ spec:
|
|||||||
- name: COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS
|
- name: COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS
|
||||||
value: {{ .Values.probo.commonPatternEnrichmentWorker.agentMaxTurns | quote }}
|
value: {{ .Values.probo.commonPatternEnrichmentWorker.agentMaxTurns | quote }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
# Common Third Party Enrichment Agent
|
||||||
|
{{- if .Values.probo.commonThirdPartyEnrichment.provider }}
|
||||||
|
- name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_PROVIDER
|
||||||
|
value: {{ .Values.probo.commonThirdPartyEnrichment.provider | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.probo.commonThirdPartyEnrichment.modelName }}
|
||||||
|
- name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MODEL_NAME
|
||||||
|
value: {{ .Values.probo.commonThirdPartyEnrichment.modelName | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.probo.commonThirdPartyEnrichment.temperature }}
|
||||||
|
- name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_TEMPERATURE
|
||||||
|
value: {{ .Values.probo.commonThirdPartyEnrichment.temperature | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.probo.commonThirdPartyEnrichment.maxTokens }}
|
||||||
|
- name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MAX_TOKENS
|
||||||
|
value: {{ .Values.probo.commonThirdPartyEnrichment.maxTokens | quote }}
|
||||||
|
{{- end }}
|
||||||
|
# Common Third Party Enrichment Worker
|
||||||
|
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.interval }}
|
||||||
|
- name: COMMON_THIRD_PARTY_ENRICHMENT_INTERVAL
|
||||||
|
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.interval | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.maxConcurrency }}
|
||||||
|
- name: COMMON_THIRD_PARTY_ENRICHMENT_MAX_CONCURRENCY
|
||||||
|
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.maxConcurrency | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.staleAfter }}
|
||||||
|
- name: COMMON_THIRD_PARTY_ENRICHMENT_STALE_AFTER
|
||||||
|
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.staleAfter | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.agentTimeout }}
|
||||||
|
- name: COMMON_THIRD_PARTY_ENRICHMENT_AGENT_TIMEOUT
|
||||||
|
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.agentTimeout | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.agentMaxTurns }}
|
||||||
|
- name: COMMON_THIRD_PARTY_ENRICHMENT_AGENT_MAX_TURNS
|
||||||
|
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.agentMaxTurns | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.confidenceThreshold }}
|
||||||
|
- name: COMMON_THIRD_PARTY_ENRICHMENT_CONFIDENCE_THRESHOLD
|
||||||
|
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.confidenceThreshold | quote }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.maxAttempts }}
|
||||||
|
- name: COMMON_THIRD_PARTY_ENRICHMENT_MAX_ATTEMPTS
|
||||||
|
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.maxAttempts | quote }}
|
||||||
|
{{- end }}
|
||||||
# Custom Domains
|
# Custom Domains
|
||||||
{{- if .Values.probo.customDomains.enabled }}
|
{{- if .Values.probo.customDomains.enabled }}
|
||||||
- name: CUSTOM_DOMAINS_RENEWAL_INTERVAL
|
- name: CUSTOM_DOMAINS_RENEWAL_INTERVAL
|
||||||
|
|||||||
@@ -229,6 +229,26 @@ probo:
|
|||||||
# agentTimeout: 45
|
# agentTimeout: 45
|
||||||
# agentMaxTurns: 10
|
# agentMaxTurns: 10
|
||||||
|
|
||||||
|
# Common third party enrichment agent (optional, fills common_third_parties
|
||||||
|
# metadata: URLs, address, certifications, logo).
|
||||||
|
# commonThirdPartyEnrichment:
|
||||||
|
# provider: "openai"
|
||||||
|
# modelName: "gpt-4o"
|
||||||
|
# temperature: "0.1"
|
||||||
|
# maxTokens: "8192"
|
||||||
|
|
||||||
|
# Common third party enrichment worker tuning (optional; seconds for
|
||||||
|
# interval/staleAfter/agentTimeout). confidenceThreshold is the 0-1 floor
|
||||||
|
# a value must clear before it is written.
|
||||||
|
# commonThirdPartyEnrichmentWorker:
|
||||||
|
# interval: 10
|
||||||
|
# maxConcurrency: 1
|
||||||
|
# staleAfter: 900
|
||||||
|
# agentTimeout: 90
|
||||||
|
# agentMaxTurns: 12
|
||||||
|
# confidenceThreshold: 0.7
|
||||||
|
# maxAttempts: 3
|
||||||
|
|
||||||
# OpenTelemetry tracing (optional)
|
# OpenTelemetry tracing (optional)
|
||||||
tracing:
|
tracing:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
|||||||
@@ -331,6 +331,30 @@ probo:
|
|||||||
agentTimeout: 45
|
agentTimeout: 45
|
||||||
agentMaxTurns: 10
|
agentMaxTurns: 10
|
||||||
|
|
||||||
|
# Common third party enrichment agent (optional, requires a provider
|
||||||
|
# key). Powers the company-profile and compliance-docs agents that fill
|
||||||
|
# common_third_parties metadata. Browser tools for the compliance-docs
|
||||||
|
# agent use the chromeDpAddr endpoint when set; web search uses
|
||||||
|
# firecrawl.apiKey when set.
|
||||||
|
commonThirdPartyEnrichment:
|
||||||
|
provider: ""
|
||||||
|
modelName: ""
|
||||||
|
temperature: ""
|
||||||
|
maxTokens: ""
|
||||||
|
|
||||||
|
# Common third party enrichment background worker tuning (optional).
|
||||||
|
# interval, staleAfter, and agentTimeout are in seconds.
|
||||||
|
# confidenceThreshold is the floor (0-1) a resolved value must clear
|
||||||
|
# before it is written to its column.
|
||||||
|
commonThirdPartyEnrichmentWorker:
|
||||||
|
interval: 10
|
||||||
|
maxConcurrency: 1
|
||||||
|
staleAfter: 900
|
||||||
|
agentTimeout: 90
|
||||||
|
agentMaxTurns: 12
|
||||||
|
confidenceThreshold: 0.7
|
||||||
|
maxAttempts: 3
|
||||||
|
|
||||||
# Custom domains configuration (optional)
|
# Custom domains configuration (optional)
|
||||||
customDomains:
|
customDomains:
|
||||||
enabled: false
|
enabled: false
|
||||||
|
|||||||
@@ -242,6 +242,15 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
|
|||||||
Temperature: b.getEnvFloatPtr("AGENT_TRACKER_ENRICHMENT_TEMPERATURE"),
|
Temperature: b.getEnvFloatPtr("AGENT_TRACKER_ENRICHMENT_TEMPERATURE"),
|
||||||
MaxTokens: new(b.getEnvIntOrDefault("AGENT_TRACKER_ENRICHMENT_MAX_TOKENS", 4096)),
|
MaxTokens: new(b.getEnvIntOrDefault("AGENT_TRACKER_ENRICHMENT_MAX_TOKENS", 4096)),
|
||||||
},
|
},
|
||||||
|
CommonThirdPartyEnrichment: probodconfig.LLMAgentConfig{
|
||||||
|
Provider: b.getEnvOrDefault("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_PROVIDER", ""),
|
||||||
|
ModelName: b.getEnvOrDefault("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MODEL_NAME", ""),
|
||||||
|
// Agent B browses pages and emits a moderate structured
|
||||||
|
// output; the budget must leave headroom for reasoning
|
||||||
|
// models whose reasoning tokens count against max_tokens.
|
||||||
|
Temperature: b.getEnvFloatPtr("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_TEMPERATURE"),
|
||||||
|
MaxTokens: new(b.getEnvIntOrDefault("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MAX_TOKENS", 8192)),
|
||||||
|
},
|
||||||
Tools: probodconfig.AgentToolsConfig{
|
Tools: probodconfig.AgentToolsConfig{
|
||||||
FirecrawlAPIKey: b.getEnv("FIRECRAWL_API_KEY"),
|
FirecrawlAPIKey: b.getEnv("FIRECRAWL_API_KEY"),
|
||||||
},
|
},
|
||||||
@@ -292,6 +301,15 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
|
|||||||
AgentTimeout: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_AGENT_TIMEOUT", 45),
|
AgentTimeout: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_AGENT_TIMEOUT", 45),
|
||||||
AgentMaxTurns: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS", 10),
|
AgentMaxTurns: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS", 10),
|
||||||
},
|
},
|
||||||
|
CommonThirdPartyEnrichmentWorker: probodconfig.CommonThirdPartyEnrichmentWorkerConfig{
|
||||||
|
Interval: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_INTERVAL", 10),
|
||||||
|
MaxConcurrency: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_MAX_CONCURRENCY", 1),
|
||||||
|
StaleAfter: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_STALE_AFTER", 900),
|
||||||
|
AgentTimeout: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_AGENT_TIMEOUT", 90),
|
||||||
|
AgentMaxTurns: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_AGENT_MAX_TURNS", 12),
|
||||||
|
ConfidenceThreshold: b.getEnvFloatOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_CONFIDENCE_THRESHOLD", 0.7),
|
||||||
|
MaxAttempts: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_MAX_ATTEMPTS", 3),
|
||||||
|
},
|
||||||
Branding: b.getEnvBoolOrDefault("BRANDING", true),
|
Branding: b.getEnvBoolOrDefault("BRANDING", true),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -245,6 +245,13 @@ func TestBuilder_Build_Defaults(t *testing.T) {
|
|||||||
assert.Equal(t, 600, cfg.Probod.CommonPatternEnrichmentWorker.StaleAfter)
|
assert.Equal(t, 600, cfg.Probod.CommonPatternEnrichmentWorker.StaleAfter)
|
||||||
assert.Equal(t, 45, cfg.Probod.CommonPatternEnrichmentWorker.AgentTimeout)
|
assert.Equal(t, 45, cfg.Probod.CommonPatternEnrichmentWorker.AgentTimeout)
|
||||||
assert.Equal(t, 10, cfg.Probod.CommonPatternEnrichmentWorker.AgentMaxTurns)
|
assert.Equal(t, 10, cfg.Probod.CommonPatternEnrichmentWorker.AgentMaxTurns)
|
||||||
|
assert.Equal(t, 10, cfg.Probod.CommonThirdPartyEnrichmentWorker.Interval)
|
||||||
|
assert.Equal(t, 1, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxConcurrency)
|
||||||
|
assert.Equal(t, 900, cfg.Probod.CommonThirdPartyEnrichmentWorker.StaleAfter)
|
||||||
|
assert.Equal(t, 90, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentTimeout)
|
||||||
|
assert.Equal(t, 12, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentMaxTurns)
|
||||||
|
assert.Equal(t, 0.7, cfg.Probod.CommonThirdPartyEnrichmentWorker.ConfidenceThreshold)
|
||||||
|
assert.Equal(t, 3, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxAttempts)
|
||||||
assert.Equal(t, 10, cfg.Probod.ThirdPartyVetting.Interval)
|
assert.Equal(t, 10, cfg.Probod.ThirdPartyVetting.Interval)
|
||||||
assert.Equal(t, 1500, cfg.Probod.ThirdPartyVetting.StaleAfter)
|
assert.Equal(t, 1500, cfg.Probod.ThirdPartyVetting.StaleAfter)
|
||||||
assert.Equal(t, 1, cfg.Probod.ThirdPartyVetting.MaxConcurrency)
|
assert.Equal(t, 1, cfg.Probod.ThirdPartyVetting.MaxConcurrency)
|
||||||
@@ -369,6 +376,17 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
|
|||||||
env["COMMON_PATTERN_ENRICHMENT_STALE_AFTER"] = "900"
|
env["COMMON_PATTERN_ENRICHMENT_STALE_AFTER"] = "900"
|
||||||
env["COMMON_PATTERN_ENRICHMENT_AGENT_TIMEOUT"] = "50"
|
env["COMMON_PATTERN_ENRICHMENT_AGENT_TIMEOUT"] = "50"
|
||||||
env["COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS"] = "5"
|
env["COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS"] = "5"
|
||||||
|
// Common third party enrichment agent + worker tuning override
|
||||||
|
env["AGENT_COMMON_THIRD_PARTY_ENRICHMENT_PROVIDER"] = "openai"
|
||||||
|
env["AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MODEL_NAME"] = "gpt-4o"
|
||||||
|
env["AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MAX_TOKENS"] = "16384"
|
||||||
|
env["COMMON_THIRD_PARTY_ENRICHMENT_INTERVAL"] = "25"
|
||||||
|
env["COMMON_THIRD_PARTY_ENRICHMENT_MAX_CONCURRENCY"] = "2"
|
||||||
|
env["COMMON_THIRD_PARTY_ENRICHMENT_STALE_AFTER"] = "1200"
|
||||||
|
env["COMMON_THIRD_PARTY_ENRICHMENT_AGENT_TIMEOUT"] = "120"
|
||||||
|
env["COMMON_THIRD_PARTY_ENRICHMENT_AGENT_MAX_TURNS"] = "8"
|
||||||
|
env["COMMON_THIRD_PARTY_ENRICHMENT_CONFIDENCE_THRESHOLD"] = "0.85"
|
||||||
|
env["COMMON_THIRD_PARTY_ENRICHMENT_MAX_ATTEMPTS"] = "5"
|
||||||
env["THIRD_PARTY_VETTING_INTERVAL"] = "15"
|
env["THIRD_PARTY_VETTING_INTERVAL"] = "15"
|
||||||
env["THIRD_PARTY_VETTING_STALE_AFTER"] = "1800"
|
env["THIRD_PARTY_VETTING_STALE_AFTER"] = "1800"
|
||||||
env["THIRD_PARTY_VETTING_MAX_CONCURRENCY"] = "2"
|
env["THIRD_PARTY_VETTING_MAX_CONCURRENCY"] = "2"
|
||||||
@@ -490,6 +508,17 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
|
|||||||
assert.Equal(t, 900, cfg.Probod.CommonPatternEnrichmentWorker.StaleAfter)
|
assert.Equal(t, 900, cfg.Probod.CommonPatternEnrichmentWorker.StaleAfter)
|
||||||
assert.Equal(t, 50, cfg.Probod.CommonPatternEnrichmentWorker.AgentTimeout)
|
assert.Equal(t, 50, cfg.Probod.CommonPatternEnrichmentWorker.AgentTimeout)
|
||||||
assert.Equal(t, 5, cfg.Probod.CommonPatternEnrichmentWorker.AgentMaxTurns)
|
assert.Equal(t, 5, cfg.Probod.CommonPatternEnrichmentWorker.AgentMaxTurns)
|
||||||
|
assert.Equal(t, "openai", cfg.Probod.Agents.CommonThirdPartyEnrichment.Provider)
|
||||||
|
assert.Equal(t, "gpt-4o", cfg.Probod.Agents.CommonThirdPartyEnrichment.ModelName)
|
||||||
|
require.NotNil(t, cfg.Probod.Agents.CommonThirdPartyEnrichment.MaxTokens)
|
||||||
|
assert.Equal(t, 16384, *cfg.Probod.Agents.CommonThirdPartyEnrichment.MaxTokens)
|
||||||
|
assert.Equal(t, 25, cfg.Probod.CommonThirdPartyEnrichmentWorker.Interval)
|
||||||
|
assert.Equal(t, 2, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxConcurrency)
|
||||||
|
assert.Equal(t, 1200, cfg.Probod.CommonThirdPartyEnrichmentWorker.StaleAfter)
|
||||||
|
assert.Equal(t, 120, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentTimeout)
|
||||||
|
assert.Equal(t, 8, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentMaxTurns)
|
||||||
|
assert.Equal(t, 0.85, cfg.Probod.CommonThirdPartyEnrichmentWorker.ConfidenceThreshold)
|
||||||
|
assert.Equal(t, 5, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxAttempts)
|
||||||
assert.Equal(t, 15, cfg.Probod.ThirdPartyVetting.Interval)
|
assert.Equal(t, 15, cfg.Probod.ThirdPartyVetting.Interval)
|
||||||
assert.Equal(t, 1800, cfg.Probod.ThirdPartyVetting.StaleAfter)
|
assert.Equal(t, 1800, cfg.Probod.ThirdPartyVetting.StaleAfter)
|
||||||
assert.Equal(t, 2, cfg.Probod.ThirdPartyVetting.MaxConcurrency)
|
assert.Equal(t, 2, cfg.Probod.ThirdPartyVetting.MaxConcurrency)
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ package coredata
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"maps"
|
"maps"
|
||||||
@@ -49,6 +50,9 @@ type (
|
|||||||
SecurityPageURL *string `db:"security_page_url"`
|
SecurityPageURL *string `db:"security_page_url"`
|
||||||
TrustPageURL *string `db:"trust_page_url"`
|
TrustPageURL *string `db:"trust_page_url"`
|
||||||
LogoFileID *gid.GID `db:"logo_file_id"`
|
LogoFileID *gid.GID `db:"logo_file_id"`
|
||||||
|
EnrichmentRequestedAt *time.Time `db:"enrichment_requested_at"`
|
||||||
|
Enrichment json.RawMessage `db:"enrichment"`
|
||||||
|
EnrichmentAttempts int `db:"enrichment_attempts"`
|
||||||
CreatedAt time.Time `db:"created_at"`
|
CreatedAt time.Time `db:"created_at"`
|
||||||
UpdatedAt time.Time `db:"updated_at"`
|
UpdatedAt time.Time `db:"updated_at"`
|
||||||
}
|
}
|
||||||
@@ -124,6 +128,9 @@ SELECT
|
|||||||
security_page_url,
|
security_page_url,
|
||||||
trust_page_url,
|
trust_page_url,
|
||||||
logo_file_id,
|
logo_file_id,
|
||||||
|
enrichment_requested_at,
|
||||||
|
enrichment,
|
||||||
|
enrichment_attempts,
|
||||||
created_at,
|
created_at,
|
||||||
updated_at
|
updated_at
|
||||||
FROM
|
FROM
|
||||||
@@ -181,6 +188,9 @@ SELECT
|
|||||||
security_page_url,
|
security_page_url,
|
||||||
trust_page_url,
|
trust_page_url,
|
||||||
logo_file_id,
|
logo_file_id,
|
||||||
|
enrichment_requested_at,
|
||||||
|
enrichment,
|
||||||
|
enrichment_attempts,
|
||||||
created_at,
|
created_at,
|
||||||
updated_at
|
updated_at
|
||||||
FROM
|
FROM
|
||||||
@@ -238,6 +248,9 @@ SELECT
|
|||||||
security_page_url,
|
security_page_url,
|
||||||
trust_page_url,
|
trust_page_url,
|
||||||
logo_file_id,
|
logo_file_id,
|
||||||
|
enrichment_requested_at,
|
||||||
|
enrichment,
|
||||||
|
enrichment_attempts,
|
||||||
created_at,
|
created_at,
|
||||||
updated_at
|
updated_at
|
||||||
FROM
|
FROM
|
||||||
@@ -294,6 +307,9 @@ INSERT INTO common_third_parties (
|
|||||||
security_page_url,
|
security_page_url,
|
||||||
trust_page_url,
|
trust_page_url,
|
||||||
logo_file_id,
|
logo_file_id,
|
||||||
|
enrichment_requested_at,
|
||||||
|
enrichment,
|
||||||
|
enrichment_attempts,
|
||||||
created_at,
|
created_at,
|
||||||
updated_at
|
updated_at
|
||||||
) VALUES (
|
) VALUES (
|
||||||
@@ -316,6 +332,9 @@ INSERT INTO common_third_parties (
|
|||||||
@security_page_url,
|
@security_page_url,
|
||||||
@trust_page_url,
|
@trust_page_url,
|
||||||
@logo_file_id,
|
@logo_file_id,
|
||||||
|
@enrichment_requested_at,
|
||||||
|
@enrichment,
|
||||||
|
@enrichment_attempts,
|
||||||
@created_at,
|
@created_at,
|
||||||
@updated_at
|
@updated_at
|
||||||
)
|
)
|
||||||
@@ -341,6 +360,9 @@ INSERT INTO common_third_parties (
|
|||||||
"security_page_url": t.SecurityPageURL,
|
"security_page_url": t.SecurityPageURL,
|
||||||
"trust_page_url": t.TrustPageURL,
|
"trust_page_url": t.TrustPageURL,
|
||||||
"logo_file_id": t.LogoFileID,
|
"logo_file_id": t.LogoFileID,
|
||||||
|
"enrichment_requested_at": t.EnrichmentRequestedAt,
|
||||||
|
"enrichment": t.Enrichment,
|
||||||
|
"enrichment_attempts": t.EnrichmentAttempts,
|
||||||
"created_at": t.CreatedAt,
|
"created_at": t.CreatedAt,
|
||||||
"updated_at": t.UpdatedAt,
|
"updated_at": t.UpdatedAt,
|
||||||
}
|
}
|
||||||
@@ -381,6 +403,9 @@ INSERT INTO common_third_parties (
|
|||||||
security_page_url,
|
security_page_url,
|
||||||
trust_page_url,
|
trust_page_url,
|
||||||
logo_file_id,
|
logo_file_id,
|
||||||
|
enrichment_requested_at,
|
||||||
|
enrichment,
|
||||||
|
enrichment_attempts,
|
||||||
created_at,
|
created_at,
|
||||||
updated_at
|
updated_at
|
||||||
) VALUES (
|
) VALUES (
|
||||||
@@ -403,6 +428,9 @@ INSERT INTO common_third_parties (
|
|||||||
@security_page_url,
|
@security_page_url,
|
||||||
@trust_page_url,
|
@trust_page_url,
|
||||||
@logo_file_id,
|
@logo_file_id,
|
||||||
|
@enrichment_requested_at,
|
||||||
|
@enrichment,
|
||||||
|
@enrichment_attempts,
|
||||||
@created_at,
|
@created_at,
|
||||||
@updated_at
|
@updated_at
|
||||||
)
|
)
|
||||||
@@ -445,6 +473,9 @@ RETURNING
|
|||||||
security_page_url,
|
security_page_url,
|
||||||
trust_page_url,
|
trust_page_url,
|
||||||
logo_file_id,
|
logo_file_id,
|
||||||
|
enrichment_requested_at,
|
||||||
|
enrichment,
|
||||||
|
enrichment_attempts,
|
||||||
created_at,
|
created_at,
|
||||||
updated_at
|
updated_at
|
||||||
`
|
`
|
||||||
@@ -471,6 +502,9 @@ RETURNING
|
|||||||
"security_page_url": t.SecurityPageURL,
|
"security_page_url": t.SecurityPageURL,
|
||||||
"trust_page_url": t.TrustPageURL,
|
"trust_page_url": t.TrustPageURL,
|
||||||
"logo_file_id": t.LogoFileID,
|
"logo_file_id": t.LogoFileID,
|
||||||
|
"enrichment_requested_at": t.EnrichmentRequestedAt,
|
||||||
|
"enrichment": t.Enrichment,
|
||||||
|
"enrichment_attempts": t.EnrichmentAttempts,
|
||||||
"created_at": t.CreatedAt,
|
"created_at": t.CreatedAt,
|
||||||
"updated_at": t.UpdatedAt,
|
"updated_at": t.UpdatedAt,
|
||||||
}
|
}
|
||||||
@@ -534,6 +568,9 @@ SELECT
|
|||||||
security_page_url,
|
security_page_url,
|
||||||
trust_page_url,
|
trust_page_url,
|
||||||
logo_file_id,
|
logo_file_id,
|
||||||
|
enrichment_requested_at,
|
||||||
|
enrichment,
|
||||||
|
enrichment_attempts,
|
||||||
created_at,
|
created_at,
|
||||||
updated_at
|
updated_at
|
||||||
FROM
|
FROM
|
||||||
@@ -585,6 +622,9 @@ SELECT
|
|||||||
security_page_url,
|
security_page_url,
|
||||||
trust_page_url,
|
trust_page_url,
|
||||||
logo_file_id,
|
logo_file_id,
|
||||||
|
enrichment_requested_at,
|
||||||
|
enrichment,
|
||||||
|
enrichment_attempts,
|
||||||
created_at,
|
created_at,
|
||||||
updated_at
|
updated_at
|
||||||
FROM
|
FROM
|
||||||
@@ -690,6 +730,9 @@ SELECT
|
|||||||
security_page_url,
|
security_page_url,
|
||||||
trust_page_url,
|
trust_page_url,
|
||||||
logo_file_id,
|
logo_file_id,
|
||||||
|
enrichment_requested_at,
|
||||||
|
enrichment,
|
||||||
|
enrichment_attempts,
|
||||||
created_at,
|
created_at,
|
||||||
updated_at
|
updated_at
|
||||||
FROM
|
FROM
|
||||||
@@ -750,3 +793,213 @@ WHERE
|
|||||||
|
|
||||||
return count, nil
|
return count, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LoadNextForEnrichmentForUpdateSkipLocked claims the oldest row queued
|
||||||
|
// for enrichment. The global catalog is not tenant-scoped, so the claim
|
||||||
|
// is intentionally cross-tenant: the enrichment worker is a system
|
||||||
|
// worker that drains the queue regardless of tenant.
|
||||||
|
func (t *CommonThirdParty) LoadNextForEnrichmentForUpdateSkipLocked(
|
||||||
|
ctx context.Context,
|
||||||
|
tx pg.Tx,
|
||||||
|
) error {
|
||||||
|
q := `
|
||||||
|
SELECT
|
||||||
|
id,
|
||||||
|
name,
|
||||||
|
slug,
|
||||||
|
category,
|
||||||
|
headquarter_address,
|
||||||
|
legal_name,
|
||||||
|
website_url,
|
||||||
|
privacy_policy_url,
|
||||||
|
service_level_agreement_url,
|
||||||
|
service_software_agreement_url,
|
||||||
|
data_processing_agreement_url,
|
||||||
|
business_associate_agreement_url,
|
||||||
|
subprocessors_list_url,
|
||||||
|
certifications,
|
||||||
|
status_page_url,
|
||||||
|
terms_of_service_url,
|
||||||
|
security_page_url,
|
||||||
|
trust_page_url,
|
||||||
|
logo_file_id,
|
||||||
|
enrichment_requested_at,
|
||||||
|
enrichment,
|
||||||
|
enrichment_attempts,
|
||||||
|
created_at,
|
||||||
|
updated_at
|
||||||
|
FROM
|
||||||
|
common_third_parties
|
||||||
|
WHERE
|
||||||
|
enrichment_requested_at IS NOT NULL
|
||||||
|
ORDER BY
|
||||||
|
enrichment_requested_at ASC
|
||||||
|
FOR UPDATE SKIP LOCKED
|
||||||
|
LIMIT 1;
|
||||||
|
`
|
||||||
|
|
||||||
|
rows, err := tx.Query(ctx, q)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot query common third party for enrichment: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
row, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[CommonThirdParty])
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return ErrResourceNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("cannot collect common third party for enrichment: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
*t = row
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ClearEnrichmentRequestedAt removes the row from the enrichment queue
|
||||||
|
// and bumps the attempt counter. It bumps updated_at so the
|
||||||
|
// stale-recovery clock starts at claim time, keeping
|
||||||
|
// ResetStaleCommonThirdPartyEnrichments from re-arming a row that is
|
||||||
|
// still being processed. The attempt counter is incremented up front so
|
||||||
|
// a crash between claim and persist still counts against the retry
|
||||||
|
// budget.
|
||||||
|
func (t *CommonThirdParty) ClearEnrichmentRequestedAt(
|
||||||
|
ctx context.Context,
|
||||||
|
tx pg.Tx,
|
||||||
|
) error {
|
||||||
|
q := `
|
||||||
|
UPDATE common_third_parties
|
||||||
|
SET
|
||||||
|
enrichment_requested_at = NULL,
|
||||||
|
enrichment_attempts = enrichment_attempts + 1,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = @id
|
||||||
|
`
|
||||||
|
|
||||||
|
args := pgx.StrictNamedArgs{"id": t.ID}
|
||||||
|
|
||||||
|
_, err := tx.Exec(ctx, q, args)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot clear enrichment requested at: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.EnrichmentRequestedAt = nil
|
||||||
|
t.EnrichmentAttempts++
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateEnrichment persists the enrichment result: the resolved metadata
|
||||||
|
// fields plus the per-field enrichment provenance JSON. It is a targeted
|
||||||
|
// partial update that never touches id, slug, category, name, logo, the
|
||||||
|
// queue column, or the attempt counter (logo is owned by
|
||||||
|
// UpdateLogoFileID; the queue column and counter are managed by
|
||||||
|
// ClearEnrichmentRequestedAt). The caller decides which scalar fields to
|
||||||
|
// write versus leave untouched, then passes the merged receiver here.
|
||||||
|
func (t CommonThirdParty) UpdateEnrichment(
|
||||||
|
ctx context.Context,
|
||||||
|
conn pg.Tx,
|
||||||
|
) error {
|
||||||
|
q := `
|
||||||
|
UPDATE common_third_parties
|
||||||
|
SET
|
||||||
|
headquarter_address = @headquarter_address,
|
||||||
|
legal_name = @legal_name,
|
||||||
|
website_url = @website_url,
|
||||||
|
privacy_policy_url = @privacy_policy_url,
|
||||||
|
service_level_agreement_url = @service_level_agreement_url,
|
||||||
|
service_software_agreement_url = @service_software_agreement_url,
|
||||||
|
data_processing_agreement_url = @data_processing_agreement_url,
|
||||||
|
business_associate_agreement_url = @business_associate_agreement_url,
|
||||||
|
subprocessors_list_url = @subprocessors_list_url,
|
||||||
|
certifications = @certifications,
|
||||||
|
status_page_url = @status_page_url,
|
||||||
|
terms_of_service_url = @terms_of_service_url,
|
||||||
|
security_page_url = @security_page_url,
|
||||||
|
trust_page_url = @trust_page_url,
|
||||||
|
enrichment = @enrichment,
|
||||||
|
updated_at = @updated_at
|
||||||
|
WHERE
|
||||||
|
id = @id
|
||||||
|
`
|
||||||
|
|
||||||
|
args := pgx.StrictNamedArgs{
|
||||||
|
"id": t.ID,
|
||||||
|
"headquarter_address": t.HeadquarterAddress,
|
||||||
|
"legal_name": t.LegalName,
|
||||||
|
"website_url": t.WebsiteURL,
|
||||||
|
"privacy_policy_url": t.PrivacyPolicyURL,
|
||||||
|
"service_level_agreement_url": t.ServiceLevelAgreementURL,
|
||||||
|
"service_software_agreement_url": t.ServiceSoftwareAgreementURL,
|
||||||
|
"data_processing_agreement_url": t.DataProcessingAgreementURL,
|
||||||
|
"business_associate_agreement_url": t.BusinessAssociateAgreementURL,
|
||||||
|
"subprocessors_list_url": t.SubprocessorsListURL,
|
||||||
|
"certifications": t.Certifications,
|
||||||
|
"status_page_url": t.StatusPageURL,
|
||||||
|
"terms_of_service_url": t.TermsOfServiceURL,
|
||||||
|
"security_page_url": t.SecurityPageURL,
|
||||||
|
"trust_page_url": t.TrustPageURL,
|
||||||
|
"enrichment": t.Enrichment,
|
||||||
|
"updated_at": t.UpdatedAt,
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := conn.Exec(ctx, q, args)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot update common third party enrichment: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.RowsAffected() == 0 {
|
||||||
|
return ErrResourceNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResetStaleCommonThirdPartyEnrichments re-arms enrichment_requested_at
|
||||||
|
// on rows whose enrichment was claimed but never completed and have been
|
||||||
|
// idle longer than staleAfter, so a crashed or timed-out run is retried.
|
||||||
|
//
|
||||||
|
// A claimed row has enrichment_attempts > 0 (Claim increments it) and a
|
||||||
|
// completed row has a non-null enrichment payload (Process always writes
|
||||||
|
// it, even on a no-result run), so the sweep targets rows that were
|
||||||
|
// claimed but carry no enrichment yet. Curated rows that were never
|
||||||
|
// enqueued keep enrichment_attempts = 0 and are left untouched. The
|
||||||
|
// max-attempts ceiling stops permanently failing rows from looping
|
||||||
|
// forever.
|
||||||
|
//
|
||||||
|
// Like the claim query, this sweep is intentionally cross-tenant: the
|
||||||
|
// enrichment worker is a system worker that drains the queue regardless
|
||||||
|
// of tenant.
|
||||||
|
func ResetStaleCommonThirdPartyEnrichments(
|
||||||
|
ctx context.Context,
|
||||||
|
conn pg.Querier,
|
||||||
|
staleAfter time.Duration,
|
||||||
|
maxAttempts int,
|
||||||
|
) error {
|
||||||
|
q := `
|
||||||
|
UPDATE common_third_parties
|
||||||
|
SET
|
||||||
|
enrichment_requested_at = NOW(),
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE
|
||||||
|
enrichment_requested_at IS NULL
|
||||||
|
AND enrichment IS NULL
|
||||||
|
AND enrichment_attempts > 0
|
||||||
|
AND enrichment_attempts < @max_attempts
|
||||||
|
AND updated_at < @stale_before
|
||||||
|
`
|
||||||
|
|
||||||
|
args := pgx.StrictNamedArgs{
|
||||||
|
"max_attempts": maxAttempts,
|
||||||
|
"stale_before": time.Now().Add(-staleAfter),
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := conn.Exec(ctx, q, args)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot reset stale common third party enrichments: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
28
pkg/coredata/migrations/20260609T120000Z.sql
Normal file
28
pkg/coredata/migrations/20260609T120000Z.sql
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
-- Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
--
|
||||||
|
-- Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
-- purpose with or without fee is hereby granted, provided that the above
|
||||||
|
-- copyright notice and this permission notice appear in all copies.
|
||||||
|
--
|
||||||
|
-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
-- PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
ALTER TABLE
|
||||||
|
common_third_parties
|
||||||
|
ADD
|
||||||
|
COLUMN enrichment_requested_at TIMESTAMP WITH TIME ZONE,
|
||||||
|
ADD
|
||||||
|
COLUMN enrichment JSONB,
|
||||||
|
ADD
|
||||||
|
COLUMN enrichment_attempts INTEGER NOT NULL DEFAULT 0;
|
||||||
|
|
||||||
|
-- Partial index backs the enrichment worker's claim query, which polls
|
||||||
|
-- only rows currently queued for enrichment.
|
||||||
|
CREATE INDEX common_third_parties_enrichment_requested_at_idx
|
||||||
|
ON common_third_parties (enrichment_requested_at)
|
||||||
|
WHERE enrichment_requested_at IS NOT NULL;
|
||||||
@@ -43,8 +43,9 @@ type (
|
|||||||
ThirdPartyVettingWorkerConfig = probodconfig.ThirdPartyVettingWorkerConfig
|
ThirdPartyVettingWorkerConfig = probodconfig.ThirdPartyVettingWorkerConfig
|
||||||
AgentsConfig = probodconfig.AgentsConfig
|
AgentsConfig = probodconfig.AgentsConfig
|
||||||
|
|
||||||
TrackerMappingWorkerConfig = probodconfig.TrackerMappingWorkerConfig
|
TrackerMappingWorkerConfig = probodconfig.TrackerMappingWorkerConfig
|
||||||
CommonPatternEnrichmentWorkerConfig = probodconfig.CommonPatternEnrichmentWorkerConfig
|
CommonPatternEnrichmentWorkerConfig = probodconfig.CommonPatternEnrichmentWorkerConfig
|
||||||
|
CommonThirdPartyEnrichmentWorkerConfig = probodconfig.CommonThirdPartyEnrichmentWorkerConfig
|
||||||
|
|
||||||
MailerConfig = probodconfig.MailerConfig
|
MailerConfig = probodconfig.MailerConfig
|
||||||
SMTPConfig = probodconfig.SMTPConfig
|
SMTPConfig = probodconfig.SMTPConfig
|
||||||
|
|||||||
73
pkg/probod/common_third_party_enrichment.go
Normal file
73
pkg/probod/common_third_party_enrichment.go
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package probod
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
"go.gearno.de/kit/log"
|
||||||
|
"go.opentelemetry.io/otel/trace"
|
||||||
|
"go.probo.inc/probo/pkg/filemanager"
|
||||||
|
"go.probo.inc/probo/pkg/thirdparty"
|
||||||
|
)
|
||||||
|
|
||||||
|
// buildCommonThirdPartyEnrichmentConfig wires the common-third-party
|
||||||
|
// enrichment worker config: the LLM client for its two agents plus the
|
||||||
|
// worker tuning, browser endpoint, and logo-storage dependencies. It is
|
||||||
|
// opt-in: a deployment that does not set
|
||||||
|
// `llm.common-third-party-enrichment.provider` gets a zero config (nil
|
||||||
|
// LLM client), so the worker runs as a no-op and the caller skips
|
||||||
|
// registration.
|
||||||
|
func (impl *Implm) buildCommonThirdPartyEnrichmentConfig(
|
||||||
|
l *log.Logger,
|
||||||
|
tp trace.TracerProvider,
|
||||||
|
r prometheus.Registerer,
|
||||||
|
fileManager *filemanager.Service,
|
||||||
|
) (thirdparty.EnrichmentConfig, error) {
|
||||||
|
if impl.cfg.Agents.CommonThirdPartyEnrichment.Provider == "" {
|
||||||
|
return thirdparty.EnrichmentConfig{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
agentCfg, llmClient, err := impl.resolveAgentClient(
|
||||||
|
"common-third-party-enrichment",
|
||||||
|
impl.cfg.Agents.CommonThirdPartyEnrichment,
|
||||||
|
l,
|
||||||
|
tp,
|
||||||
|
r,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return thirdparty.EnrichmentConfig{}, fmt.Errorf("cannot resolve common third party enrichment agent client: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
workerCfg := impl.cfg.CommonThirdPartyEnrichmentWorker
|
||||||
|
|
||||||
|
return thirdparty.EnrichmentConfig{
|
||||||
|
LLMClient: llmClient,
|
||||||
|
Model: agentCfg.ModelName,
|
||||||
|
MaxTokens: agentCfg.MaxTokens,
|
||||||
|
Temperature: agentCfg.Temperature,
|
||||||
|
FirecrawlAPIKey: impl.cfg.Agents.Tools.FirecrawlAPIKey,
|
||||||
|
ChromeAddr: impl.cfg.ChromeDPAddr,
|
||||||
|
AgentTimeout: time.Duration(workerCfg.AgentTimeout) * time.Second,
|
||||||
|
MaxTurns: workerCfg.AgentMaxTurns,
|
||||||
|
ConfidenceThreshold: workerCfg.ConfidenceThreshold,
|
||||||
|
StaleAfter: time.Duration(workerCfg.StaleAfter) * time.Second,
|
||||||
|
MaxAttempts: workerCfg.MaxAttempts,
|
||||||
|
FileManager: fileManager,
|
||||||
|
Bucket: impl.cfg.AWS.Bucket,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
@@ -181,6 +181,15 @@ func New() *Implm {
|
|||||||
StaleAfter: 1500,
|
StaleAfter: 1500,
|
||||||
MaxConcurrency: 1,
|
MaxConcurrency: 1,
|
||||||
},
|
},
|
||||||
|
CommonThirdPartyEnrichmentWorker: CommonThirdPartyEnrichmentWorkerConfig{
|
||||||
|
Interval: 10,
|
||||||
|
MaxConcurrency: 1,
|
||||||
|
StaleAfter: 900,
|
||||||
|
AgentTimeout: 90,
|
||||||
|
AgentMaxTurns: 12,
|
||||||
|
ConfidenceThreshold: 0.7,
|
||||||
|
MaxAttempts: 3,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -328,6 +337,11 @@ func (impl *Implm) Run(
|
|||||||
|
|
||||||
fileManagerService := filemanager.NewService(pgClient, baseURL, s3Client)
|
fileManagerService := filemanager.NewService(pgClient, baseURL, s3Client)
|
||||||
|
|
||||||
|
commonThirdPartyEnrichmentCfg, err := impl.buildCommonThirdPartyEnrichmentConfig(l, tp, r, fileManagerService)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
samlCert *x509.Certificate
|
samlCert *x509.Certificate
|
||||||
samlKey *rsa.PrivateKey
|
samlKey *rsa.PrivateKey
|
||||||
@@ -819,6 +833,34 @@ func (impl *Implm) Run(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The common-third-party enrichment worker fills catalog metadata
|
||||||
|
// (URLs, address, certifications, logo) via two agents plus a
|
||||||
|
// deterministic logo step. It needs an LLM client, so it is only
|
||||||
|
// started when its agent config is present.
|
||||||
|
stopCommonThirdPartyEnrichmentWorker := func() {}
|
||||||
|
|
||||||
|
if commonThirdPartyEnrichmentCfg.LLMClient != nil {
|
||||||
|
commonThirdPartyEnrichmentWorker := thirdparty.NewCommonThirdPartyEnrichmentWorker(
|
||||||
|
pgClient,
|
||||||
|
l.Named("common-third-party-enrichment-worker"),
|
||||||
|
commonThirdPartyEnrichmentCfg,
|
||||||
|
worker.WithInterval(time.Duration(impl.cfg.CommonThirdPartyEnrichmentWorker.Interval)*time.Second),
|
||||||
|
worker.WithMaxConcurrency(impl.cfg.CommonThirdPartyEnrichmentWorker.MaxConcurrency),
|
||||||
|
)
|
||||||
|
|
||||||
|
var commonThirdPartyEnrichmentWorkerCtx context.Context
|
||||||
|
|
||||||
|
commonThirdPartyEnrichmentWorkerCtx, stopCommonThirdPartyEnrichmentWorker = context.WithCancel(context.Background())
|
||||||
|
|
||||||
|
wg.Go(
|
||||||
|
func() {
|
||||||
|
if err := commonThirdPartyEnrichmentWorker.Run(commonThirdPartyEnrichmentWorkerCtx); err != nil {
|
||||||
|
cancel(fmt.Errorf("common third party enrichment worker crashed: %w", err))
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
mailingListWorker := mailman.NewMailingListWorker(mailmanService, pgClient, l.Named("mailing-list-worker"))
|
mailingListWorker := mailman.NewMailingListWorker(mailmanService, pgClient, l.Named("mailing-list-worker"))
|
||||||
mailingListWorkerCtx, stopMailingListWorker := context.WithCancel(context.Background())
|
mailingListWorkerCtx, stopMailingListWorker := context.WithCancel(context.Background())
|
||||||
|
|
||||||
@@ -910,6 +952,7 @@ func (impl *Implm) Run(
|
|||||||
stopTrackerPolicyWorker()
|
stopTrackerPolicyWorker()
|
||||||
stopTrackerMappingWorker()
|
stopTrackerMappingWorker()
|
||||||
stopCommonPatternEnrichmentWorker()
|
stopCommonPatternEnrichmentWorker()
|
||||||
|
stopCommonThirdPartyEnrichmentWorker()
|
||||||
stopMailingListWorker()
|
stopMailingListWorker()
|
||||||
stopVettingWorker()
|
stopVettingWorker()
|
||||||
stopEvidenceDescriptionWorker()
|
stopEvidenceDescriptionWorker()
|
||||||
|
|||||||
@@ -62,8 +62,9 @@ type (
|
|||||||
EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"`
|
EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"`
|
||||||
ThirdPartyVetting ThirdPartyVettingWorkerConfig `json:"third-party-vetting-worker"`
|
ThirdPartyVetting ThirdPartyVettingWorkerConfig `json:"third-party-vetting-worker"`
|
||||||
|
|
||||||
TrackerMappingWorker TrackerMappingWorkerConfig `json:"tracker-mapping-worker"`
|
TrackerMappingWorker TrackerMappingWorkerConfig `json:"tracker-mapping-worker"`
|
||||||
CommonPatternEnrichmentWorker CommonPatternEnrichmentWorkerConfig `json:"common-pattern-enrichment-worker"`
|
CommonPatternEnrichmentWorker CommonPatternEnrichmentWorkerConfig `json:"common-pattern-enrichment-worker"`
|
||||||
|
CommonThirdPartyEnrichmentWorker CommonThirdPartyEnrichmentWorkerConfig `json:"common-third-party-enrichment-worker"`
|
||||||
|
|
||||||
ChromeDPAddr string `json:"chrome-dp-addr"`
|
ChromeDPAddr string `json:"chrome-dp-addr"`
|
||||||
CustomDomains CustomDomainsConfig `json:"custom-domains"`
|
CustomDomains CustomDomainsConfig `json:"custom-domains"`
|
||||||
|
|||||||
@@ -76,6 +76,22 @@ type (
|
|||||||
AgentMaxTurns int `json:"agent-max-turns"`
|
AgentMaxTurns int `json:"agent-max-turns"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CommonThirdPartyEnrichmentWorkerConfig holds worker-side tuning for
|
||||||
|
// the common-third-party enrichment background worker. LLM parameters
|
||||||
|
// for its agents live under AgentsConfig.CommonThirdPartyEnrichment.
|
||||||
|
// ConfidenceThreshold is the floor a resolved value must clear before
|
||||||
|
// it is written to its column; MaxAttempts caps stale-recovery
|
||||||
|
// retries.
|
||||||
|
CommonThirdPartyEnrichmentWorkerConfig struct {
|
||||||
|
Interval int `json:"interval"` // seconds between polls
|
||||||
|
MaxConcurrency int `json:"max-concurrency"`
|
||||||
|
StaleAfter int `json:"stale-after"` // seconds before a claim is recycled
|
||||||
|
AgentTimeout int `json:"agent-timeout"` // seconds, single agent run
|
||||||
|
AgentMaxTurns int `json:"agent-max-turns"`
|
||||||
|
ConfidenceThreshold float64 `json:"confidence-threshold"`
|
||||||
|
MaxAttempts int `json:"max-attempts"`
|
||||||
|
}
|
||||||
|
|
||||||
// AgentToolsConfig holds API keys and settings for external tools
|
// AgentToolsConfig holds API keys and settings for external tools
|
||||||
// that agents can use (web search, scraping, etc.).
|
// that agents can use (web search, scraping, etc.).
|
||||||
AgentToolsConfig struct {
|
AgentToolsConfig struct {
|
||||||
@@ -86,15 +102,16 @@ type (
|
|||||||
// settings. Default is used as a fallback when an agent-specific field
|
// settings. Default is used as a fallback when an agent-specific field
|
||||||
// is zero-valued.
|
// is zero-valued.
|
||||||
AgentsConfig struct {
|
AgentsConfig struct {
|
||||||
Providers map[string]LLMProviderConfig `json:"providers"`
|
Providers map[string]LLMProviderConfig `json:"providers"`
|
||||||
Default LLMAgentConfig `json:"defaults"`
|
Default LLMAgentConfig `json:"defaults"`
|
||||||
Probo LLMAgentConfig `json:"probo"`
|
Probo LLMAgentConfig `json:"probo"`
|
||||||
EvidenceDescriber LLMAgentConfig `json:"evidence-describer"`
|
EvidenceDescriber LLMAgentConfig `json:"evidence-describer"`
|
||||||
ThirdPartyVetter LLMAgentConfig `json:"third-party-vetter"`
|
ThirdPartyVetter LLMAgentConfig `json:"third-party-vetter"`
|
||||||
ThirdPartyDisambiguation LLMAgentConfig `json:"third-party-disambiguation"`
|
ThirdPartyDisambiguation LLMAgentConfig `json:"third-party-disambiguation"`
|
||||||
TrackerMapping LLMAgentConfig `json:"tracker-mapping"`
|
TrackerMapping LLMAgentConfig `json:"tracker-mapping"`
|
||||||
TrackerEnrichment LLMAgentConfig `json:"tracker-enrichment"`
|
TrackerEnrichment LLMAgentConfig `json:"tracker-enrichment"`
|
||||||
Tools AgentToolsConfig `json:"tools"`
|
CommonThirdPartyEnrichment LLMAgentConfig `json:"common-third-party-enrichment"`
|
||||||
|
Tools AgentToolsConfig `json:"tools"`
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
94
pkg/thirdparty/common_third_party_company_profile_agent.go
vendored
Normal file
94
pkg/thirdparty/common_third_party_company_profile_agent.go
vendored
Normal file
@@ -0,0 +1,94 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package thirdparty
|
||||||
|
|
||||||
|
import (
|
||||||
|
_ "embed"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"go.gearno.de/kit/log"
|
||||||
|
"go.probo.inc/probo/pkg/agent"
|
||||||
|
"go.probo.inc/probo/pkg/agent/tools/search"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:embed prompts/common_third_party_company_profile.txt.tmpl
|
||||||
|
var companyProfilePrompt string
|
||||||
|
|
||||||
|
// CompanyProfileResult is the structured output of the company-profile
|
||||||
|
// agent (Agent A): the vendor's identity facts. website_url is the key
|
||||||
|
// signal the compliance-docs agent and the logo step depend on, so it is
|
||||||
|
// resolved here first.
|
||||||
|
type CompanyProfileResult struct {
|
||||||
|
LegalName EnrichedField `json:"legal_name" jsonschema:"The vendor's full legal company name including the entity suffix (e.g. 'Acme Technologies, Inc.')."`
|
||||||
|
HeadquarterAddress EnrichedField `json:"headquarter_address" jsonschema:"The vendor's headquarters postal address (street, city, region, country)."`
|
||||||
|
WebsiteURL EnrichedField `json:"website_url" jsonschema:"The vendor's canonical primary marketing website URL (https scheme, no tracking query parameters, no trailing path)."`
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildCompanyProfileAgent(
|
||||||
|
cfg EnrichmentConfig,
|
||||||
|
logger *log.Logger,
|
||||||
|
) *agent.Agent {
|
||||||
|
var tools []agent.Tool
|
||||||
|
|
||||||
|
if cfg.FirecrawlAPIKey != "" {
|
||||||
|
tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlAPIKey))
|
||||||
|
}
|
||||||
|
|
||||||
|
outputType, err := agent.NewOutputType[CompanyProfileResult]("common_third_party_company_profile")
|
||||||
|
if err != nil {
|
||||||
|
panic(fmt.Sprintf("thirdparty: cannot build company profile output type: %s", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
opts := []agent.Option{
|
||||||
|
agent.WithInstructions(companyProfilePrompt),
|
||||||
|
agent.WithModel(cfg.Model),
|
||||||
|
agent.WithOutputType(outputType),
|
||||||
|
agent.WithMaxTurns(resolveEnrichmentMaxTurns(cfg.MaxTurns)),
|
||||||
|
agent.WithMaxTokens(resolveEnrichmentMaxTokens(cfg.MaxTokens)),
|
||||||
|
agent.WithLogger(logger),
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(tools) > 0 {
|
||||||
|
opts = append(opts, agent.WithTools(tools...))
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.Temperature != nil {
|
||||||
|
opts = append(opts, agent.WithTemperature(*cfg.Temperature))
|
||||||
|
}
|
||||||
|
|
||||||
|
return agent.New("common-third-party-company-profile", cfg.LLMClient, opts...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildCompanyProfilePrompt renders the per-row input for Agent A. Any
|
||||||
|
// values already on the row are passed as hints so the agent confirms or
|
||||||
|
// corrects them rather than starting cold.
|
||||||
|
func buildCompanyProfilePrompt(party coredata.CommonThirdParty) string {
|
||||||
|
var b strings.Builder
|
||||||
|
|
||||||
|
fmt.Fprintf(&b, "Research this company and return its profile.\n\n")
|
||||||
|
fmt.Fprintf(&b, "<name> %s </name>\n", party.Name)
|
||||||
|
|
||||||
|
if party.WebsiteURL != nil && strings.TrimSpace(*party.WebsiteURL) != "" {
|
||||||
|
fmt.Fprintf(&b, "<known_website> %s </known_website>\n", *party.WebsiteURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
if party.LegalName != nil && strings.TrimSpace(*party.LegalName) != "" {
|
||||||
|
fmt.Fprintf(&b, "<known_legal_name> %s </known_legal_name>\n", *party.LegalName)
|
||||||
|
}
|
||||||
|
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
105
pkg/thirdparty/common_third_party_compliance_docs_agent.go
vendored
Normal file
105
pkg/thirdparty/common_third_party_compliance_docs_agent.go
vendored
Normal file
@@ -0,0 +1,105 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package thirdparty
|
||||||
|
|
||||||
|
import (
|
||||||
|
_ "embed"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"go.gearno.de/kit/log"
|
||||||
|
"go.probo.inc/probo/pkg/agent"
|
||||||
|
"go.probo.inc/probo/pkg/agent/tools/search"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:embed prompts/common_third_party_compliance_docs.txt.tmpl
|
||||||
|
var complianceDocsPrompt string
|
||||||
|
|
||||||
|
// ComplianceDocsResult is the structured output of the compliance-docs
|
||||||
|
// agent (Agent B): the legal-document URLs, trust/security/status pages,
|
||||||
|
// and certifications. These all live in the same source ecosystem (the
|
||||||
|
// vendor footer and trust portal), so one agent resolves them together.
|
||||||
|
type ComplianceDocsResult struct {
|
||||||
|
PrivacyPolicyURL EnrichedField `json:"privacy_policy_url" jsonschema:"URL of the vendor's privacy policy."`
|
||||||
|
TermsOfServiceURL EnrichedField `json:"terms_of_service_url" jsonschema:"URL of the vendor's terms of service / terms of use."`
|
||||||
|
ServiceLevelAgreementURL EnrichedField `json:"service_level_agreement_url" jsonschema:"URL of the vendor's public service level agreement (SLA). Often gated behind sales; return empty when not public."`
|
||||||
|
ServiceSoftwareAgreementURL EnrichedField `json:"service_software_agreement_url" jsonschema:"URL of the vendor's master software/subscription agreement (MSA). Often gated or identical to the terms of service; return empty when not public."`
|
||||||
|
DataProcessingAgreementURL EnrichedField `json:"data_processing_agreement_url" jsonschema:"URL of the vendor's data processing agreement (DPA). Often a PDF; return empty when only available on request."`
|
||||||
|
BusinessAssociateAgreementURL EnrichedField `json:"business_associate_agreement_url" jsonschema:"URL of the vendor's HIPAA business associate agreement (BAA). Almost always gated behind sales; return empty when not public."`
|
||||||
|
SubprocessorsListURL EnrichedField `json:"subprocessors_list_url" jsonschema:"URL of the vendor's sub-processors list page."`
|
||||||
|
StatusPageURL EnrichedField `json:"status_page_url" jsonschema:"URL of the vendor's uptime/status page (e.g. status.vendor.com)."`
|
||||||
|
SecurityPageURL EnrichedField `json:"security_page_url" jsonschema:"URL of the vendor's security page or security overview."`
|
||||||
|
TrustPageURL EnrichedField `json:"trust_page_url" jsonschema:"URL of the vendor's trust center / trust portal (e.g. Vanta, SafeBase, Drata hosted)."`
|
||||||
|
Certifications CertificationsField `json:"certifications" jsonschema:"Certifications and compliance frameworks the vendor publicly claims."`
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildComplianceDocsAgent builds Agent B. extraTools carries the browser
|
||||||
|
// read-only toolset when a headless Chrome endpoint is configured; it is
|
||||||
|
// empty otherwise, in which case the agent relies on web_search alone.
|
||||||
|
func buildComplianceDocsAgent(
|
||||||
|
cfg EnrichmentConfig,
|
||||||
|
logger *log.Logger,
|
||||||
|
extraTools []agent.Tool,
|
||||||
|
) *agent.Agent {
|
||||||
|
tools := append([]agent.Tool{}, extraTools...)
|
||||||
|
|
||||||
|
if cfg.FirecrawlAPIKey != "" {
|
||||||
|
tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlAPIKey))
|
||||||
|
}
|
||||||
|
|
||||||
|
outputType, err := agent.NewOutputType[ComplianceDocsResult]("common_third_party_compliance_docs")
|
||||||
|
if err != nil {
|
||||||
|
panic(fmt.Sprintf("thirdparty: cannot build compliance docs output type: %s", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
opts := []agent.Option{
|
||||||
|
agent.WithInstructions(complianceDocsPrompt),
|
||||||
|
agent.WithModel(cfg.Model),
|
||||||
|
agent.WithOutputType(outputType),
|
||||||
|
agent.WithMaxTurns(resolveEnrichmentMaxTurns(cfg.MaxTurns)),
|
||||||
|
agent.WithMaxTokens(resolveEnrichmentMaxTokens(cfg.MaxTokens)),
|
||||||
|
agent.WithLogger(logger),
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(tools) > 0 {
|
||||||
|
opts = append(opts, agent.WithTools(tools...))
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.Temperature != nil {
|
||||||
|
opts = append(opts, agent.WithTemperature(*cfg.Temperature))
|
||||||
|
}
|
||||||
|
|
||||||
|
return agent.New("common-third-party-compliance-docs", cfg.LLMClient, opts...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildComplianceDocsPrompt renders the per-row input for Agent B,
|
||||||
|
// seeding it with the vendor name and the website/legal name resolved by
|
||||||
|
// Agent A so it can scope its search to the vendor's own domain.
|
||||||
|
func buildComplianceDocsPrompt(name, websiteURL, legalName string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
|
||||||
|
fmt.Fprintf(&b, "Find the compliance documents and trust pages for this vendor.\n\n")
|
||||||
|
fmt.Fprintf(&b, "<name> %s </name>\n", name)
|
||||||
|
|
||||||
|
if w := strings.TrimSpace(websiteURL); w != "" {
|
||||||
|
fmt.Fprintf(&b, "<website> %s </website>\n", w)
|
||||||
|
}
|
||||||
|
|
||||||
|
if l := strings.TrimSpace(legalName); l != "" {
|
||||||
|
fmt.Fprintf(&b, "<legal_name> %s </legal_name>\n", l)
|
||||||
|
}
|
||||||
|
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
345
pkg/thirdparty/common_third_party_enrichment.go
vendored
Normal file
345
pkg/thirdparty/common_third_party_enrichment.go
vendored
Normal file
@@ -0,0 +1,345 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package thirdparty
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Provenance sources and per-field statuses recorded in the enrichment
|
||||||
|
// JSON. The "source" distinguishes values written by this enricher from
|
||||||
|
// values owned externally (curated seed data or a human edit), which the
|
||||||
|
// enricher must never overwrite.
|
||||||
|
const (
|
||||||
|
enrichmentSourceEnrichment = "enrichment"
|
||||||
|
enrichmentSourceExternal = "external"
|
||||||
|
|
||||||
|
enrichmentFieldStatusFound = "found"
|
||||||
|
enrichmentFieldStatusNotFound = "not_found"
|
||||||
|
enrichmentFieldStatusLowConfidence = "low_confidence"
|
||||||
|
enrichmentFieldStatusExternal = "exists_external"
|
||||||
|
|
||||||
|
// Run-level status recorded at the top of the enrichment payload.
|
||||||
|
enrichmentStatusDone = "done"
|
||||||
|
enrichmentStatusPartial = "partial"
|
||||||
|
enrichmentStatusFailed = "failed"
|
||||||
|
)
|
||||||
|
|
||||||
|
type (
|
||||||
|
// EnrichedField is the per-field unit the enrichment agents return:
|
||||||
|
// the resolved value plus a self-assessed confidence and the source
|
||||||
|
// URL where it was verified. The worker applies a confidence
|
||||||
|
// threshold before writing the value to its column.
|
||||||
|
EnrichedField struct {
|
||||||
|
Value string `json:"value" jsonschema:"The resolved value, or an empty string when not confidently found. Never guess."`
|
||||||
|
Confidence float64 `json:"confidence" jsonschema:"Confidence from 0.0 to 1.0 that the value is correct. Use 0 when the value was not found."`
|
||||||
|
SourceURL string `json:"source_url" jsonschema:"The URL where this value was verified, or an empty string."`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CertificationsField is the list-valued counterpart of EnrichedField
|
||||||
|
// used for the certifications array.
|
||||||
|
CertificationsField struct {
|
||||||
|
Values []string `json:"values" jsonschema:"Certification or compliance framework names the vendor publicly claims (e.g. 'SOC 2 Type II', 'ISO 27001', 'HIPAA'). Empty when none are found."`
|
||||||
|
Confidence float64 `json:"confidence" jsonschema:"Confidence from 0.0 to 1.0 in the certifications list. Use 0 when none are found."`
|
||||||
|
SourceURL string `json:"source_url" jsonschema:"The URL where the certifications were found (trust or security page), or an empty string."`
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnrichmentFieldMeta is the per-field provenance recorded in the
|
||||||
|
// common_third_parties.enrichment JSON column.
|
||||||
|
EnrichmentFieldMeta struct {
|
||||||
|
Confidence float64 `json:"confidence"`
|
||||||
|
SourceURL string `json:"source_url,omitempty"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
Source string `json:"source"`
|
||||||
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnrichmentMetadata is the full payload stored in the enrichment
|
||||||
|
// JSON column: run-level bookkeeping plus per-field provenance keyed
|
||||||
|
// by the column name.
|
||||||
|
EnrichmentMetadata struct {
|
||||||
|
Model string `json:"model,omitempty"`
|
||||||
|
AttemptedAt time.Time `json:"attempted_at"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
Fields map[string]EnrichmentFieldMeta `json:"fields"`
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
// scalarField describes one *string column the enricher can fill,
|
||||||
|
// pairing the agent's resolved value with accessors into the receiver.
|
||||||
|
type scalarField struct {
|
||||||
|
name string
|
||||||
|
get func(*coredata.CommonThirdParty) *string
|
||||||
|
set func(*coredata.CommonThirdParty, *string)
|
||||||
|
result EnrichedField
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseEnrichmentFields extracts the prior per-field provenance from a
|
||||||
|
// row's enrichment payload. A missing or malformed payload yields an
|
||||||
|
// empty map, which the merge treats as "no field is enrichment-owned".
|
||||||
|
func parseEnrichmentFields(raw json.RawMessage) map[string]EnrichmentFieldMeta {
|
||||||
|
if len(raw) == 0 {
|
||||||
|
return map[string]EnrichmentFieldMeta{}
|
||||||
|
}
|
||||||
|
|
||||||
|
var meta EnrichmentMetadata
|
||||||
|
if err := json.Unmarshal(raw, &meta); err != nil {
|
||||||
|
return map[string]EnrichmentFieldMeta{}
|
||||||
|
}
|
||||||
|
|
||||||
|
if meta.Fields == nil {
|
||||||
|
return map[string]EnrichmentFieldMeta{}
|
||||||
|
}
|
||||||
|
|
||||||
|
return meta.Fields
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyScalarField merges one resolved scalar field into party, honoring
|
||||||
|
// the confidence threshold and prior provenance, and records the
|
||||||
|
// resulting per-field metadata. A value already present that this
|
||||||
|
// enricher did not write (curated seed data or a human edit) is left
|
||||||
|
// untouched. The column is written only when the value clears the
|
||||||
|
// threshold; below it the column keeps its prior value and the metadata
|
||||||
|
// records why nothing was written.
|
||||||
|
func applyScalarField(
|
||||||
|
party *coredata.CommonThirdParty,
|
||||||
|
meta map[string]EnrichmentFieldMeta,
|
||||||
|
prior map[string]EnrichmentFieldMeta,
|
||||||
|
field scalarField,
|
||||||
|
threshold float64,
|
||||||
|
now time.Time,
|
||||||
|
) {
|
||||||
|
value := strings.TrimSpace(field.result.Value)
|
||||||
|
sourceURL := strings.TrimSpace(field.result.SourceURL)
|
||||||
|
|
||||||
|
existing := field.get(party)
|
||||||
|
hasExisting := existing != nil && strings.TrimSpace(*existing) != ""
|
||||||
|
|
||||||
|
priorMeta, hadPrior := prior[field.name]
|
||||||
|
enrichmentOwned := hadPrior && priorMeta.Source == enrichmentSourceEnrichment
|
||||||
|
|
||||||
|
if hasExisting && !enrichmentOwned {
|
||||||
|
meta[field.name] = EnrichmentFieldMeta{
|
||||||
|
Status: enrichmentFieldStatusExternal,
|
||||||
|
Source: enrichmentSourceExternal,
|
||||||
|
UpdatedAt: now,
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if value != "" && field.result.Confidence >= threshold {
|
||||||
|
v := value
|
||||||
|
field.set(party, &v)
|
||||||
|
meta[field.name] = EnrichmentFieldMeta{
|
||||||
|
Confidence: field.result.Confidence,
|
||||||
|
SourceURL: sourceURL,
|
||||||
|
Status: enrichmentFieldStatusFound,
|
||||||
|
Source: enrichmentSourceEnrichment,
|
||||||
|
UpdatedAt: now,
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
status := enrichmentFieldStatusNotFound
|
||||||
|
if value != "" {
|
||||||
|
status = enrichmentFieldStatusLowConfidence
|
||||||
|
}
|
||||||
|
|
||||||
|
meta[field.name] = EnrichmentFieldMeta{
|
||||||
|
Confidence: field.result.Confidence,
|
||||||
|
SourceURL: sourceURL,
|
||||||
|
Status: status,
|
||||||
|
Source: enrichmentSourceEnrichment,
|
||||||
|
UpdatedAt: now,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyCertifications is the list-valued counterpart of
|
||||||
|
// applyScalarField for the certifications column.
|
||||||
|
func applyCertifications(
|
||||||
|
party *coredata.CommonThirdParty,
|
||||||
|
meta map[string]EnrichmentFieldMeta,
|
||||||
|
prior map[string]EnrichmentFieldMeta,
|
||||||
|
result CertificationsField,
|
||||||
|
threshold float64,
|
||||||
|
now time.Time,
|
||||||
|
) {
|
||||||
|
const name = "certifications"
|
||||||
|
|
||||||
|
values := normalizeCertifications(result.Values)
|
||||||
|
sourceURL := strings.TrimSpace(result.SourceURL)
|
||||||
|
|
||||||
|
hasExisting := len(party.Certifications) > 0
|
||||||
|
|
||||||
|
priorMeta, hadPrior := prior[name]
|
||||||
|
enrichmentOwned := hadPrior && priorMeta.Source == enrichmentSourceEnrichment
|
||||||
|
|
||||||
|
if hasExisting && !enrichmentOwned {
|
||||||
|
meta[name] = EnrichmentFieldMeta{
|
||||||
|
Status: enrichmentFieldStatusExternal,
|
||||||
|
Source: enrichmentSourceExternal,
|
||||||
|
UpdatedAt: now,
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(values) > 0 && result.Confidence >= threshold {
|
||||||
|
party.Certifications = values
|
||||||
|
meta[name] = EnrichmentFieldMeta{
|
||||||
|
Confidence: result.Confidence,
|
||||||
|
SourceURL: sourceURL,
|
||||||
|
Status: enrichmentFieldStatusFound,
|
||||||
|
Source: enrichmentSourceEnrichment,
|
||||||
|
UpdatedAt: now,
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
status := enrichmentFieldStatusNotFound
|
||||||
|
if len(values) > 0 {
|
||||||
|
status = enrichmentFieldStatusLowConfidence
|
||||||
|
}
|
||||||
|
|
||||||
|
meta[name] = EnrichmentFieldMeta{
|
||||||
|
Confidence: result.Confidence,
|
||||||
|
SourceURL: sourceURL,
|
||||||
|
Status: status,
|
||||||
|
Source: enrichmentSourceEnrichment,
|
||||||
|
UpdatedAt: now,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeCertifications trims, drops blanks, and de-duplicates the
|
||||||
|
// certification names returned by the agent, preserving order.
|
||||||
|
func normalizeCertifications(values []string) []string {
|
||||||
|
seen := make(map[string]struct{}, len(values))
|
||||||
|
out := make([]string, 0, len(values))
|
||||||
|
|
||||||
|
for _, v := range values {
|
||||||
|
v = strings.TrimSpace(v)
|
||||||
|
if v == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
key := strings.ToLower(v)
|
||||||
|
if _, ok := seen[key]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
out = append(out, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// scalarFields returns the descriptor list pairing each *string column
|
||||||
|
// with the resolved value from the two agents. website_url comes from
|
||||||
|
// the company-profile agent; the document and page URLs come from the
|
||||||
|
// compliance-docs agent.
|
||||||
|
func scalarFields(
|
||||||
|
company CompanyProfileResult,
|
||||||
|
compliance ComplianceDocsResult,
|
||||||
|
) []scalarField {
|
||||||
|
return []scalarField{
|
||||||
|
{
|
||||||
|
name: "legal_name",
|
||||||
|
get: func(p *coredata.CommonThirdParty) *string { return p.LegalName },
|
||||||
|
set: func(p *coredata.CommonThirdParty, v *string) { p.LegalName = v },
|
||||||
|
result: company.LegalName,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "headquarter_address",
|
||||||
|
get: func(p *coredata.CommonThirdParty) *string { return p.HeadquarterAddress },
|
||||||
|
set: func(p *coredata.CommonThirdParty, v *string) { p.HeadquarterAddress = v },
|
||||||
|
result: company.HeadquarterAddress,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "website_url",
|
||||||
|
get: func(p *coredata.CommonThirdParty) *string { return p.WebsiteURL },
|
||||||
|
set: func(p *coredata.CommonThirdParty, v *string) { p.WebsiteURL = v },
|
||||||
|
result: company.WebsiteURL,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "privacy_policy_url",
|
||||||
|
get: func(p *coredata.CommonThirdParty) *string { return p.PrivacyPolicyURL },
|
||||||
|
set: func(p *coredata.CommonThirdParty, v *string) { p.PrivacyPolicyURL = v },
|
||||||
|
result: compliance.PrivacyPolicyURL,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "terms_of_service_url",
|
||||||
|
get: func(p *coredata.CommonThirdParty) *string { return p.TermsOfServiceURL },
|
||||||
|
set: func(p *coredata.CommonThirdParty, v *string) { p.TermsOfServiceURL = v },
|
||||||
|
result: compliance.TermsOfServiceURL,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "service_level_agreement_url",
|
||||||
|
get: func(p *coredata.CommonThirdParty) *string { return p.ServiceLevelAgreementURL },
|
||||||
|
set: func(p *coredata.CommonThirdParty, v *string) { p.ServiceLevelAgreementURL = v },
|
||||||
|
result: compliance.ServiceLevelAgreementURL,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "service_software_agreement_url",
|
||||||
|
get: func(p *coredata.CommonThirdParty) *string { return p.ServiceSoftwareAgreementURL },
|
||||||
|
set: func(p *coredata.CommonThirdParty, v *string) { p.ServiceSoftwareAgreementURL = v },
|
||||||
|
result: compliance.ServiceSoftwareAgreementURL,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "data_processing_agreement_url",
|
||||||
|
get: func(p *coredata.CommonThirdParty) *string { return p.DataProcessingAgreementURL },
|
||||||
|
set: func(p *coredata.CommonThirdParty, v *string) { p.DataProcessingAgreementURL = v },
|
||||||
|
result: compliance.DataProcessingAgreementURL,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "business_associate_agreement_url",
|
||||||
|
get: func(p *coredata.CommonThirdParty) *string { return p.BusinessAssociateAgreementURL },
|
||||||
|
set: func(p *coredata.CommonThirdParty, v *string) { p.BusinessAssociateAgreementURL = v },
|
||||||
|
result: compliance.BusinessAssociateAgreementURL,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "subprocessors_list_url",
|
||||||
|
get: func(p *coredata.CommonThirdParty) *string { return p.SubprocessorsListURL },
|
||||||
|
set: func(p *coredata.CommonThirdParty, v *string) { p.SubprocessorsListURL = v },
|
||||||
|
result: compliance.SubprocessorsListURL,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "status_page_url",
|
||||||
|
get: func(p *coredata.CommonThirdParty) *string { return p.StatusPageURL },
|
||||||
|
set: func(p *coredata.CommonThirdParty, v *string) { p.StatusPageURL = v },
|
||||||
|
result: compliance.StatusPageURL,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "security_page_url",
|
||||||
|
get: func(p *coredata.CommonThirdParty) *string { return p.SecurityPageURL },
|
||||||
|
set: func(p *coredata.CommonThirdParty, v *string) { p.SecurityPageURL = v },
|
||||||
|
result: compliance.SecurityPageURL,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "trust_page_url",
|
||||||
|
get: func(p *coredata.CommonThirdParty) *string { return p.TrustPageURL },
|
||||||
|
set: func(p *coredata.CommonThirdParty, v *string) { p.TrustPageURL = v },
|
||||||
|
result: compliance.TrustPageURL,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
456
pkg/thirdparty/common_third_party_enrichment_worker.go
vendored
Normal file
456
pkg/thirdparty/common_third_party_enrichment_worker.go
vendored
Normal file
@@ -0,0 +1,456 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package thirdparty
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.gearno.de/kit/httpclient"
|
||||||
|
"go.gearno.de/kit/log"
|
||||||
|
"go.gearno.de/kit/pg"
|
||||||
|
"go.gearno.de/kit/worker"
|
||||||
|
"go.probo.inc/probo/pkg/agent"
|
||||||
|
"go.probo.inc/probo/pkg/agent/tools/browser"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
"go.probo.inc/probo/pkg/filemanager"
|
||||||
|
"go.probo.inc/probo/pkg/gid"
|
||||||
|
"go.probo.inc/probo/pkg/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// defaultEnrichmentAgentTimeout caps a single enrichment agent run.
|
||||||
|
// It is generous because Agent B browses several pages on top of the
|
||||||
|
// LLM round-trips.
|
||||||
|
defaultEnrichmentAgentTimeout = 90 * time.Second
|
||||||
|
|
||||||
|
// defaultEnrichmentMaxTurns bounds an agent's reasoning loop (LLM
|
||||||
|
// call plus tool round-trips). Agent B may navigate the site footer
|
||||||
|
// and trust portal across several turns before synthesizing.
|
||||||
|
defaultEnrichmentMaxTurns = 12
|
||||||
|
|
||||||
|
// defaultEnrichmentMaxTokens caps agent output. The structured
|
||||||
|
// output is moderate; the budget leaves headroom for reasoning
|
||||||
|
// models whose reasoning tokens count against max_tokens.
|
||||||
|
defaultEnrichmentMaxTokens = 8192
|
||||||
|
|
||||||
|
// defaultEnrichmentConfidenceThreshold is the floor a resolved value
|
||||||
|
// must clear before it is written to its column. Values below it are
|
||||||
|
// recorded in the enrichment metadata but not promoted.
|
||||||
|
defaultEnrichmentConfidenceThreshold = 0.7
|
||||||
|
|
||||||
|
// defaultEnrichmentStaleAfter is the idle window after which a
|
||||||
|
// claimed-but-unfinished enrichment is re-armed.
|
||||||
|
defaultEnrichmentStaleAfter = 15 * time.Minute
|
||||||
|
|
||||||
|
// defaultEnrichmentMaxAttempts caps how many times a row is retried
|
||||||
|
// before stale recovery leaves it alone, so a permanently failing
|
||||||
|
// row does not loop forever.
|
||||||
|
defaultEnrichmentMaxAttempts = 3
|
||||||
|
|
||||||
|
enrichmentLogoUserAgent = "Probo-Enricher/1.0"
|
||||||
|
)
|
||||||
|
|
||||||
|
// EnrichmentConfig configures the common-third-party enrichment worker
|
||||||
|
// and the two agents it runs. The worker no-ops when LLMClient is nil;
|
||||||
|
// callers gate registration on config presence. Browser tools for Agent
|
||||||
|
// B are enabled only when ChromeAddr is set; otherwise it relies on
|
||||||
|
// web_search alone. Logo storage is enabled only when FileManager and
|
||||||
|
// Bucket are both set.
|
||||||
|
type EnrichmentConfig struct {
|
||||||
|
LLMClient *llm.Client
|
||||||
|
Model string
|
||||||
|
MaxTokens *int
|
||||||
|
Temperature *float64
|
||||||
|
FirecrawlAPIKey string
|
||||||
|
ChromeAddr string
|
||||||
|
AgentTimeout time.Duration
|
||||||
|
MaxTurns int
|
||||||
|
ConfidenceThreshold float64
|
||||||
|
StaleAfter time.Duration
|
||||||
|
MaxAttempts int
|
||||||
|
|
||||||
|
FileManager *filemanager.Service
|
||||||
|
Bucket string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c EnrichmentConfig) withDefaults() EnrichmentConfig {
|
||||||
|
if c.AgentTimeout <= 0 {
|
||||||
|
c.AgentTimeout = defaultEnrichmentAgentTimeout
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.MaxTurns < 1 {
|
||||||
|
c.MaxTurns = defaultEnrichmentMaxTurns
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.ConfidenceThreshold <= 0 {
|
||||||
|
c.ConfidenceThreshold = defaultEnrichmentConfidenceThreshold
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.StaleAfter <= 0 {
|
||||||
|
c.StaleAfter = defaultEnrichmentStaleAfter
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.MaxAttempts < 1 {
|
||||||
|
c.MaxAttempts = defaultEnrichmentMaxAttempts
|
||||||
|
}
|
||||||
|
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveEnrichmentMaxTurns(configured int) int {
|
||||||
|
if configured > 0 {
|
||||||
|
return configured
|
||||||
|
}
|
||||||
|
|
||||||
|
return defaultEnrichmentMaxTurns
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveEnrichmentMaxTokens(configured *int) int {
|
||||||
|
if configured != nil && *configured > 0 {
|
||||||
|
return *configured
|
||||||
|
}
|
||||||
|
|
||||||
|
return defaultEnrichmentMaxTokens
|
||||||
|
}
|
||||||
|
|
||||||
|
type enrichmentHandler struct {
|
||||||
|
pg *pg.Client
|
||||||
|
logger *log.Logger
|
||||||
|
cfg EnrichmentConfig
|
||||||
|
companyAgent *agent.Agent
|
||||||
|
httpClient *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCommonThirdPartyEnrichmentWorker builds the worker that enriches
|
||||||
|
// global common_third_parties rows. It is a system worker: the catalog
|
||||||
|
// is not tenant-scoped, so a single enrichment benefits all tenants. The
|
||||||
|
// worker no-ops when no LLM client is configured.
|
||||||
|
func NewCommonThirdPartyEnrichmentWorker(
|
||||||
|
pgClient *pg.Client,
|
||||||
|
logger *log.Logger,
|
||||||
|
cfg EnrichmentConfig,
|
||||||
|
opts ...worker.Option,
|
||||||
|
) *worker.Worker[coredata.CommonThirdParty] {
|
||||||
|
cfg = cfg.withDefaults()
|
||||||
|
|
||||||
|
h := &enrichmentHandler{
|
||||||
|
pg: pgClient,
|
||||||
|
logger: logger,
|
||||||
|
cfg: cfg,
|
||||||
|
httpClient: newEnrichmentHTTPClient(),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Agent A has no browser, so it is built once and reused. Agent B is
|
||||||
|
// built per Process because it needs a per-run browser bound to the
|
||||||
|
// process context.
|
||||||
|
if cfg.LLMClient != nil {
|
||||||
|
h.companyAgent = buildCompanyProfileAgent(cfg, logger)
|
||||||
|
}
|
||||||
|
|
||||||
|
return worker.New(
|
||||||
|
"common-third-party-enrichment-worker",
|
||||||
|
h,
|
||||||
|
logger,
|
||||||
|
opts...,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *enrichmentHandler) Claim(ctx context.Context) (coredata.CommonThirdParty, error) {
|
||||||
|
var party coredata.CommonThirdParty
|
||||||
|
|
||||||
|
if err := h.pg.WithTx(
|
||||||
|
ctx,
|
||||||
|
func(ctx context.Context, tx pg.Tx) error {
|
||||||
|
if err := party.LoadNextForEnrichmentForUpdateSkipLocked(ctx, tx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return party.ClearEnrichmentRequestedAt(ctx, tx)
|
||||||
|
},
|
||||||
|
); err != nil {
|
||||||
|
if errors.Is(err, coredata.ErrResourceNotFound) {
|
||||||
|
return coredata.CommonThirdParty{}, worker.ErrNoTask
|
||||||
|
}
|
||||||
|
|
||||||
|
return coredata.CommonThirdParty{}, fmt.Errorf("cannot claim common third party enrichment task: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return party, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process runs the enrichment pipeline for one catalog row: Agent A
|
||||||
|
// (company profile) first, then Agent B (compliance docs) and the
|
||||||
|
// deterministic logo step, all outside any transaction. The merged
|
||||||
|
// result and per-field provenance are persisted in a single final
|
||||||
|
// transaction. Process always writes an enrichment payload, even on a
|
||||||
|
// no-result run, so stale recovery does not re-queue the row.
|
||||||
|
func (h *enrichmentHandler) Process(ctx context.Context, party coredata.CommonThirdParty) error {
|
||||||
|
if h.companyAgent == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
prior := parseEnrichmentFields(party.Enrichment)
|
||||||
|
|
||||||
|
var (
|
||||||
|
runErrors []string
|
||||||
|
anySuccess bool
|
||||||
|
)
|
||||||
|
|
||||||
|
// Agent A first: it resolves website_url, which Agent B and the logo
|
||||||
|
// step depend on.
|
||||||
|
company, err := h.runCompanyProfile(ctx, party)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.WarnCtx(ctx, "company profile agent failed", log.Error(err), log.String("common_third_party_id", party.ID.String()))
|
||||||
|
runErrors = append(runErrors, "company_profile: "+err.Error())
|
||||||
|
} else {
|
||||||
|
anySuccess = true
|
||||||
|
}
|
||||||
|
|
||||||
|
website := effectiveWebsiteURL(party, company, h.cfg.ConfidenceThreshold)
|
||||||
|
legalName := effectiveLegalName(party, company, h.cfg.ConfidenceThreshold)
|
||||||
|
|
||||||
|
// Agent B: compliance documents and trust pages.
|
||||||
|
compliance, err := h.runComplianceDocs(ctx, party.Name, website, legalName)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.WarnCtx(ctx, "compliance docs agent failed", log.Error(err), log.String("common_third_party_id", party.ID.String()))
|
||||||
|
runErrors = append(runErrors, "compliance_docs: "+err.Error())
|
||||||
|
} else {
|
||||||
|
anySuccess = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deterministic logo step (no LLM). Uploads to S3 outside the final
|
||||||
|
// transaction; the File row is inserted below.
|
||||||
|
logoFile := h.prepareLogo(ctx, party, website)
|
||||||
|
|
||||||
|
meta := make(map[string]EnrichmentFieldMeta)
|
||||||
|
|
||||||
|
for _, field := range scalarFields(company, compliance) {
|
||||||
|
applyScalarField(&party, meta, prior, field, h.cfg.ConfidenceThreshold, now)
|
||||||
|
}
|
||||||
|
|
||||||
|
applyCertifications(&party, meta, prior, compliance.Certifications, h.cfg.ConfidenceThreshold, now)
|
||||||
|
|
||||||
|
status := enrichmentStatusDone
|
||||||
|
switch {
|
||||||
|
case !anySuccess:
|
||||||
|
status = enrichmentStatusFailed
|
||||||
|
case len(runErrors) > 0:
|
||||||
|
status = enrichmentStatusPartial
|
||||||
|
}
|
||||||
|
|
||||||
|
payload := EnrichmentMetadata{
|
||||||
|
Model: h.cfg.Model,
|
||||||
|
AttemptedAt: now,
|
||||||
|
Status: status,
|
||||||
|
Error: strings.Join(runErrors, "; "),
|
||||||
|
Fields: meta,
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, err := json.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot marshal enrichment metadata: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
party.Enrichment = raw
|
||||||
|
party.UpdatedAt = now
|
||||||
|
|
||||||
|
return h.pg.WithTx(
|
||||||
|
ctx,
|
||||||
|
func(ctx context.Context, tx pg.Tx) error {
|
||||||
|
if logoFile != nil {
|
||||||
|
if err := logoFile.Insert(ctx, tx, coredata.NewScope(gid.NilTenant)); err != nil {
|
||||||
|
return fmt.Errorf("cannot insert common third party logo file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
party.LogoFileID = &logoFile.ID
|
||||||
|
|
||||||
|
if err := party.UpdateLogoFileID(ctx, tx); err != nil {
|
||||||
|
return fmt.Errorf("cannot update common third party logo: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := party.UpdateEnrichment(ctx, tx); err != nil {
|
||||||
|
return fmt.Errorf("cannot persist common third party enrichment: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
h.logger.InfoCtx(
|
||||||
|
ctx,
|
||||||
|
"enriched common third party",
|
||||||
|
log.String("common_third_party_id", party.ID.String()),
|
||||||
|
log.String("name", party.Name),
|
||||||
|
log.String("status", status),
|
||||||
|
log.Bool("logo_stored", logoFile != nil),
|
||||||
|
)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecoverStale re-arms enrichment for rows whose run was claimed but
|
||||||
|
// never finished. Claim clears enrichment_requested_at up front, so a
|
||||||
|
// crash between phases would otherwise strand the row.
|
||||||
|
func (h *enrichmentHandler) RecoverStale(ctx context.Context) error {
|
||||||
|
return h.pg.WithConn(
|
||||||
|
ctx,
|
||||||
|
func(ctx context.Context, conn pg.Querier) error {
|
||||||
|
if err := coredata.ResetStaleCommonThirdPartyEnrichments(ctx, conn, h.cfg.StaleAfter, h.cfg.MaxAttempts); err != nil {
|
||||||
|
return fmt.Errorf("cannot reset stale common third party enrichments: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *enrichmentHandler) runCompanyProfile(
|
||||||
|
ctx context.Context,
|
||||||
|
party coredata.CommonThirdParty,
|
||||||
|
) (CompanyProfileResult, error) {
|
||||||
|
prompt := buildCompanyProfilePrompt(party)
|
||||||
|
|
||||||
|
agentCtx, cancel := context.WithTimeout(ctx, h.cfg.AgentTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
result, err := agent.RunTyped[CompanyProfileResult](
|
||||||
|
agentCtx,
|
||||||
|
h.companyAgent,
|
||||||
|
[]llm.Message{
|
||||||
|
{
|
||||||
|
Role: llm.RoleUser,
|
||||||
|
Parts: []llm.Part{llm.TextPart{Text: prompt}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return CompanyProfileResult{}, fmt.Errorf("company profile agent run failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.Output, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// runComplianceDocs builds Agent B with a per-run browser when a Chrome
|
||||||
|
// endpoint is configured, then runs it. The browser is closed when the
|
||||||
|
// run returns. The browser is intentionally not pinned to the vendor
|
||||||
|
// domain so the agent can follow links to hosted trust portals (Vanta,
|
||||||
|
// SafeBase, etc.); SSRF protection still blocks non-public hosts.
|
||||||
|
func (h *enrichmentHandler) runComplianceDocs(
|
||||||
|
ctx context.Context,
|
||||||
|
name string,
|
||||||
|
website string,
|
||||||
|
legalName string,
|
||||||
|
) (ComplianceDocsResult, error) {
|
||||||
|
var browserTools []agent.Tool
|
||||||
|
|
||||||
|
if h.cfg.ChromeAddr != "" {
|
||||||
|
webBrowser := browser.NewBrowser(ctx, h.cfg.ChromeAddr)
|
||||||
|
defer webBrowser.Close()
|
||||||
|
|
||||||
|
browserTools = browser.NewReadOnlyToolset(webBrowser).Tools()
|
||||||
|
}
|
||||||
|
|
||||||
|
complianceAgent := buildComplianceDocsAgent(h.cfg, h.logger, browserTools)
|
||||||
|
|
||||||
|
prompt := buildComplianceDocsPrompt(name, website, legalName)
|
||||||
|
|
||||||
|
agentCtx, cancel := context.WithTimeout(ctx, h.cfg.AgentTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
result, err := agent.RunTyped[ComplianceDocsResult](
|
||||||
|
agentCtx,
|
||||||
|
complianceAgent,
|
||||||
|
[]llm.Message{
|
||||||
|
{
|
||||||
|
Role: llm.RoleUser,
|
||||||
|
Parts: []llm.Part{llm.TextPart{Text: prompt}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return ComplianceDocsResult{}, fmt.Errorf("compliance docs agent run failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.Output, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// effectiveWebsiteURL is the website passed to Agent B and the logo step.
|
||||||
|
// A curated value already on the row wins (seed data and human edits are
|
||||||
|
// trusted); otherwise Agent A's value is used when it clears the
|
||||||
|
// confidence threshold.
|
||||||
|
func effectiveWebsiteURL(
|
||||||
|
party coredata.CommonThirdParty,
|
||||||
|
company CompanyProfileResult,
|
||||||
|
threshold float64,
|
||||||
|
) string {
|
||||||
|
if party.WebsiteURL != nil {
|
||||||
|
if v := strings.TrimSpace(*party.WebsiteURL); v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if v := strings.TrimSpace(company.WebsiteURL.Value); v != "" && company.WebsiteURL.Confidence >= threshold {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// effectiveLegalName is the legal name hint passed to Agent B, resolved
|
||||||
|
// the same way as effectiveWebsiteURL.
|
||||||
|
func effectiveLegalName(
|
||||||
|
party coredata.CommonThirdParty,
|
||||||
|
company CompanyProfileResult,
|
||||||
|
threshold float64,
|
||||||
|
) string {
|
||||||
|
if party.LegalName != nil {
|
||||||
|
if v := strings.TrimSpace(*party.LegalName); v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if v := strings.TrimSpace(company.LegalName.Value); v != "" && company.LegalName.Confidence >= threshold {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
type userAgentRoundTripper struct {
|
||||||
|
next http.RoundTripper
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *userAgentRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||||
|
r2 := r.Clone(r.Context())
|
||||||
|
r2.Header.Set("User-Agent", enrichmentLogoUserAgent)
|
||||||
|
|
||||||
|
return t.next.RoundTrip(r2)
|
||||||
|
}
|
||||||
|
|
||||||
|
// newEnrichmentHTTPClient builds the SSRF-protected client used by the
|
||||||
|
// deterministic logo step.
|
||||||
|
func newEnrichmentHTTPClient() *http.Client {
|
||||||
|
client := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
|
||||||
|
client.Timeout = 20 * time.Second
|
||||||
|
client.Transport = &userAgentRoundTripper{next: client.Transport}
|
||||||
|
|
||||||
|
return client
|
||||||
|
}
|
||||||
207
pkg/thirdparty/common_third_party_logo.go
vendored
Normal file
207
pkg/thirdparty/common_third_party_logo.go
vendored
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package thirdparty
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.gearno.de/crypto/uuid"
|
||||||
|
"go.gearno.de/kit/log"
|
||||||
|
"go.probo.inc/probo/pkg/coredata"
|
||||||
|
"go.probo.inc/probo/pkg/gid"
|
||||||
|
"go.probo.inc/probo/pkg/webinspect"
|
||||||
|
)
|
||||||
|
|
||||||
|
// maxLogoSize caps a downloaded logo image. Logos are small; the cap
|
||||||
|
// guards against an oversized or malicious response.
|
||||||
|
const maxLogoSize = 5 << 20 // 5 MiB
|
||||||
|
|
||||||
|
// prepareLogo discovers and uploads a vendor logo to S3, returning a
|
||||||
|
// fully-populated (but not yet inserted) File record for the caller to
|
||||||
|
// persist in its transaction. It is deterministic and best-effort: any
|
||||||
|
// failure logs and returns (nil, nil) so a missing logo never fails the
|
||||||
|
// enrichment run.
|
||||||
|
//
|
||||||
|
// It no-ops when the row already has a logo, when logo storage is not
|
||||||
|
// configured, or when no website is known. The S3 upload happens here,
|
||||||
|
// outside any transaction; the caller inserts the File row and links it
|
||||||
|
// via UpdateLogoFileID.
|
||||||
|
func (h *enrichmentHandler) prepareLogo(
|
||||||
|
ctx context.Context,
|
||||||
|
party coredata.CommonThirdParty,
|
||||||
|
websiteURL string,
|
||||||
|
) *coredata.File {
|
||||||
|
if party.LogoFileID != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if h.cfg.FileManager == nil || h.cfg.Bucket == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
website := strings.TrimSpace(websiteURL)
|
||||||
|
if website == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
data, contentType, err := fetchCommonThirdPartyLogo(ctx, h.httpClient, website)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.InfoCtx(
|
||||||
|
ctx,
|
||||||
|
"could not fetch common third party logo",
|
||||||
|
log.String("common_third_party_id", party.ID.String()),
|
||||||
|
log.Error(err),
|
||||||
|
)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
objectKey, err := uuid.NewV7()
|
||||||
|
if err != nil {
|
||||||
|
h.logger.WarnCtx(ctx, "cannot generate logo object key", log.Error(err))
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
fileRecord := &coredata.File{
|
||||||
|
ID: gid.New(gid.NilTenant, coredata.FileEntityType),
|
||||||
|
OrganizationID: gid.Nil,
|
||||||
|
BucketName: h.cfg.Bucket,
|
||||||
|
MimeType: contentType,
|
||||||
|
FileName: party.Name + "-logo" + webinspect.ExtensionForMIME(contentType),
|
||||||
|
FileKey: objectKey.String(),
|
||||||
|
FileSize: int64(len(data)),
|
||||||
|
Visibility: coredata.FileVisibilityPublic,
|
||||||
|
CreatedAt: now,
|
||||||
|
UpdatedAt: now,
|
||||||
|
}
|
||||||
|
|
||||||
|
size, err := h.cfg.FileManager.PutFile(
|
||||||
|
ctx,
|
||||||
|
fileRecord,
|
||||||
|
bytes.NewReader(data),
|
||||||
|
map[string]string{
|
||||||
|
"type": "common-third-party-logo",
|
||||||
|
"common-third-party-id": party.ID.String(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.WarnCtx(
|
||||||
|
ctx,
|
||||||
|
"cannot upload common third party logo",
|
||||||
|
log.String("common_third_party_id", party.ID.String()),
|
||||||
|
log.Error(err),
|
||||||
|
)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
fileRecord.FileSize = size
|
||||||
|
|
||||||
|
return fileRecord
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchCommonThirdPartyLogo finds the best logo for a website and
|
||||||
|
// downloads it. It first parses the page's <head> for icon links
|
||||||
|
// (webinspect), then falls back to well-known icon paths on the same
|
||||||
|
// host. The supplied client must enforce SSRF protection.
|
||||||
|
func fetchCommonThirdPartyLogo(
|
||||||
|
ctx context.Context,
|
||||||
|
client *http.Client,
|
||||||
|
websiteURL string,
|
||||||
|
) (data []byte, contentType string, err error) {
|
||||||
|
candidates := make([]string, 0, 3)
|
||||||
|
|
||||||
|
pageInfo, parseErr := webinspect.Parse(ctx, client, websiteURL)
|
||||||
|
if parseErr == nil {
|
||||||
|
if logoURL, logoErr := webinspect.FindLogoURL(pageInfo); logoErr == nil {
|
||||||
|
candidates = append(candidates, logoURL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if parsed, parseURLErr := url.Parse(websiteURL); parseURLErr == nil && parsed.Host != "" {
|
||||||
|
base := url.URL{Scheme: parsed.Scheme, Host: parsed.Host}
|
||||||
|
if base.Scheme == "" {
|
||||||
|
base.Scheme = "https"
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates = append(
|
||||||
|
candidates,
|
||||||
|
base.ResolveReference(&url.URL{Path: "/apple-touch-icon.png"}).String(),
|
||||||
|
base.ResolveReference(&url.URL{Path: "/favicon.ico"}).String(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, candidate := range candidates {
|
||||||
|
data, contentType, err = downloadImage(ctx, client, candidate)
|
||||||
|
if err == nil {
|
||||||
|
return data, contentType, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, "", fmt.Errorf("cannot fetch logo for %s", websiteURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
// downloadImage fetches a single candidate URL and returns its bytes when
|
||||||
|
// the response is a non-empty image within the size cap.
|
||||||
|
func downloadImage(
|
||||||
|
ctx context.Context,
|
||||||
|
client *http.Client,
|
||||||
|
rawURL string,
|
||||||
|
) ([]byte, string, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("cannot create logo request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("cannot fetch logo: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, "", fmt.Errorf("cannot fetch logo: status %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
contentType := resp.Header.Get("Content-Type")
|
||||||
|
if idx := strings.Index(contentType, ";"); idx != -1 {
|
||||||
|
contentType = contentType[:idx]
|
||||||
|
}
|
||||||
|
contentType = strings.TrimSpace(contentType)
|
||||||
|
|
||||||
|
if !strings.HasPrefix(contentType, "image/") {
|
||||||
|
return nil, "", fmt.Errorf("logo response is not an image: %q", contentType)
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := io.ReadAll(io.LimitReader(resp.Body, maxLogoSize))
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", fmt.Errorf("cannot read logo body: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(body) == 0 {
|
||||||
|
return nil, "", fmt.Errorf("logo response is empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
return body, contentType, nil
|
||||||
|
}
|
||||||
33
pkg/thirdparty/prompts/common_third_party_company_profile.txt.tmpl
vendored
Normal file
33
pkg/thirdparty/prompts/common_third_party_company_profile.txt.tmpl
vendored
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
You are a research agent that builds a factual company profile for a software
|
||||||
|
vendor or service provider. You are given the vendor's name (and sometimes a
|
||||||
|
known website). Return only verifiable identity facts.
|
||||||
|
|
||||||
|
Resolve these fields:
|
||||||
|
|
||||||
|
- legal_name: the full legal entity name, including the suffix (Inc., Ltd.,
|
||||||
|
GmbH, S.A.S., etc.). Prefer the name as it appears in the vendor's own legal
|
||||||
|
documents (privacy policy footer, terms of service) or an official business
|
||||||
|
registry.
|
||||||
|
- headquarter_address: the postal address of the company's headquarters.
|
||||||
|
- website_url: the canonical primary marketing website. Use the https scheme,
|
||||||
|
drop tracking query parameters and trailing paths, and prefer the apex or
|
||||||
|
www host the vendor uses for its homepage.
|
||||||
|
|
||||||
|
Method:
|
||||||
|
|
||||||
|
- Use the web_search tool when it is available to confirm facts. Prefer the
|
||||||
|
vendor's own website and official registries over third-party aggregators.
|
||||||
|
- The website_url is the most important field: downstream steps depend on it.
|
||||||
|
Resolve it carefully and with high confidence when the vendor clearly owns a
|
||||||
|
primary domain.
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- Never guess. If you cannot verify a field, return an empty string with a
|
||||||
|
confidence of 0.
|
||||||
|
- confidence is your own 0.0-1.0 estimate that the value is correct. Reserve
|
||||||
|
values above 0.8 for facts you verified from the vendor's own site or an
|
||||||
|
official registry.
|
||||||
|
- source_url is the page where you verified the value. Leave it empty when the
|
||||||
|
value was not found.
|
||||||
|
- Do not include commentary; return only the structured fields.
|
||||||
51
pkg/thirdparty/prompts/common_third_party_compliance_docs.txt.tmpl
vendored
Normal file
51
pkg/thirdparty/prompts/common_third_party_compliance_docs.txt.tmpl
vendored
Normal file
@@ -0,0 +1,51 @@
|
|||||||
|
You are a research agent that locates a software vendor's public compliance
|
||||||
|
documents and trust pages. You are given the vendor's name and, usually, its
|
||||||
|
website. Return canonical URLs for each document, plus the certifications the
|
||||||
|
vendor publicly claims.
|
||||||
|
|
||||||
|
Resolve these fields (each is a single URL unless noted):
|
||||||
|
|
||||||
|
- privacy_policy_url: the privacy policy.
|
||||||
|
- terms_of_service_url: the terms of service / terms of use.
|
||||||
|
- service_level_agreement_url: the public SLA. Frequently gated behind sales.
|
||||||
|
- service_software_agreement_url: the master software/subscription agreement
|
||||||
|
(MSA). Often gated, or the same document as the terms of service.
|
||||||
|
- data_processing_agreement_url: the DPA. Often a downloadable PDF; sometimes
|
||||||
|
only available on request.
|
||||||
|
- business_associate_agreement_url: the HIPAA BAA. Almost always gated behind
|
||||||
|
sales or an enterprise plan.
|
||||||
|
- subprocessors_list_url: the sub-processors list page.
|
||||||
|
- status_page_url: the uptime/status page (commonly status.<domain> or a
|
||||||
|
hosted statuspage.io / instatus / better-uptime page).
|
||||||
|
- security_page_url: the security overview page.
|
||||||
|
- trust_page_url: the trust center / trust portal. Many vendors host this on
|
||||||
|
Vanta, SafeBase, Drata, Conveyor, or a /trust path.
|
||||||
|
- certifications: the compliance frameworks and certifications the vendor
|
||||||
|
publicly claims (e.g. SOC 2 Type II, ISO 27001, ISO 27701, PCI DSS, HIPAA,
|
||||||
|
GDPR, FedRAMP). Read these from the trust or security page.
|
||||||
|
|
||||||
|
Method:
|
||||||
|
|
||||||
|
- Start from the vendor's <website> when provided. Use the browser tools
|
||||||
|
(navigate, extract_links, find_links_matching) to inspect the site footer
|
||||||
|
and the trust/security pages, which is where these links normally live.
|
||||||
|
- Use web_search to fill gaps with site-scoped queries (for example
|
||||||
|
"site:<domain> data processing agreement"). Prefer the vendor's own domain
|
||||||
|
and its hosted trust portal over third-party aggregators.
|
||||||
|
- Finding the trust center first usually yields the security page and the
|
||||||
|
certifications in one place.
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
|
||||||
|
- Return the most specific canonical URL. Prefer a direct document/page URL
|
||||||
|
over a generic legal-index page.
|
||||||
|
- Never guess or fabricate a URL. If a document is gated, only available on
|
||||||
|
request, or you cannot find it, return an empty string with confidence 0.
|
||||||
|
Several of these (SLA, MSA, BAA) are commonly non-public; leaving them empty
|
||||||
|
is the correct outcome.
|
||||||
|
- confidence is your own 0.0-1.0 estimate that the URL is correct and current.
|
||||||
|
Reserve values above 0.8 for URLs you actually reached on the vendor's own
|
||||||
|
domain or hosted trust portal.
|
||||||
|
- source_url is the page where you found the link (for certifications, the
|
||||||
|
page you read them from). Leave it empty when nothing was found.
|
||||||
|
- Do not include commentary; return only the structured fields.
|
||||||
10
pkg/thirdparty/resolver.go
vendored
10
pkg/thirdparty/resolver.go
vendored
@@ -70,8 +70,14 @@ func ResolveOrCreateCommonThirdParty(
|
|||||||
Slug: partySlug,
|
Slug: partySlug,
|
||||||
Category: category,
|
Category: category,
|
||||||
Certifications: []string{},
|
Certifications: []string{},
|
||||||
CreatedAt: now,
|
// Request enrichment at creation: a freshly resolved catalog row
|
||||||
UpdatedAt: now,
|
// carries only name/slug/category, so the enrichment worker fills
|
||||||
|
// the rest (URLs, address, certifications, logo). Curated seed
|
||||||
|
// rows are inserted via Upsert without this flag, so a full
|
||||||
|
// re-seed does not trigger an enrichment storm.
|
||||||
|
EnrichmentRequestedAt: &now,
|
||||||
|
CreatedAt: now,
|
||||||
|
UpdatedAt: now,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Insert inside a savepoint so a concurrent transaction that created
|
// Insert inside a savepoint so a concurrent transaction that created
|
||||||
|
|||||||
180
pkg/webinspect/logo.go
Normal file
180
pkg/webinspect/logo.go
Normal file
@@ -0,0 +1,180 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package webinspect
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"golang.org/x/net/html"
|
||||||
|
)
|
||||||
|
|
||||||
|
func FindLogoURL(info *PageInfo) (string, error) {
|
||||||
|
head := findElement(info.Root, "head")
|
||||||
|
if head == nil {
|
||||||
|
return "", fmt.Errorf("cannot find logo: no head element")
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
svgIcon string
|
||||||
|
appleTouchIcon string
|
||||||
|
appleTouchSize int
|
||||||
|
largestIcon string
|
||||||
|
largestSize int
|
||||||
|
msTileImage string
|
||||||
|
)
|
||||||
|
|
||||||
|
for _, n := range findAllIn(head, "link") {
|
||||||
|
rel := strings.ToLower(attrVal(n, "rel"))
|
||||||
|
href := attrVal(n, "href")
|
||||||
|
if href == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case strings.Contains(rel, "icon") && !strings.Contains(rel, "apple-touch-icon") && attrVal(n, "type") == "image/svg+xml":
|
||||||
|
svgIcon = href
|
||||||
|
case strings.Contains(rel, "apple-touch-icon"):
|
||||||
|
size := parseSizeAttr(attrVal(n, "sizes"))
|
||||||
|
if appleTouchIcon == "" || size > appleTouchSize {
|
||||||
|
appleTouchIcon = href
|
||||||
|
appleTouchSize = size
|
||||||
|
}
|
||||||
|
case strings.Contains(rel, "icon") && !strings.Contains(rel, "apple-touch-icon"):
|
||||||
|
size := parseSizeAttr(attrVal(n, "sizes"))
|
||||||
|
if largestIcon == "" || size > largestSize {
|
||||||
|
largestIcon = href
|
||||||
|
largestSize = size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, n := range findAllIn(head, "meta") {
|
||||||
|
name := strings.ToLower(attrVal(n, "name"))
|
||||||
|
content := attrVal(n, "content")
|
||||||
|
if name == "msapplication-tileimage" && content != "" {
|
||||||
|
msTileImage = content
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates := []string{
|
||||||
|
svgIcon,
|
||||||
|
appleTouchIcon,
|
||||||
|
largestIcon,
|
||||||
|
msTileImage,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, href := range candidates {
|
||||||
|
if href != "" {
|
||||||
|
return info.ResolveHref(href), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", fmt.Errorf("cannot find logo")
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseSizeAttr(sizes string) int {
|
||||||
|
if sizes == "" || strings.EqualFold(sizes, "any") {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
best := 0
|
||||||
|
for token := range strings.FieldsSeq(sizes) {
|
||||||
|
token = strings.ToLower(token)
|
||||||
|
parts := strings.SplitN(token, "x", 2)
|
||||||
|
if len(parts) != 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
w, err := strconv.Atoi(parts[0])
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if w > best {
|
||||||
|
best = w
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return best
|
||||||
|
}
|
||||||
|
|
||||||
|
func ExtensionForMIME(contentType string) string {
|
||||||
|
ct := strings.ToLower(contentType)
|
||||||
|
if idx := strings.Index(ct, ";"); idx != -1 {
|
||||||
|
ct = ct[:idx]
|
||||||
|
}
|
||||||
|
ct = strings.TrimSpace(ct)
|
||||||
|
|
||||||
|
switch ct {
|
||||||
|
case "image/svg+xml":
|
||||||
|
return ".svg"
|
||||||
|
case "image/png":
|
||||||
|
return ".png"
|
||||||
|
case "image/jpeg":
|
||||||
|
return ".jpg"
|
||||||
|
case "image/gif":
|
||||||
|
return ".gif"
|
||||||
|
case "image/webp":
|
||||||
|
return ".webp"
|
||||||
|
case "image/x-icon", "image/vnd.microsoft.icon":
|
||||||
|
return ".ico"
|
||||||
|
default:
|
||||||
|
return ".png"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// HeadLinks returns all <link> nodes from <head> whose rel attribute
|
||||||
|
// contains the given value (case-insensitive partial match).
|
||||||
|
func (p *PageInfo) HeadLinks(rel string) []*html.Node {
|
||||||
|
head := findElement(p.Root, "head")
|
||||||
|
if head == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
rel = strings.ToLower(rel)
|
||||||
|
var matches []*html.Node
|
||||||
|
|
||||||
|
for _, n := range findAllIn(head, "link") {
|
||||||
|
if strings.Contains(strings.ToLower(attrVal(n, "rel")), rel) {
|
||||||
|
matches = append(matches, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return matches
|
||||||
|
}
|
||||||
|
|
||||||
|
// HeadMeta returns the content attribute of the first <meta> tag in <head>
|
||||||
|
// whose name attribute matches (case-insensitive).
|
||||||
|
func (p *PageInfo) HeadMeta(name string) (string, bool) {
|
||||||
|
head := findElement(p.Root, "head")
|
||||||
|
if head == nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
name = strings.ToLower(name)
|
||||||
|
|
||||||
|
for _, n := range findAllIn(head, "meta") {
|
||||||
|
if strings.EqualFold(attrVal(n, "name"), name) {
|
||||||
|
content := attrVal(n, "content")
|
||||||
|
if content != "" {
|
||||||
|
return content, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
163
pkg/webinspect/logo_test.go
Normal file
163
pkg/webinspect/logo_test.go
Normal file
@@ -0,0 +1,163 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
package webinspect_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"go.probo.inc/probo/pkg/webinspect"
|
||||||
|
)
|
||||||
|
|
||||||
|
func parseTestHTML(t *testing.T, rawURL string, body string) *webinspect.PageInfo {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
u, err := url.Parse(rawURL)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
info, err := webinspect.ParseHTML(u, strings.NewReader(body))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
return info
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFindLogoURL_SVGPreferred(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
info := parseTestHTML(t, "https://example.com", `<html><head>
|
||||||
|
<link rel="icon" type="image/svg+xml" href="/favicon.svg">
|
||||||
|
<link rel="apple-touch-icon" href="/apple-touch-icon.png">
|
||||||
|
<link rel="icon" sizes="192x192" href="/icon-192.png">
|
||||||
|
</head><body></body></html>`)
|
||||||
|
|
||||||
|
got, err := webinspect.FindLogoURL(info)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "https://example.com/favicon.svg", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFindLogoURL_AppleTouchIconSecond(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
info := parseTestHTML(t, "https://example.com", `<html><head>
|
||||||
|
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
|
||||||
|
<link rel="icon" sizes="32x32" href="/favicon-32.png">
|
||||||
|
</head><body></body></html>`)
|
||||||
|
|
||||||
|
got, err := webinspect.FindLogoURL(info)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "https://example.com/apple-touch-icon.png", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFindLogoURL_AppleTouchIconLargest(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
info := parseTestHTML(t, "https://example.com", `<html><head>
|
||||||
|
<link rel="apple-touch-icon" sizes="120x120" href="/touch-120.png">
|
||||||
|
<link rel="apple-touch-icon" sizes="180x180" href="/touch-180.png">
|
||||||
|
<link rel="apple-touch-icon" sizes="152x152" href="/touch-152.png">
|
||||||
|
</head><body></body></html>`)
|
||||||
|
|
||||||
|
got, err := webinspect.FindLogoURL(info)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "https://example.com/touch-180.png", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFindLogoURL_LargestIcon(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
info := parseTestHTML(t, "https://example.com", `<html><head>
|
||||||
|
<link rel="icon" sizes="32x32" href="/icon-32.png">
|
||||||
|
<link rel="icon" sizes="192x192" href="/icon-192.png">
|
||||||
|
<link rel="icon" sizes="16x16" href="/icon-16.png">
|
||||||
|
</head><body></body></html>`)
|
||||||
|
|
||||||
|
got, err := webinspect.FindLogoURL(info)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "https://example.com/icon-192.png", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFindLogoURL_MsTileImage(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
info := parseTestHTML(t, "https://example.com", `<html><head>
|
||||||
|
<meta name="msapplication-TileImage" content="/mstile-144.png">
|
||||||
|
</head><body></body></html>`)
|
||||||
|
|
||||||
|
got, err := webinspect.FindLogoURL(info)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "https://example.com/mstile-144.png", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFindLogoURL_RelativeHrefResolved(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
info := parseTestHTML(t, "https://cdn.example.com/app/", `<html><head>
|
||||||
|
<link rel="icon" type="image/svg+xml" href="../assets/logo.svg">
|
||||||
|
</head><body></body></html>`)
|
||||||
|
|
||||||
|
got, err := webinspect.FindLogoURL(info)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "https://cdn.example.com/assets/logo.svg", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFindLogoURL_NoHead(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
info := parseTestHTML(t, "https://example.com", `<html><body><p>no head</p></body></html>`)
|
||||||
|
|
||||||
|
_, err := webinspect.FindLogoURL(info)
|
||||||
|
assert.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFindLogoURL_NothingFound(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
info := parseTestHTML(t, "https://example.com", `<html><head></head><body></body></html>`)
|
||||||
|
|
||||||
|
_, err := webinspect.FindLogoURL(info)
|
||||||
|
assert.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtensionForMIME(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
contentType string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{"image/svg+xml", ".svg"},
|
||||||
|
{"image/png", ".png"},
|
||||||
|
{"image/jpeg", ".jpg"},
|
||||||
|
{"image/gif", ".gif"},
|
||||||
|
{"image/webp", ".webp"},
|
||||||
|
{"image/x-icon", ".ico"},
|
||||||
|
{"image/vnd.microsoft.icon", ".ico"},
|
||||||
|
{"image/png; charset=utf-8", ".png"},
|
||||||
|
{"application/octet-stream", ".png"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(
|
||||||
|
tt.contentType,
|
||||||
|
func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
assert.Equal(t, tt.expected, webinspect.ExtensionForMIME(tt.contentType))
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
117
pkg/webinspect/parse.go
Normal file
117
pkg/webinspect/parse.go
Normal file
@@ -0,0 +1,117 @@
|
|||||||
|
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
|
||||||
|
//
|
||||||
|
// Permission to use, copy, modify, and/or distribute this software for any
|
||||||
|
// purpose with or without fee is hereby granted, provided that the above
|
||||||
|
// copyright notice and this permission notice appear in all copies.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
|
||||||
|
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||||
|
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
|
||||||
|
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
|
||||||
|
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
|
||||||
|
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
// PERFORMANCE OF THIS SOFTWARE.
|
||||||
|
|
||||||
|
// Package webinspect parses a web page's static HTML to extract metadata
|
||||||
|
// from its <head> (icons/logos, meta tags, link relations). It is a pure,
|
||||||
|
// deterministic helper: callers supply an http.Client (e.g. one with SSRF
|
||||||
|
// protection) so the package itself makes no policy decisions about which
|
||||||
|
// hosts are reachable. The logo discovery is reused by the common
|
||||||
|
// third-party enricher to populate logo_file_id without an LLM.
|
||||||
|
package webinspect
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
|
||||||
|
"golang.org/x/net/html"
|
||||||
|
)
|
||||||
|
|
||||||
|
type PageInfo struct {
|
||||||
|
URL *url.URL
|
||||||
|
Root *html.Node
|
||||||
|
}
|
||||||
|
|
||||||
|
func Parse(ctx context.Context, client *http.Client, websiteURL string) (*PageInfo, error) {
|
||||||
|
parsed, err := url.Parse(websiteURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot parse website URL: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, websiteURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot create request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot fetch page: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("cannot fetch page: status %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
const maxHTMLSize = 10 << 20 // 10 MiB
|
||||||
|
return ParseHTML(parsed, io.LimitReader(resp.Body, maxHTMLSize))
|
||||||
|
}
|
||||||
|
|
||||||
|
func ParseHTML(baseURL *url.URL, r io.Reader) (*PageInfo, error) {
|
||||||
|
root, err := html.Parse(r)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot parse HTML: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &PageInfo{URL: baseURL, Root: root}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *PageInfo) ResolveHref(href string) string {
|
||||||
|
ref, err := url.Parse(href)
|
||||||
|
if err != nil {
|
||||||
|
return href
|
||||||
|
}
|
||||||
|
|
||||||
|
return p.URL.ResolveReference(ref).String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func findElement(n *html.Node, tag string) *html.Node {
|
||||||
|
if n.Type == html.ElementNode && n.Data == tag {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||||
|
if found := findElement(c, tag); found != nil {
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func findAllIn(parent *html.Node, tag string) []*html.Node {
|
||||||
|
var nodes []*html.Node
|
||||||
|
|
||||||
|
for c := parent.FirstChild; c != nil; c = c.NextSibling {
|
||||||
|
if c.Type == html.ElementNode && c.Data == tag {
|
||||||
|
nodes = append(nodes, c)
|
||||||
|
}
|
||||||
|
|
||||||
|
nodes = append(nodes, findAllIn(c, tag)...)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nodes
|
||||||
|
}
|
||||||
|
|
||||||
|
func attrVal(n *html.Node, key string) string {
|
||||||
|
for _, a := range n.Attr {
|
||||||
|
if a.Key == key {
|
||||||
|
return a.Val
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ""
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user