Add common third party enricher worker

Introduce a poll-based worker that fills the global common_third_parties
catalog (URLs, headquarter address, legal name, certifications, logo)
so each tenant no longer starts from sparse, name-only rows. Enrichment
is requested at row creation by ResolveOrCreateCommonThirdParty; curated
seed rows are not enqueued, to avoid a re-seed storm.

The pipeline uses two specialized agents plus a deterministic logo step.
Agent A (company profile) resolves legal name, headquarter address, and
the canonical website over web search; its website and legal name feed
Agent B and the logo step. Agent B (compliance docs) resolves the legal
document URLs, trust/security/status pages, and certifications using the
browser read-only toolset (gated on ChromeDPAddr) plus web search. The
logo step restores pkg/webinspect as a pure deterministic package and
stores the discovered icon in S3, linked via logo_file_id.

Each agent returns per-field value/confidence/source_url. The worker
writes a column only when confidence clears a configurable threshold and
the field is not externally owned (seed or human), and always records
full per-field provenance in a new enrichment JSONB column so re-runs
fill only gaps and human edits are never clobbered. New bookkeeping
columns (enrichment_requested_at, enrichment, enrichment_attempts) back
the claim queue and stale recovery; agents run outside transactions and
results persist in one final transaction.

The worker is opt-in: it no-ops unless its agent provider is configured.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-06-10 13:49:56 +02:00
parent 99d568d07d
commit 229c6b99c6
23 changed files with 2325 additions and 15 deletions

View File

@@ -378,6 +378,52 @@ spec:
- name: COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS
value: {{ .Values.probo.commonPatternEnrichmentWorker.agentMaxTurns | quote }}
{{- end }}
# Common Third Party Enrichment Agent
{{- if .Values.probo.commonThirdPartyEnrichment.provider }}
- name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_PROVIDER
value: {{ .Values.probo.commonThirdPartyEnrichment.provider | quote }}
{{- end }}
{{- if .Values.probo.commonThirdPartyEnrichment.modelName }}
- name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MODEL_NAME
value: {{ .Values.probo.commonThirdPartyEnrichment.modelName | quote }}
{{- end }}
{{- if .Values.probo.commonThirdPartyEnrichment.temperature }}
- name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_TEMPERATURE
value: {{ .Values.probo.commonThirdPartyEnrichment.temperature | quote }}
{{- end }}
{{- if .Values.probo.commonThirdPartyEnrichment.maxTokens }}
- name: AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MAX_TOKENS
value: {{ .Values.probo.commonThirdPartyEnrichment.maxTokens | quote }}
{{- end }}
# Common Third Party Enrichment Worker
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.interval }}
- name: COMMON_THIRD_PARTY_ENRICHMENT_INTERVAL
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.interval | quote }}
{{- end }}
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.maxConcurrency }}
- name: COMMON_THIRD_PARTY_ENRICHMENT_MAX_CONCURRENCY
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.maxConcurrency | quote }}
{{- end }}
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.staleAfter }}
- name: COMMON_THIRD_PARTY_ENRICHMENT_STALE_AFTER
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.staleAfter | quote }}
{{- end }}
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.agentTimeout }}
- name: COMMON_THIRD_PARTY_ENRICHMENT_AGENT_TIMEOUT
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.agentTimeout | quote }}
{{- end }}
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.agentMaxTurns }}
- name: COMMON_THIRD_PARTY_ENRICHMENT_AGENT_MAX_TURNS
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.agentMaxTurns | quote }}
{{- end }}
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.confidenceThreshold }}
- name: COMMON_THIRD_PARTY_ENRICHMENT_CONFIDENCE_THRESHOLD
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.confidenceThreshold | quote }}
{{- end }}
{{- if .Values.probo.commonThirdPartyEnrichmentWorker.maxAttempts }}
- name: COMMON_THIRD_PARTY_ENRICHMENT_MAX_ATTEMPTS
value: {{ .Values.probo.commonThirdPartyEnrichmentWorker.maxAttempts | quote }}
{{- end }}
# Custom Domains
{{- if .Values.probo.customDomains.enabled }}
- name: CUSTOM_DOMAINS_RENEWAL_INTERVAL

View File

@@ -229,6 +229,26 @@ probo:
# agentTimeout: 45
# agentMaxTurns: 10
# Common third party enrichment agent (optional, fills common_third_parties
# metadata: URLs, address, certifications, logo).
# commonThirdPartyEnrichment:
# provider: "openai"
# modelName: "gpt-4o"
# temperature: "0.1"
# maxTokens: "8192"
# Common third party enrichment worker tuning (optional; seconds for
# interval/staleAfter/agentTimeout). confidenceThreshold is the 0-1 floor
# a value must clear before it is written.
# commonThirdPartyEnrichmentWorker:
# interval: 10
# maxConcurrency: 1
# staleAfter: 900
# agentTimeout: 90
# agentMaxTurns: 12
# confidenceThreshold: 0.7
# maxAttempts: 3
# OpenTelemetry tracing (optional)
tracing:
enabled: true

View File

@@ -331,6 +331,30 @@ probo:
agentTimeout: 45
agentMaxTurns: 10
# Common third party enrichment agent (optional, requires a provider
# key). Powers the company-profile and compliance-docs agents that fill
# common_third_parties metadata. Browser tools for the compliance-docs
# agent use the chromeDpAddr endpoint when set; web search uses
# firecrawl.apiKey when set.
commonThirdPartyEnrichment:
provider: ""
modelName: ""
temperature: ""
maxTokens: ""
# Common third party enrichment background worker tuning (optional).
# interval, staleAfter, and agentTimeout are in seconds.
# confidenceThreshold is the floor (0-1) a resolved value must clear
# before it is written to its column.
commonThirdPartyEnrichmentWorker:
interval: 10
maxConcurrency: 1
staleAfter: 900
agentTimeout: 90
agentMaxTurns: 12
confidenceThreshold: 0.7
maxAttempts: 3
# Custom domains configuration (optional)
customDomains:
enabled: false

View File

@@ -242,6 +242,15 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
Temperature: b.getEnvFloatPtr("AGENT_TRACKER_ENRICHMENT_TEMPERATURE"),
MaxTokens: new(b.getEnvIntOrDefault("AGENT_TRACKER_ENRICHMENT_MAX_TOKENS", 4096)),
},
CommonThirdPartyEnrichment: probodconfig.LLMAgentConfig{
Provider: b.getEnvOrDefault("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_PROVIDER", ""),
ModelName: b.getEnvOrDefault("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MODEL_NAME", ""),
// Agent B browses pages and emits a moderate structured
// output; the budget must leave headroom for reasoning
// models whose reasoning tokens count against max_tokens.
Temperature: b.getEnvFloatPtr("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_TEMPERATURE"),
MaxTokens: new(b.getEnvIntOrDefault("AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MAX_TOKENS", 8192)),
},
Tools: probodconfig.AgentToolsConfig{
FirecrawlAPIKey: b.getEnv("FIRECRAWL_API_KEY"),
},
@@ -292,6 +301,15 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
AgentTimeout: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_AGENT_TIMEOUT", 45),
AgentMaxTurns: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS", 10),
},
CommonThirdPartyEnrichmentWorker: probodconfig.CommonThirdPartyEnrichmentWorkerConfig{
Interval: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_INTERVAL", 10),
MaxConcurrency: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_MAX_CONCURRENCY", 1),
StaleAfter: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_STALE_AFTER", 900),
AgentTimeout: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_AGENT_TIMEOUT", 90),
AgentMaxTurns: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_AGENT_MAX_TURNS", 12),
ConfidenceThreshold: b.getEnvFloatOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_CONFIDENCE_THRESHOLD", 0.7),
MaxAttempts: b.getEnvIntOrDefault("COMMON_THIRD_PARTY_ENRICHMENT_MAX_ATTEMPTS", 3),
},
Branding: b.getEnvBoolOrDefault("BRANDING", true),
},
}

View File

@@ -245,6 +245,13 @@ func TestBuilder_Build_Defaults(t *testing.T) {
assert.Equal(t, 600, cfg.Probod.CommonPatternEnrichmentWorker.StaleAfter)
assert.Equal(t, 45, cfg.Probod.CommonPatternEnrichmentWorker.AgentTimeout)
assert.Equal(t, 10, cfg.Probod.CommonPatternEnrichmentWorker.AgentMaxTurns)
assert.Equal(t, 10, cfg.Probod.CommonThirdPartyEnrichmentWorker.Interval)
assert.Equal(t, 1, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxConcurrency)
assert.Equal(t, 900, cfg.Probod.CommonThirdPartyEnrichmentWorker.StaleAfter)
assert.Equal(t, 90, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentTimeout)
assert.Equal(t, 12, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentMaxTurns)
assert.Equal(t, 0.7, cfg.Probod.CommonThirdPartyEnrichmentWorker.ConfidenceThreshold)
assert.Equal(t, 3, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxAttempts)
assert.Equal(t, 10, cfg.Probod.ThirdPartyVetting.Interval)
assert.Equal(t, 1500, cfg.Probod.ThirdPartyVetting.StaleAfter)
assert.Equal(t, 1, cfg.Probod.ThirdPartyVetting.MaxConcurrency)
@@ -369,6 +376,17 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
env["COMMON_PATTERN_ENRICHMENT_STALE_AFTER"] = "900"
env["COMMON_PATTERN_ENRICHMENT_AGENT_TIMEOUT"] = "50"
env["COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS"] = "5"
// Common third party enrichment agent + worker tuning override
env["AGENT_COMMON_THIRD_PARTY_ENRICHMENT_PROVIDER"] = "openai"
env["AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MODEL_NAME"] = "gpt-4o"
env["AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MAX_TOKENS"] = "16384"
env["COMMON_THIRD_PARTY_ENRICHMENT_INTERVAL"] = "25"
env["COMMON_THIRD_PARTY_ENRICHMENT_MAX_CONCURRENCY"] = "2"
env["COMMON_THIRD_PARTY_ENRICHMENT_STALE_AFTER"] = "1200"
env["COMMON_THIRD_PARTY_ENRICHMENT_AGENT_TIMEOUT"] = "120"
env["COMMON_THIRD_PARTY_ENRICHMENT_AGENT_MAX_TURNS"] = "8"
env["COMMON_THIRD_PARTY_ENRICHMENT_CONFIDENCE_THRESHOLD"] = "0.85"
env["COMMON_THIRD_PARTY_ENRICHMENT_MAX_ATTEMPTS"] = "5"
env["THIRD_PARTY_VETTING_INTERVAL"] = "15"
env["THIRD_PARTY_VETTING_STALE_AFTER"] = "1800"
env["THIRD_PARTY_VETTING_MAX_CONCURRENCY"] = "2"
@@ -490,6 +508,17 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
assert.Equal(t, 900, cfg.Probod.CommonPatternEnrichmentWorker.StaleAfter)
assert.Equal(t, 50, cfg.Probod.CommonPatternEnrichmentWorker.AgentTimeout)
assert.Equal(t, 5, cfg.Probod.CommonPatternEnrichmentWorker.AgentMaxTurns)
assert.Equal(t, "openai", cfg.Probod.Agents.CommonThirdPartyEnrichment.Provider)
assert.Equal(t, "gpt-4o", cfg.Probod.Agents.CommonThirdPartyEnrichment.ModelName)
require.NotNil(t, cfg.Probod.Agents.CommonThirdPartyEnrichment.MaxTokens)
assert.Equal(t, 16384, *cfg.Probod.Agents.CommonThirdPartyEnrichment.MaxTokens)
assert.Equal(t, 25, cfg.Probod.CommonThirdPartyEnrichmentWorker.Interval)
assert.Equal(t, 2, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxConcurrency)
assert.Equal(t, 1200, cfg.Probod.CommonThirdPartyEnrichmentWorker.StaleAfter)
assert.Equal(t, 120, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentTimeout)
assert.Equal(t, 8, cfg.Probod.CommonThirdPartyEnrichmentWorker.AgentMaxTurns)
assert.Equal(t, 0.85, cfg.Probod.CommonThirdPartyEnrichmentWorker.ConfidenceThreshold)
assert.Equal(t, 5, cfg.Probod.CommonThirdPartyEnrichmentWorker.MaxAttempts)
assert.Equal(t, 15, cfg.Probod.ThirdPartyVetting.Interval)
assert.Equal(t, 1800, cfg.Probod.ThirdPartyVetting.StaleAfter)
assert.Equal(t, 2, cfg.Probod.ThirdPartyVetting.MaxConcurrency)

View File

@@ -16,6 +16,7 @@ package coredata
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
@@ -49,6 +50,9 @@ type (
SecurityPageURL *string `db:"security_page_url"`
TrustPageURL *string `db:"trust_page_url"`
LogoFileID *gid.GID `db:"logo_file_id"`
EnrichmentRequestedAt *time.Time `db:"enrichment_requested_at"`
Enrichment json.RawMessage `db:"enrichment"`
EnrichmentAttempts int `db:"enrichment_attempts"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
}
@@ -124,6 +128,9 @@ SELECT
security_page_url,
trust_page_url,
logo_file_id,
enrichment_requested_at,
enrichment,
enrichment_attempts,
created_at,
updated_at
FROM
@@ -181,6 +188,9 @@ SELECT
security_page_url,
trust_page_url,
logo_file_id,
enrichment_requested_at,
enrichment,
enrichment_attempts,
created_at,
updated_at
FROM
@@ -238,6 +248,9 @@ SELECT
security_page_url,
trust_page_url,
logo_file_id,
enrichment_requested_at,
enrichment,
enrichment_attempts,
created_at,
updated_at
FROM
@@ -294,6 +307,9 @@ INSERT INTO common_third_parties (
security_page_url,
trust_page_url,
logo_file_id,
enrichment_requested_at,
enrichment,
enrichment_attempts,
created_at,
updated_at
) VALUES (
@@ -316,6 +332,9 @@ INSERT INTO common_third_parties (
@security_page_url,
@trust_page_url,
@logo_file_id,
@enrichment_requested_at,
@enrichment,
@enrichment_attempts,
@created_at,
@updated_at
)
@@ -341,6 +360,9 @@ INSERT INTO common_third_parties (
"security_page_url": t.SecurityPageURL,
"trust_page_url": t.TrustPageURL,
"logo_file_id": t.LogoFileID,
"enrichment_requested_at": t.EnrichmentRequestedAt,
"enrichment": t.Enrichment,
"enrichment_attempts": t.EnrichmentAttempts,
"created_at": t.CreatedAt,
"updated_at": t.UpdatedAt,
}
@@ -381,6 +403,9 @@ INSERT INTO common_third_parties (
security_page_url,
trust_page_url,
logo_file_id,
enrichment_requested_at,
enrichment,
enrichment_attempts,
created_at,
updated_at
) VALUES (
@@ -403,6 +428,9 @@ INSERT INTO common_third_parties (
@security_page_url,
@trust_page_url,
@logo_file_id,
@enrichment_requested_at,
@enrichment,
@enrichment_attempts,
@created_at,
@updated_at
)
@@ -445,6 +473,9 @@ RETURNING
security_page_url,
trust_page_url,
logo_file_id,
enrichment_requested_at,
enrichment,
enrichment_attempts,
created_at,
updated_at
`
@@ -471,6 +502,9 @@ RETURNING
"security_page_url": t.SecurityPageURL,
"trust_page_url": t.TrustPageURL,
"logo_file_id": t.LogoFileID,
"enrichment_requested_at": t.EnrichmentRequestedAt,
"enrichment": t.Enrichment,
"enrichment_attempts": t.EnrichmentAttempts,
"created_at": t.CreatedAt,
"updated_at": t.UpdatedAt,
}
@@ -534,6 +568,9 @@ SELECT
security_page_url,
trust_page_url,
logo_file_id,
enrichment_requested_at,
enrichment,
enrichment_attempts,
created_at,
updated_at
FROM
@@ -585,6 +622,9 @@ SELECT
security_page_url,
trust_page_url,
logo_file_id,
enrichment_requested_at,
enrichment,
enrichment_attempts,
created_at,
updated_at
FROM
@@ -690,6 +730,9 @@ SELECT
security_page_url,
trust_page_url,
logo_file_id,
enrichment_requested_at,
enrichment,
enrichment_attempts,
created_at,
updated_at
FROM
@@ -750,3 +793,213 @@ WHERE
return count, nil
}
// LoadNextForEnrichmentForUpdateSkipLocked claims the oldest row queued
// for enrichment. The global catalog is not tenant-scoped, so the claim
// is intentionally cross-tenant: the enrichment worker is a system
// worker that drains the queue regardless of tenant.
func (t *CommonThirdParty) LoadNextForEnrichmentForUpdateSkipLocked(
ctx context.Context,
tx pg.Tx,
) error {
q := `
SELECT
id,
name,
slug,
category,
headquarter_address,
legal_name,
website_url,
privacy_policy_url,
service_level_agreement_url,
service_software_agreement_url,
data_processing_agreement_url,
business_associate_agreement_url,
subprocessors_list_url,
certifications,
status_page_url,
terms_of_service_url,
security_page_url,
trust_page_url,
logo_file_id,
enrichment_requested_at,
enrichment,
enrichment_attempts,
created_at,
updated_at
FROM
common_third_parties
WHERE
enrichment_requested_at IS NOT NULL
ORDER BY
enrichment_requested_at ASC
FOR UPDATE SKIP LOCKED
LIMIT 1;
`
rows, err := tx.Query(ctx, q)
if err != nil {
return fmt.Errorf("cannot query common third party for enrichment: %w", err)
}
defer rows.Close()
row, err := pgx.CollectExactlyOneRow(rows, pgx.RowToStructByName[CommonThirdParty])
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return ErrResourceNotFound
}
return fmt.Errorf("cannot collect common third party for enrichment: %w", err)
}
*t = row
return nil
}
// ClearEnrichmentRequestedAt removes the row from the enrichment queue
// and bumps the attempt counter. It bumps updated_at so the
// stale-recovery clock starts at claim time, keeping
// ResetStaleCommonThirdPartyEnrichments from re-arming a row that is
// still being processed. The attempt counter is incremented up front so
// a crash between claim and persist still counts against the retry
// budget.
func (t *CommonThirdParty) ClearEnrichmentRequestedAt(
ctx context.Context,
tx pg.Tx,
) error {
q := `
UPDATE common_third_parties
SET
enrichment_requested_at = NULL,
enrichment_attempts = enrichment_attempts + 1,
updated_at = NOW()
WHERE id = @id
`
args := pgx.StrictNamedArgs{"id": t.ID}
_, err := tx.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot clear enrichment requested at: %w", err)
}
t.EnrichmentRequestedAt = nil
t.EnrichmentAttempts++
return nil
}
// UpdateEnrichment persists the enrichment result: the resolved metadata
// fields plus the per-field enrichment provenance JSON. It is a targeted
// partial update that never touches id, slug, category, name, logo, the
// queue column, or the attempt counter (logo is owned by
// UpdateLogoFileID; the queue column and counter are managed by
// ClearEnrichmentRequestedAt). The caller decides which scalar fields to
// write versus leave untouched, then passes the merged receiver here.
func (t CommonThirdParty) UpdateEnrichment(
ctx context.Context,
conn pg.Tx,
) error {
q := `
UPDATE common_third_parties
SET
headquarter_address = @headquarter_address,
legal_name = @legal_name,
website_url = @website_url,
privacy_policy_url = @privacy_policy_url,
service_level_agreement_url = @service_level_agreement_url,
service_software_agreement_url = @service_software_agreement_url,
data_processing_agreement_url = @data_processing_agreement_url,
business_associate_agreement_url = @business_associate_agreement_url,
subprocessors_list_url = @subprocessors_list_url,
certifications = @certifications,
status_page_url = @status_page_url,
terms_of_service_url = @terms_of_service_url,
security_page_url = @security_page_url,
trust_page_url = @trust_page_url,
enrichment = @enrichment,
updated_at = @updated_at
WHERE
id = @id
`
args := pgx.StrictNamedArgs{
"id": t.ID,
"headquarter_address": t.HeadquarterAddress,
"legal_name": t.LegalName,
"website_url": t.WebsiteURL,
"privacy_policy_url": t.PrivacyPolicyURL,
"service_level_agreement_url": t.ServiceLevelAgreementURL,
"service_software_agreement_url": t.ServiceSoftwareAgreementURL,
"data_processing_agreement_url": t.DataProcessingAgreementURL,
"business_associate_agreement_url": t.BusinessAssociateAgreementURL,
"subprocessors_list_url": t.SubprocessorsListURL,
"certifications": t.Certifications,
"status_page_url": t.StatusPageURL,
"terms_of_service_url": t.TermsOfServiceURL,
"security_page_url": t.SecurityPageURL,
"trust_page_url": t.TrustPageURL,
"enrichment": t.Enrichment,
"updated_at": t.UpdatedAt,
}
result, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot update common third party enrichment: %w", err)
}
if result.RowsAffected() == 0 {
return ErrResourceNotFound
}
return nil
}
// ResetStaleCommonThirdPartyEnrichments re-arms enrichment_requested_at
// on rows whose enrichment was claimed but never completed and have been
// idle longer than staleAfter, so a crashed or timed-out run is retried.
//
// A claimed row has enrichment_attempts > 0 (Claim increments it) and a
// completed row has a non-null enrichment payload (Process always writes
// it, even on a no-result run), so the sweep targets rows that were
// claimed but carry no enrichment yet. Curated rows that were never
// enqueued keep enrichment_attempts = 0 and are left untouched. The
// max-attempts ceiling stops permanently failing rows from looping
// forever.
//
// Like the claim query, this sweep is intentionally cross-tenant: the
// enrichment worker is a system worker that drains the queue regardless
// of tenant.
func ResetStaleCommonThirdPartyEnrichments(
ctx context.Context,
conn pg.Querier,
staleAfter time.Duration,
maxAttempts int,
) error {
q := `
UPDATE common_third_parties
SET
enrichment_requested_at = NOW(),
updated_at = NOW()
WHERE
enrichment_requested_at IS NULL
AND enrichment IS NULL
AND enrichment_attempts > 0
AND enrichment_attempts < @max_attempts
AND updated_at < @stale_before
`
args := pgx.StrictNamedArgs{
"max_attempts": maxAttempts,
"stale_before": time.Now().Add(-staleAfter),
}
_, err := conn.Exec(ctx, q, args)
if err != nil {
return fmt.Errorf("cannot reset stale common third party enrichments: %w", err)
}
return nil
}

View File

@@ -0,0 +1,28 @@
-- Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
--
-- Permission to use, copy, modify, and/or distribute this software for any
-- purpose with or without fee is hereby granted, provided that the above
-- copyright notice and this permission notice appear in all copies.
--
-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-- PERFORMANCE OF THIS SOFTWARE.
ALTER TABLE
common_third_parties
ADD
COLUMN enrichment_requested_at TIMESTAMP WITH TIME ZONE,
ADD
COLUMN enrichment JSONB,
ADD
COLUMN enrichment_attempts INTEGER NOT NULL DEFAULT 0;
-- Partial index backs the enrichment worker's claim query, which polls
-- only rows currently queued for enrichment.
CREATE INDEX common_third_parties_enrichment_requested_at_idx
ON common_third_parties (enrichment_requested_at)
WHERE enrichment_requested_at IS NOT NULL;

View File

@@ -43,8 +43,9 @@ type (
ThirdPartyVettingWorkerConfig = probodconfig.ThirdPartyVettingWorkerConfig
AgentsConfig = probodconfig.AgentsConfig
TrackerMappingWorkerConfig = probodconfig.TrackerMappingWorkerConfig
CommonPatternEnrichmentWorkerConfig = probodconfig.CommonPatternEnrichmentWorkerConfig
TrackerMappingWorkerConfig = probodconfig.TrackerMappingWorkerConfig
CommonPatternEnrichmentWorkerConfig = probodconfig.CommonPatternEnrichmentWorkerConfig
CommonThirdPartyEnrichmentWorkerConfig = probodconfig.CommonThirdPartyEnrichmentWorkerConfig
MailerConfig = probodconfig.MailerConfig
SMTPConfig = probodconfig.SMTPConfig

View File

@@ -0,0 +1,73 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package probod
import (
"fmt"
"time"
"github.com/prometheus/client_golang/prometheus"
"go.gearno.de/kit/log"
"go.opentelemetry.io/otel/trace"
"go.probo.inc/probo/pkg/filemanager"
"go.probo.inc/probo/pkg/thirdparty"
)
// buildCommonThirdPartyEnrichmentConfig wires the common-third-party
// enrichment worker config: the LLM client for its two agents plus the
// worker tuning, browser endpoint, and logo-storage dependencies. It is
// opt-in: a deployment that does not set
// `llm.common-third-party-enrichment.provider` gets a zero config (nil
// LLM client), so the worker runs as a no-op and the caller skips
// registration.
func (impl *Implm) buildCommonThirdPartyEnrichmentConfig(
l *log.Logger,
tp trace.TracerProvider,
r prometheus.Registerer,
fileManager *filemanager.Service,
) (thirdparty.EnrichmentConfig, error) {
if impl.cfg.Agents.CommonThirdPartyEnrichment.Provider == "" {
return thirdparty.EnrichmentConfig{}, nil
}
agentCfg, llmClient, err := impl.resolveAgentClient(
"common-third-party-enrichment",
impl.cfg.Agents.CommonThirdPartyEnrichment,
l,
tp,
r,
)
if err != nil {
return thirdparty.EnrichmentConfig{}, fmt.Errorf("cannot resolve common third party enrichment agent client: %w", err)
}
workerCfg := impl.cfg.CommonThirdPartyEnrichmentWorker
return thirdparty.EnrichmentConfig{
LLMClient: llmClient,
Model: agentCfg.ModelName,
MaxTokens: agentCfg.MaxTokens,
Temperature: agentCfg.Temperature,
FirecrawlAPIKey: impl.cfg.Agents.Tools.FirecrawlAPIKey,
ChromeAddr: impl.cfg.ChromeDPAddr,
AgentTimeout: time.Duration(workerCfg.AgentTimeout) * time.Second,
MaxTurns: workerCfg.AgentMaxTurns,
ConfidenceThreshold: workerCfg.ConfidenceThreshold,
StaleAfter: time.Duration(workerCfg.StaleAfter) * time.Second,
MaxAttempts: workerCfg.MaxAttempts,
FileManager: fileManager,
Bucket: impl.cfg.AWS.Bucket,
}, nil
}

View File

@@ -181,6 +181,15 @@ func New() *Implm {
StaleAfter: 1500,
MaxConcurrency: 1,
},
CommonThirdPartyEnrichmentWorker: CommonThirdPartyEnrichmentWorkerConfig{
Interval: 10,
MaxConcurrency: 1,
StaleAfter: 900,
AgentTimeout: 90,
AgentMaxTurns: 12,
ConfidenceThreshold: 0.7,
MaxAttempts: 3,
},
},
}
}
@@ -328,6 +337,11 @@ func (impl *Implm) Run(
fileManagerService := filemanager.NewService(pgClient, baseURL, s3Client)
commonThirdPartyEnrichmentCfg, err := impl.buildCommonThirdPartyEnrichmentConfig(l, tp, r, fileManagerService)
if err != nil {
return err
}
var (
samlCert *x509.Certificate
samlKey *rsa.PrivateKey
@@ -819,6 +833,34 @@ func (impl *Implm) Run(
)
}
// The common-third-party enrichment worker fills catalog metadata
// (URLs, address, certifications, logo) via two agents plus a
// deterministic logo step. It needs an LLM client, so it is only
// started when its agent config is present.
stopCommonThirdPartyEnrichmentWorker := func() {}
if commonThirdPartyEnrichmentCfg.LLMClient != nil {
commonThirdPartyEnrichmentWorker := thirdparty.NewCommonThirdPartyEnrichmentWorker(
pgClient,
l.Named("common-third-party-enrichment-worker"),
commonThirdPartyEnrichmentCfg,
worker.WithInterval(time.Duration(impl.cfg.CommonThirdPartyEnrichmentWorker.Interval)*time.Second),
worker.WithMaxConcurrency(impl.cfg.CommonThirdPartyEnrichmentWorker.MaxConcurrency),
)
var commonThirdPartyEnrichmentWorkerCtx context.Context
commonThirdPartyEnrichmentWorkerCtx, stopCommonThirdPartyEnrichmentWorker = context.WithCancel(context.Background())
wg.Go(
func() {
if err := commonThirdPartyEnrichmentWorker.Run(commonThirdPartyEnrichmentWorkerCtx); err != nil {
cancel(fmt.Errorf("common third party enrichment worker crashed: %w", err))
}
},
)
}
mailingListWorker := mailman.NewMailingListWorker(mailmanService, pgClient, l.Named("mailing-list-worker"))
mailingListWorkerCtx, stopMailingListWorker := context.WithCancel(context.Background())
@@ -910,6 +952,7 @@ func (impl *Implm) Run(
stopTrackerPolicyWorker()
stopTrackerMappingWorker()
stopCommonPatternEnrichmentWorker()
stopCommonThirdPartyEnrichmentWorker()
stopMailingListWorker()
stopVettingWorker()
stopEvidenceDescriptionWorker()

View File

@@ -62,8 +62,9 @@ type (
EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"`
ThirdPartyVetting ThirdPartyVettingWorkerConfig `json:"third-party-vetting-worker"`
TrackerMappingWorker TrackerMappingWorkerConfig `json:"tracker-mapping-worker"`
CommonPatternEnrichmentWorker CommonPatternEnrichmentWorkerConfig `json:"common-pattern-enrichment-worker"`
TrackerMappingWorker TrackerMappingWorkerConfig `json:"tracker-mapping-worker"`
CommonPatternEnrichmentWorker CommonPatternEnrichmentWorkerConfig `json:"common-pattern-enrichment-worker"`
CommonThirdPartyEnrichmentWorker CommonThirdPartyEnrichmentWorkerConfig `json:"common-third-party-enrichment-worker"`
ChromeDPAddr string `json:"chrome-dp-addr"`
CustomDomains CustomDomainsConfig `json:"custom-domains"`

View File

@@ -76,6 +76,22 @@ type (
AgentMaxTurns int `json:"agent-max-turns"`
}
// CommonThirdPartyEnrichmentWorkerConfig holds worker-side tuning for
// the common-third-party enrichment background worker. LLM parameters
// for its agents live under AgentsConfig.CommonThirdPartyEnrichment.
// ConfidenceThreshold is the floor a resolved value must clear before
// it is written to its column; MaxAttempts caps stale-recovery
// retries.
CommonThirdPartyEnrichmentWorkerConfig struct {
Interval int `json:"interval"` // seconds between polls
MaxConcurrency int `json:"max-concurrency"`
StaleAfter int `json:"stale-after"` // seconds before a claim is recycled
AgentTimeout int `json:"agent-timeout"` // seconds, single agent run
AgentMaxTurns int `json:"agent-max-turns"`
ConfidenceThreshold float64 `json:"confidence-threshold"`
MaxAttempts int `json:"max-attempts"`
}
// AgentToolsConfig holds API keys and settings for external tools
// that agents can use (web search, scraping, etc.).
AgentToolsConfig struct {
@@ -86,15 +102,16 @@ type (
// settings. Default is used as a fallback when an agent-specific field
// is zero-valued.
AgentsConfig struct {
Providers map[string]LLMProviderConfig `json:"providers"`
Default LLMAgentConfig `json:"defaults"`
Probo LLMAgentConfig `json:"probo"`
EvidenceDescriber LLMAgentConfig `json:"evidence-describer"`
ThirdPartyVetter LLMAgentConfig `json:"third-party-vetter"`
ThirdPartyDisambiguation LLMAgentConfig `json:"third-party-disambiguation"`
TrackerMapping LLMAgentConfig `json:"tracker-mapping"`
TrackerEnrichment LLMAgentConfig `json:"tracker-enrichment"`
Tools AgentToolsConfig `json:"tools"`
Providers map[string]LLMProviderConfig `json:"providers"`
Default LLMAgentConfig `json:"defaults"`
Probo LLMAgentConfig `json:"probo"`
EvidenceDescriber LLMAgentConfig `json:"evidence-describer"`
ThirdPartyVetter LLMAgentConfig `json:"third-party-vetter"`
ThirdPartyDisambiguation LLMAgentConfig `json:"third-party-disambiguation"`
TrackerMapping LLMAgentConfig `json:"tracker-mapping"`
TrackerEnrichment LLMAgentConfig `json:"tracker-enrichment"`
CommonThirdPartyEnrichment LLMAgentConfig `json:"common-third-party-enrichment"`
Tools AgentToolsConfig `json:"tools"`
}
)

View File

@@ -0,0 +1,94 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package thirdparty
import (
_ "embed"
"fmt"
"strings"
"go.gearno.de/kit/log"
"go.probo.inc/probo/pkg/agent"
"go.probo.inc/probo/pkg/agent/tools/search"
"go.probo.inc/probo/pkg/coredata"
)
//go:embed prompts/common_third_party_company_profile.txt.tmpl
var companyProfilePrompt string
// CompanyProfileResult is the structured output of the company-profile
// agent (Agent A): the vendor's identity facts. website_url is the key
// signal the compliance-docs agent and the logo step depend on, so it is
// resolved here first.
type CompanyProfileResult struct {
LegalName EnrichedField `json:"legal_name" jsonschema:"The vendor's full legal company name including the entity suffix (e.g. 'Acme Technologies, Inc.')."`
HeadquarterAddress EnrichedField `json:"headquarter_address" jsonschema:"The vendor's headquarters postal address (street, city, region, country)."`
WebsiteURL EnrichedField `json:"website_url" jsonschema:"The vendor's canonical primary marketing website URL (https scheme, no tracking query parameters, no trailing path)."`
}
func buildCompanyProfileAgent(
cfg EnrichmentConfig,
logger *log.Logger,
) *agent.Agent {
var tools []agent.Tool
if cfg.FirecrawlAPIKey != "" {
tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlAPIKey))
}
outputType, err := agent.NewOutputType[CompanyProfileResult]("common_third_party_company_profile")
if err != nil {
panic(fmt.Sprintf("thirdparty: cannot build company profile output type: %s", err))
}
opts := []agent.Option{
agent.WithInstructions(companyProfilePrompt),
agent.WithModel(cfg.Model),
agent.WithOutputType(outputType),
agent.WithMaxTurns(resolveEnrichmentMaxTurns(cfg.MaxTurns)),
agent.WithMaxTokens(resolveEnrichmentMaxTokens(cfg.MaxTokens)),
agent.WithLogger(logger),
}
if len(tools) > 0 {
opts = append(opts, agent.WithTools(tools...))
}
if cfg.Temperature != nil {
opts = append(opts, agent.WithTemperature(*cfg.Temperature))
}
return agent.New("common-third-party-company-profile", cfg.LLMClient, opts...)
}
// buildCompanyProfilePrompt renders the per-row input for Agent A. Any
// values already on the row are passed as hints so the agent confirms or
// corrects them rather than starting cold.
func buildCompanyProfilePrompt(party coredata.CommonThirdParty) string {
var b strings.Builder
fmt.Fprintf(&b, "Research this company and return its profile.\n\n")
fmt.Fprintf(&b, "<name> %s </name>\n", party.Name)
if party.WebsiteURL != nil && strings.TrimSpace(*party.WebsiteURL) != "" {
fmt.Fprintf(&b, "<known_website> %s </known_website>\n", *party.WebsiteURL)
}
if party.LegalName != nil && strings.TrimSpace(*party.LegalName) != "" {
fmt.Fprintf(&b, "<known_legal_name> %s </known_legal_name>\n", *party.LegalName)
}
return b.String()
}

View File

@@ -0,0 +1,105 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package thirdparty
import (
_ "embed"
"fmt"
"strings"
"go.gearno.de/kit/log"
"go.probo.inc/probo/pkg/agent"
"go.probo.inc/probo/pkg/agent/tools/search"
)
//go:embed prompts/common_third_party_compliance_docs.txt.tmpl
var complianceDocsPrompt string
// ComplianceDocsResult is the structured output of the compliance-docs
// agent (Agent B): the legal-document URLs, trust/security/status pages,
// and certifications. These all live in the same source ecosystem (the
// vendor footer and trust portal), so one agent resolves them together.
type ComplianceDocsResult struct {
PrivacyPolicyURL EnrichedField `json:"privacy_policy_url" jsonschema:"URL of the vendor's privacy policy."`
TermsOfServiceURL EnrichedField `json:"terms_of_service_url" jsonschema:"URL of the vendor's terms of service / terms of use."`
ServiceLevelAgreementURL EnrichedField `json:"service_level_agreement_url" jsonschema:"URL of the vendor's public service level agreement (SLA). Often gated behind sales; return empty when not public."`
ServiceSoftwareAgreementURL EnrichedField `json:"service_software_agreement_url" jsonschema:"URL of the vendor's master software/subscription agreement (MSA). Often gated or identical to the terms of service; return empty when not public."`
DataProcessingAgreementURL EnrichedField `json:"data_processing_agreement_url" jsonschema:"URL of the vendor's data processing agreement (DPA). Often a PDF; return empty when only available on request."`
BusinessAssociateAgreementURL EnrichedField `json:"business_associate_agreement_url" jsonschema:"URL of the vendor's HIPAA business associate agreement (BAA). Almost always gated behind sales; return empty when not public."`
SubprocessorsListURL EnrichedField `json:"subprocessors_list_url" jsonschema:"URL of the vendor's sub-processors list page."`
StatusPageURL EnrichedField `json:"status_page_url" jsonschema:"URL of the vendor's uptime/status page (e.g. status.vendor.com)."`
SecurityPageURL EnrichedField `json:"security_page_url" jsonschema:"URL of the vendor's security page or security overview."`
TrustPageURL EnrichedField `json:"trust_page_url" jsonschema:"URL of the vendor's trust center / trust portal (e.g. Vanta, SafeBase, Drata hosted)."`
Certifications CertificationsField `json:"certifications" jsonschema:"Certifications and compliance frameworks the vendor publicly claims."`
}
// buildComplianceDocsAgent builds Agent B. extraTools carries the browser
// read-only toolset when a headless Chrome endpoint is configured; it is
// empty otherwise, in which case the agent relies on web_search alone.
func buildComplianceDocsAgent(
cfg EnrichmentConfig,
logger *log.Logger,
extraTools []agent.Tool,
) *agent.Agent {
tools := append([]agent.Tool{}, extraTools...)
if cfg.FirecrawlAPIKey != "" {
tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlAPIKey))
}
outputType, err := agent.NewOutputType[ComplianceDocsResult]("common_third_party_compliance_docs")
if err != nil {
panic(fmt.Sprintf("thirdparty: cannot build compliance docs output type: %s", err))
}
opts := []agent.Option{
agent.WithInstructions(complianceDocsPrompt),
agent.WithModel(cfg.Model),
agent.WithOutputType(outputType),
agent.WithMaxTurns(resolveEnrichmentMaxTurns(cfg.MaxTurns)),
agent.WithMaxTokens(resolveEnrichmentMaxTokens(cfg.MaxTokens)),
agent.WithLogger(logger),
}
if len(tools) > 0 {
opts = append(opts, agent.WithTools(tools...))
}
if cfg.Temperature != nil {
opts = append(opts, agent.WithTemperature(*cfg.Temperature))
}
return agent.New("common-third-party-compliance-docs", cfg.LLMClient, opts...)
}
// buildComplianceDocsPrompt renders the per-row input for Agent B,
// seeding it with the vendor name and the website/legal name resolved by
// Agent A so it can scope its search to the vendor's own domain.
func buildComplianceDocsPrompt(name, websiteURL, legalName string) string {
var b strings.Builder
fmt.Fprintf(&b, "Find the compliance documents and trust pages for this vendor.\n\n")
fmt.Fprintf(&b, "<name> %s </name>\n", name)
if w := strings.TrimSpace(websiteURL); w != "" {
fmt.Fprintf(&b, "<website> %s </website>\n", w)
}
if l := strings.TrimSpace(legalName); l != "" {
fmt.Fprintf(&b, "<legal_name> %s </legal_name>\n", l)
}
return b.String()
}

View File

@@ -0,0 +1,345 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package thirdparty
import (
"encoding/json"
"strings"
"time"
"go.probo.inc/probo/pkg/coredata"
)
// Provenance sources and per-field statuses recorded in the enrichment
// JSON. The "source" distinguishes values written by this enricher from
// values owned externally (curated seed data or a human edit), which the
// enricher must never overwrite.
const (
enrichmentSourceEnrichment = "enrichment"
enrichmentSourceExternal = "external"
enrichmentFieldStatusFound = "found"
enrichmentFieldStatusNotFound = "not_found"
enrichmentFieldStatusLowConfidence = "low_confidence"
enrichmentFieldStatusExternal = "exists_external"
// Run-level status recorded at the top of the enrichment payload.
enrichmentStatusDone = "done"
enrichmentStatusPartial = "partial"
enrichmentStatusFailed = "failed"
)
type (
// EnrichedField is the per-field unit the enrichment agents return:
// the resolved value plus a self-assessed confidence and the source
// URL where it was verified. The worker applies a confidence
// threshold before writing the value to its column.
EnrichedField struct {
Value string `json:"value" jsonschema:"The resolved value, or an empty string when not confidently found. Never guess."`
Confidence float64 `json:"confidence" jsonschema:"Confidence from 0.0 to 1.0 that the value is correct. Use 0 when the value was not found."`
SourceURL string `json:"source_url" jsonschema:"The URL where this value was verified, or an empty string."`
}
// CertificationsField is the list-valued counterpart of EnrichedField
// used for the certifications array.
CertificationsField struct {
Values []string `json:"values" jsonschema:"Certification or compliance framework names the vendor publicly claims (e.g. 'SOC 2 Type II', 'ISO 27001', 'HIPAA'). Empty when none are found."`
Confidence float64 `json:"confidence" jsonschema:"Confidence from 0.0 to 1.0 in the certifications list. Use 0 when none are found."`
SourceURL string `json:"source_url" jsonschema:"The URL where the certifications were found (trust or security page), or an empty string."`
}
// EnrichmentFieldMeta is the per-field provenance recorded in the
// common_third_parties.enrichment JSON column.
EnrichmentFieldMeta struct {
Confidence float64 `json:"confidence"`
SourceURL string `json:"source_url,omitempty"`
Status string `json:"status"`
Source string `json:"source"`
UpdatedAt time.Time `json:"updated_at"`
}
// EnrichmentMetadata is the full payload stored in the enrichment
// JSON column: run-level bookkeeping plus per-field provenance keyed
// by the column name.
EnrichmentMetadata struct {
Model string `json:"model,omitempty"`
AttemptedAt time.Time `json:"attempted_at"`
Status string `json:"status"`
Error string `json:"error,omitempty"`
Fields map[string]EnrichmentFieldMeta `json:"fields"`
}
)
// scalarField describes one *string column the enricher can fill,
// pairing the agent's resolved value with accessors into the receiver.
type scalarField struct {
name string
get func(*coredata.CommonThirdParty) *string
set func(*coredata.CommonThirdParty, *string)
result EnrichedField
}
// parseEnrichmentFields extracts the prior per-field provenance from a
// row's enrichment payload. A missing or malformed payload yields an
// empty map, which the merge treats as "no field is enrichment-owned".
func parseEnrichmentFields(raw json.RawMessage) map[string]EnrichmentFieldMeta {
if len(raw) == 0 {
return map[string]EnrichmentFieldMeta{}
}
var meta EnrichmentMetadata
if err := json.Unmarshal(raw, &meta); err != nil {
return map[string]EnrichmentFieldMeta{}
}
if meta.Fields == nil {
return map[string]EnrichmentFieldMeta{}
}
return meta.Fields
}
// applyScalarField merges one resolved scalar field into party, honoring
// the confidence threshold and prior provenance, and records the
// resulting per-field metadata. A value already present that this
// enricher did not write (curated seed data or a human edit) is left
// untouched. The column is written only when the value clears the
// threshold; below it the column keeps its prior value and the metadata
// records why nothing was written.
func applyScalarField(
party *coredata.CommonThirdParty,
meta map[string]EnrichmentFieldMeta,
prior map[string]EnrichmentFieldMeta,
field scalarField,
threshold float64,
now time.Time,
) {
value := strings.TrimSpace(field.result.Value)
sourceURL := strings.TrimSpace(field.result.SourceURL)
existing := field.get(party)
hasExisting := existing != nil && strings.TrimSpace(*existing) != ""
priorMeta, hadPrior := prior[field.name]
enrichmentOwned := hadPrior && priorMeta.Source == enrichmentSourceEnrichment
if hasExisting && !enrichmentOwned {
meta[field.name] = EnrichmentFieldMeta{
Status: enrichmentFieldStatusExternal,
Source: enrichmentSourceExternal,
UpdatedAt: now,
}
return
}
if value != "" && field.result.Confidence >= threshold {
v := value
field.set(party, &v)
meta[field.name] = EnrichmentFieldMeta{
Confidence: field.result.Confidence,
SourceURL: sourceURL,
Status: enrichmentFieldStatusFound,
Source: enrichmentSourceEnrichment,
UpdatedAt: now,
}
return
}
status := enrichmentFieldStatusNotFound
if value != "" {
status = enrichmentFieldStatusLowConfidence
}
meta[field.name] = EnrichmentFieldMeta{
Confidence: field.result.Confidence,
SourceURL: sourceURL,
Status: status,
Source: enrichmentSourceEnrichment,
UpdatedAt: now,
}
}
// applyCertifications is the list-valued counterpart of
// applyScalarField for the certifications column.
func applyCertifications(
party *coredata.CommonThirdParty,
meta map[string]EnrichmentFieldMeta,
prior map[string]EnrichmentFieldMeta,
result CertificationsField,
threshold float64,
now time.Time,
) {
const name = "certifications"
values := normalizeCertifications(result.Values)
sourceURL := strings.TrimSpace(result.SourceURL)
hasExisting := len(party.Certifications) > 0
priorMeta, hadPrior := prior[name]
enrichmentOwned := hadPrior && priorMeta.Source == enrichmentSourceEnrichment
if hasExisting && !enrichmentOwned {
meta[name] = EnrichmentFieldMeta{
Status: enrichmentFieldStatusExternal,
Source: enrichmentSourceExternal,
UpdatedAt: now,
}
return
}
if len(values) > 0 && result.Confidence >= threshold {
party.Certifications = values
meta[name] = EnrichmentFieldMeta{
Confidence: result.Confidence,
SourceURL: sourceURL,
Status: enrichmentFieldStatusFound,
Source: enrichmentSourceEnrichment,
UpdatedAt: now,
}
return
}
status := enrichmentFieldStatusNotFound
if len(values) > 0 {
status = enrichmentFieldStatusLowConfidence
}
meta[name] = EnrichmentFieldMeta{
Confidence: result.Confidence,
SourceURL: sourceURL,
Status: status,
Source: enrichmentSourceEnrichment,
UpdatedAt: now,
}
}
// normalizeCertifications trims, drops blanks, and de-duplicates the
// certification names returned by the agent, preserving order.
func normalizeCertifications(values []string) []string {
seen := make(map[string]struct{}, len(values))
out := make([]string, 0, len(values))
for _, v := range values {
v = strings.TrimSpace(v)
if v == "" {
continue
}
key := strings.ToLower(v)
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
out = append(out, v)
}
return out
}
// scalarFields returns the descriptor list pairing each *string column
// with the resolved value from the two agents. website_url comes from
// the company-profile agent; the document and page URLs come from the
// compliance-docs agent.
func scalarFields(
company CompanyProfileResult,
compliance ComplianceDocsResult,
) []scalarField {
return []scalarField{
{
name: "legal_name",
get: func(p *coredata.CommonThirdParty) *string { return p.LegalName },
set: func(p *coredata.CommonThirdParty, v *string) { p.LegalName = v },
result: company.LegalName,
},
{
name: "headquarter_address",
get: func(p *coredata.CommonThirdParty) *string { return p.HeadquarterAddress },
set: func(p *coredata.CommonThirdParty, v *string) { p.HeadquarterAddress = v },
result: company.HeadquarterAddress,
},
{
name: "website_url",
get: func(p *coredata.CommonThirdParty) *string { return p.WebsiteURL },
set: func(p *coredata.CommonThirdParty, v *string) { p.WebsiteURL = v },
result: company.WebsiteURL,
},
{
name: "privacy_policy_url",
get: func(p *coredata.CommonThirdParty) *string { return p.PrivacyPolicyURL },
set: func(p *coredata.CommonThirdParty, v *string) { p.PrivacyPolicyURL = v },
result: compliance.PrivacyPolicyURL,
},
{
name: "terms_of_service_url",
get: func(p *coredata.CommonThirdParty) *string { return p.TermsOfServiceURL },
set: func(p *coredata.CommonThirdParty, v *string) { p.TermsOfServiceURL = v },
result: compliance.TermsOfServiceURL,
},
{
name: "service_level_agreement_url",
get: func(p *coredata.CommonThirdParty) *string { return p.ServiceLevelAgreementURL },
set: func(p *coredata.CommonThirdParty, v *string) { p.ServiceLevelAgreementURL = v },
result: compliance.ServiceLevelAgreementURL,
},
{
name: "service_software_agreement_url",
get: func(p *coredata.CommonThirdParty) *string { return p.ServiceSoftwareAgreementURL },
set: func(p *coredata.CommonThirdParty, v *string) { p.ServiceSoftwareAgreementURL = v },
result: compliance.ServiceSoftwareAgreementURL,
},
{
name: "data_processing_agreement_url",
get: func(p *coredata.CommonThirdParty) *string { return p.DataProcessingAgreementURL },
set: func(p *coredata.CommonThirdParty, v *string) { p.DataProcessingAgreementURL = v },
result: compliance.DataProcessingAgreementURL,
},
{
name: "business_associate_agreement_url",
get: func(p *coredata.CommonThirdParty) *string { return p.BusinessAssociateAgreementURL },
set: func(p *coredata.CommonThirdParty, v *string) { p.BusinessAssociateAgreementURL = v },
result: compliance.BusinessAssociateAgreementURL,
},
{
name: "subprocessors_list_url",
get: func(p *coredata.CommonThirdParty) *string { return p.SubprocessorsListURL },
set: func(p *coredata.CommonThirdParty, v *string) { p.SubprocessorsListURL = v },
result: compliance.SubprocessorsListURL,
},
{
name: "status_page_url",
get: func(p *coredata.CommonThirdParty) *string { return p.StatusPageURL },
set: func(p *coredata.CommonThirdParty, v *string) { p.StatusPageURL = v },
result: compliance.StatusPageURL,
},
{
name: "security_page_url",
get: func(p *coredata.CommonThirdParty) *string { return p.SecurityPageURL },
set: func(p *coredata.CommonThirdParty, v *string) { p.SecurityPageURL = v },
result: compliance.SecurityPageURL,
},
{
name: "trust_page_url",
get: func(p *coredata.CommonThirdParty) *string { return p.TrustPageURL },
set: func(p *coredata.CommonThirdParty, v *string) { p.TrustPageURL = v },
result: compliance.TrustPageURL,
},
}
}

View File

@@ -0,0 +1,456 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package thirdparty
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"time"
"go.gearno.de/kit/httpclient"
"go.gearno.de/kit/log"
"go.gearno.de/kit/pg"
"go.gearno.de/kit/worker"
"go.probo.inc/probo/pkg/agent"
"go.probo.inc/probo/pkg/agent/tools/browser"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/filemanager"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/llm"
)
const (
// defaultEnrichmentAgentTimeout caps a single enrichment agent run.
// It is generous because Agent B browses several pages on top of the
// LLM round-trips.
defaultEnrichmentAgentTimeout = 90 * time.Second
// defaultEnrichmentMaxTurns bounds an agent's reasoning loop (LLM
// call plus tool round-trips). Agent B may navigate the site footer
// and trust portal across several turns before synthesizing.
defaultEnrichmentMaxTurns = 12
// defaultEnrichmentMaxTokens caps agent output. The structured
// output is moderate; the budget leaves headroom for reasoning
// models whose reasoning tokens count against max_tokens.
defaultEnrichmentMaxTokens = 8192
// defaultEnrichmentConfidenceThreshold is the floor a resolved value
// must clear before it is written to its column. Values below it are
// recorded in the enrichment metadata but not promoted.
defaultEnrichmentConfidenceThreshold = 0.7
// defaultEnrichmentStaleAfter is the idle window after which a
// claimed-but-unfinished enrichment is re-armed.
defaultEnrichmentStaleAfter = 15 * time.Minute
// defaultEnrichmentMaxAttempts caps how many times a row is retried
// before stale recovery leaves it alone, so a permanently failing
// row does not loop forever.
defaultEnrichmentMaxAttempts = 3
enrichmentLogoUserAgent = "Probo-Enricher/1.0"
)
// EnrichmentConfig configures the common-third-party enrichment worker
// and the two agents it runs. The worker no-ops when LLMClient is nil;
// callers gate registration on config presence. Browser tools for Agent
// B are enabled only when ChromeAddr is set; otherwise it relies on
// web_search alone. Logo storage is enabled only when FileManager and
// Bucket are both set.
type EnrichmentConfig struct {
LLMClient *llm.Client
Model string
MaxTokens *int
Temperature *float64
FirecrawlAPIKey string
ChromeAddr string
AgentTimeout time.Duration
MaxTurns int
ConfidenceThreshold float64
StaleAfter time.Duration
MaxAttempts int
FileManager *filemanager.Service
Bucket string
}
func (c EnrichmentConfig) withDefaults() EnrichmentConfig {
if c.AgentTimeout <= 0 {
c.AgentTimeout = defaultEnrichmentAgentTimeout
}
if c.MaxTurns < 1 {
c.MaxTurns = defaultEnrichmentMaxTurns
}
if c.ConfidenceThreshold <= 0 {
c.ConfidenceThreshold = defaultEnrichmentConfidenceThreshold
}
if c.StaleAfter <= 0 {
c.StaleAfter = defaultEnrichmentStaleAfter
}
if c.MaxAttempts < 1 {
c.MaxAttempts = defaultEnrichmentMaxAttempts
}
return c
}
func resolveEnrichmentMaxTurns(configured int) int {
if configured > 0 {
return configured
}
return defaultEnrichmentMaxTurns
}
func resolveEnrichmentMaxTokens(configured *int) int {
if configured != nil && *configured > 0 {
return *configured
}
return defaultEnrichmentMaxTokens
}
type enrichmentHandler struct {
pg *pg.Client
logger *log.Logger
cfg EnrichmentConfig
companyAgent *agent.Agent
httpClient *http.Client
}
// NewCommonThirdPartyEnrichmentWorker builds the worker that enriches
// global common_third_parties rows. It is a system worker: the catalog
// is not tenant-scoped, so a single enrichment benefits all tenants. The
// worker no-ops when no LLM client is configured.
func NewCommonThirdPartyEnrichmentWorker(
pgClient *pg.Client,
logger *log.Logger,
cfg EnrichmentConfig,
opts ...worker.Option,
) *worker.Worker[coredata.CommonThirdParty] {
cfg = cfg.withDefaults()
h := &enrichmentHandler{
pg: pgClient,
logger: logger,
cfg: cfg,
httpClient: newEnrichmentHTTPClient(),
}
// Agent A has no browser, so it is built once and reused. Agent B is
// built per Process because it needs a per-run browser bound to the
// process context.
if cfg.LLMClient != nil {
h.companyAgent = buildCompanyProfileAgent(cfg, logger)
}
return worker.New(
"common-third-party-enrichment-worker",
h,
logger,
opts...,
)
}
func (h *enrichmentHandler) Claim(ctx context.Context) (coredata.CommonThirdParty, error) {
var party coredata.CommonThirdParty
if err := h.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
if err := party.LoadNextForEnrichmentForUpdateSkipLocked(ctx, tx); err != nil {
return err
}
return party.ClearEnrichmentRequestedAt(ctx, tx)
},
); err != nil {
if errors.Is(err, coredata.ErrResourceNotFound) {
return coredata.CommonThirdParty{}, worker.ErrNoTask
}
return coredata.CommonThirdParty{}, fmt.Errorf("cannot claim common third party enrichment task: %w", err)
}
return party, nil
}
// Process runs the enrichment pipeline for one catalog row: Agent A
// (company profile) first, then Agent B (compliance docs) and the
// deterministic logo step, all outside any transaction. The merged
// result and per-field provenance are persisted in a single final
// transaction. Process always writes an enrichment payload, even on a
// no-result run, so stale recovery does not re-queue the row.
func (h *enrichmentHandler) Process(ctx context.Context, party coredata.CommonThirdParty) error {
if h.companyAgent == nil {
return nil
}
now := time.Now()
prior := parseEnrichmentFields(party.Enrichment)
var (
runErrors []string
anySuccess bool
)
// Agent A first: it resolves website_url, which Agent B and the logo
// step depend on.
company, err := h.runCompanyProfile(ctx, party)
if err != nil {
h.logger.WarnCtx(ctx, "company profile agent failed", log.Error(err), log.String("common_third_party_id", party.ID.String()))
runErrors = append(runErrors, "company_profile: "+err.Error())
} else {
anySuccess = true
}
website := effectiveWebsiteURL(party, company, h.cfg.ConfidenceThreshold)
legalName := effectiveLegalName(party, company, h.cfg.ConfidenceThreshold)
// Agent B: compliance documents and trust pages.
compliance, err := h.runComplianceDocs(ctx, party.Name, website, legalName)
if err != nil {
h.logger.WarnCtx(ctx, "compliance docs agent failed", log.Error(err), log.String("common_third_party_id", party.ID.String()))
runErrors = append(runErrors, "compliance_docs: "+err.Error())
} else {
anySuccess = true
}
// Deterministic logo step (no LLM). Uploads to S3 outside the final
// transaction; the File row is inserted below.
logoFile := h.prepareLogo(ctx, party, website)
meta := make(map[string]EnrichmentFieldMeta)
for _, field := range scalarFields(company, compliance) {
applyScalarField(&party, meta, prior, field, h.cfg.ConfidenceThreshold, now)
}
applyCertifications(&party, meta, prior, compliance.Certifications, h.cfg.ConfidenceThreshold, now)
status := enrichmentStatusDone
switch {
case !anySuccess:
status = enrichmentStatusFailed
case len(runErrors) > 0:
status = enrichmentStatusPartial
}
payload := EnrichmentMetadata{
Model: h.cfg.Model,
AttemptedAt: now,
Status: status,
Error: strings.Join(runErrors, "; "),
Fields: meta,
}
raw, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("cannot marshal enrichment metadata: %w", err)
}
party.Enrichment = raw
party.UpdatedAt = now
return h.pg.WithTx(
ctx,
func(ctx context.Context, tx pg.Tx) error {
if logoFile != nil {
if err := logoFile.Insert(ctx, tx, coredata.NewScope(gid.NilTenant)); err != nil {
return fmt.Errorf("cannot insert common third party logo file: %w", err)
}
party.LogoFileID = &logoFile.ID
if err := party.UpdateLogoFileID(ctx, tx); err != nil {
return fmt.Errorf("cannot update common third party logo: %w", err)
}
}
if err := party.UpdateEnrichment(ctx, tx); err != nil {
return fmt.Errorf("cannot persist common third party enrichment: %w", err)
}
h.logger.InfoCtx(
ctx,
"enriched common third party",
log.String("common_third_party_id", party.ID.String()),
log.String("name", party.Name),
log.String("status", status),
log.Bool("logo_stored", logoFile != nil),
)
return nil
},
)
}
// RecoverStale re-arms enrichment for rows whose run was claimed but
// never finished. Claim clears enrichment_requested_at up front, so a
// crash between phases would otherwise strand the row.
func (h *enrichmentHandler) RecoverStale(ctx context.Context) error {
return h.pg.WithConn(
ctx,
func(ctx context.Context, conn pg.Querier) error {
if err := coredata.ResetStaleCommonThirdPartyEnrichments(ctx, conn, h.cfg.StaleAfter, h.cfg.MaxAttempts); err != nil {
return fmt.Errorf("cannot reset stale common third party enrichments: %w", err)
}
return nil
},
)
}
func (h *enrichmentHandler) runCompanyProfile(
ctx context.Context,
party coredata.CommonThirdParty,
) (CompanyProfileResult, error) {
prompt := buildCompanyProfilePrompt(party)
agentCtx, cancel := context.WithTimeout(ctx, h.cfg.AgentTimeout)
defer cancel()
result, err := agent.RunTyped[CompanyProfileResult](
agentCtx,
h.companyAgent,
[]llm.Message{
{
Role: llm.RoleUser,
Parts: []llm.Part{llm.TextPart{Text: prompt}},
},
},
)
if err != nil {
return CompanyProfileResult{}, fmt.Errorf("company profile agent run failed: %w", err)
}
return result.Output, nil
}
// runComplianceDocs builds Agent B with a per-run browser when a Chrome
// endpoint is configured, then runs it. The browser is closed when the
// run returns. The browser is intentionally not pinned to the vendor
// domain so the agent can follow links to hosted trust portals (Vanta,
// SafeBase, etc.); SSRF protection still blocks non-public hosts.
func (h *enrichmentHandler) runComplianceDocs(
ctx context.Context,
name string,
website string,
legalName string,
) (ComplianceDocsResult, error) {
var browserTools []agent.Tool
if h.cfg.ChromeAddr != "" {
webBrowser := browser.NewBrowser(ctx, h.cfg.ChromeAddr)
defer webBrowser.Close()
browserTools = browser.NewReadOnlyToolset(webBrowser).Tools()
}
complianceAgent := buildComplianceDocsAgent(h.cfg, h.logger, browserTools)
prompt := buildComplianceDocsPrompt(name, website, legalName)
agentCtx, cancel := context.WithTimeout(ctx, h.cfg.AgentTimeout)
defer cancel()
result, err := agent.RunTyped[ComplianceDocsResult](
agentCtx,
complianceAgent,
[]llm.Message{
{
Role: llm.RoleUser,
Parts: []llm.Part{llm.TextPart{Text: prompt}},
},
},
)
if err != nil {
return ComplianceDocsResult{}, fmt.Errorf("compliance docs agent run failed: %w", err)
}
return result.Output, nil
}
// effectiveWebsiteURL is the website passed to Agent B and the logo step.
// A curated value already on the row wins (seed data and human edits are
// trusted); otherwise Agent A's value is used when it clears the
// confidence threshold.
func effectiveWebsiteURL(
party coredata.CommonThirdParty,
company CompanyProfileResult,
threshold float64,
) string {
if party.WebsiteURL != nil {
if v := strings.TrimSpace(*party.WebsiteURL); v != "" {
return v
}
}
if v := strings.TrimSpace(company.WebsiteURL.Value); v != "" && company.WebsiteURL.Confidence >= threshold {
return v
}
return ""
}
// effectiveLegalName is the legal name hint passed to Agent B, resolved
// the same way as effectiveWebsiteURL.
func effectiveLegalName(
party coredata.CommonThirdParty,
company CompanyProfileResult,
threshold float64,
) string {
if party.LegalName != nil {
if v := strings.TrimSpace(*party.LegalName); v != "" {
return v
}
}
if v := strings.TrimSpace(company.LegalName.Value); v != "" && company.LegalName.Confidence >= threshold {
return v
}
return ""
}
type userAgentRoundTripper struct {
next http.RoundTripper
}
func (t *userAgentRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
r2 := r.Clone(r.Context())
r2.Header.Set("User-Agent", enrichmentLogoUserAgent)
return t.next.RoundTrip(r2)
}
// newEnrichmentHTTPClient builds the SSRF-protected client used by the
// deterministic logo step.
func newEnrichmentHTTPClient() *http.Client {
client := httpclient.DefaultPooledClient(httpclient.WithSSRFProtection())
client.Timeout = 20 * time.Second
client.Transport = &userAgentRoundTripper{next: client.Transport}
return client
}

View File

@@ -0,0 +1,207 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package thirdparty
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"go.gearno.de/crypto/uuid"
"go.gearno.de/kit/log"
"go.probo.inc/probo/pkg/coredata"
"go.probo.inc/probo/pkg/gid"
"go.probo.inc/probo/pkg/webinspect"
)
// maxLogoSize caps a downloaded logo image. Logos are small; the cap
// guards against an oversized or malicious response.
const maxLogoSize = 5 << 20 // 5 MiB
// prepareLogo discovers and uploads a vendor logo to S3, returning a
// fully-populated (but not yet inserted) File record for the caller to
// persist in its transaction. It is deterministic and best-effort: any
// failure logs and returns (nil, nil) so a missing logo never fails the
// enrichment run.
//
// It no-ops when the row already has a logo, when logo storage is not
// configured, or when no website is known. The S3 upload happens here,
// outside any transaction; the caller inserts the File row and links it
// via UpdateLogoFileID.
func (h *enrichmentHandler) prepareLogo(
ctx context.Context,
party coredata.CommonThirdParty,
websiteURL string,
) *coredata.File {
if party.LogoFileID != nil {
return nil
}
if h.cfg.FileManager == nil || h.cfg.Bucket == "" {
return nil
}
website := strings.TrimSpace(websiteURL)
if website == "" {
return nil
}
data, contentType, err := fetchCommonThirdPartyLogo(ctx, h.httpClient, website)
if err != nil {
h.logger.InfoCtx(
ctx,
"could not fetch common third party logo",
log.String("common_third_party_id", party.ID.String()),
log.Error(err),
)
return nil
}
objectKey, err := uuid.NewV7()
if err != nil {
h.logger.WarnCtx(ctx, "cannot generate logo object key", log.Error(err))
return nil
}
now := time.Now()
fileRecord := &coredata.File{
ID: gid.New(gid.NilTenant, coredata.FileEntityType),
OrganizationID: gid.Nil,
BucketName: h.cfg.Bucket,
MimeType: contentType,
FileName: party.Name + "-logo" + webinspect.ExtensionForMIME(contentType),
FileKey: objectKey.String(),
FileSize: int64(len(data)),
Visibility: coredata.FileVisibilityPublic,
CreatedAt: now,
UpdatedAt: now,
}
size, err := h.cfg.FileManager.PutFile(
ctx,
fileRecord,
bytes.NewReader(data),
map[string]string{
"type": "common-third-party-logo",
"common-third-party-id": party.ID.String(),
},
)
if err != nil {
h.logger.WarnCtx(
ctx,
"cannot upload common third party logo",
log.String("common_third_party_id", party.ID.String()),
log.Error(err),
)
return nil
}
fileRecord.FileSize = size
return fileRecord
}
// fetchCommonThirdPartyLogo finds the best logo for a website and
// downloads it. It first parses the page's <head> for icon links
// (webinspect), then falls back to well-known icon paths on the same
// host. The supplied client must enforce SSRF protection.
func fetchCommonThirdPartyLogo(
ctx context.Context,
client *http.Client,
websiteURL string,
) (data []byte, contentType string, err error) {
candidates := make([]string, 0, 3)
pageInfo, parseErr := webinspect.Parse(ctx, client, websiteURL)
if parseErr == nil {
if logoURL, logoErr := webinspect.FindLogoURL(pageInfo); logoErr == nil {
candidates = append(candidates, logoURL)
}
}
if parsed, parseURLErr := url.Parse(websiteURL); parseURLErr == nil && parsed.Host != "" {
base := url.URL{Scheme: parsed.Scheme, Host: parsed.Host}
if base.Scheme == "" {
base.Scheme = "https"
}
candidates = append(
candidates,
base.ResolveReference(&url.URL{Path: "/apple-touch-icon.png"}).String(),
base.ResolveReference(&url.URL{Path: "/favicon.ico"}).String(),
)
}
for _, candidate := range candidates {
data, contentType, err = downloadImage(ctx, client, candidate)
if err == nil {
return data, contentType, nil
}
}
return nil, "", fmt.Errorf("cannot fetch logo for %s", websiteURL)
}
// downloadImage fetches a single candidate URL and returns its bytes when
// the response is a non-empty image within the size cap.
func downloadImage(
ctx context.Context,
client *http.Client,
rawURL string,
) ([]byte, string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return nil, "", fmt.Errorf("cannot create logo request: %w", err)
}
resp, err := client.Do(req)
if err != nil {
return nil, "", fmt.Errorf("cannot fetch logo: %w", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return nil, "", fmt.Errorf("cannot fetch logo: status %d", resp.StatusCode)
}
contentType := resp.Header.Get("Content-Type")
if idx := strings.Index(contentType, ";"); idx != -1 {
contentType = contentType[:idx]
}
contentType = strings.TrimSpace(contentType)
if !strings.HasPrefix(contentType, "image/") {
return nil, "", fmt.Errorf("logo response is not an image: %q", contentType)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxLogoSize))
if err != nil {
return nil, "", fmt.Errorf("cannot read logo body: %w", err)
}
if len(body) == 0 {
return nil, "", fmt.Errorf("logo response is empty")
}
return body, contentType, nil
}

View File

@@ -0,0 +1,33 @@
You are a research agent that builds a factual company profile for a software
vendor or service provider. You are given the vendor's name (and sometimes a
known website). Return only verifiable identity facts.
Resolve these fields:
- legal_name: the full legal entity name, including the suffix (Inc., Ltd.,
GmbH, S.A.S., etc.). Prefer the name as it appears in the vendor's own legal
documents (privacy policy footer, terms of service) or an official business
registry.
- headquarter_address: the postal address of the company's headquarters.
- website_url: the canonical primary marketing website. Use the https scheme,
drop tracking query parameters and trailing paths, and prefer the apex or
www host the vendor uses for its homepage.
Method:
- Use the web_search tool when it is available to confirm facts. Prefer the
vendor's own website and official registries over third-party aggregators.
- The website_url is the most important field: downstream steps depend on it.
Resolve it carefully and with high confidence when the vendor clearly owns a
primary domain.
Rules:
- Never guess. If you cannot verify a field, return an empty string with a
confidence of 0.
- confidence is your own 0.0-1.0 estimate that the value is correct. Reserve
values above 0.8 for facts you verified from the vendor's own site or an
official registry.
- source_url is the page where you verified the value. Leave it empty when the
value was not found.
- Do not include commentary; return only the structured fields.

View File

@@ -0,0 +1,51 @@
You are a research agent that locates a software vendor's public compliance
documents and trust pages. You are given the vendor's name and, usually, its
website. Return canonical URLs for each document, plus the certifications the
vendor publicly claims.
Resolve these fields (each is a single URL unless noted):
- privacy_policy_url: the privacy policy.
- terms_of_service_url: the terms of service / terms of use.
- service_level_agreement_url: the public SLA. Frequently gated behind sales.
- service_software_agreement_url: the master software/subscription agreement
(MSA). Often gated, or the same document as the terms of service.
- data_processing_agreement_url: the DPA. Often a downloadable PDF; sometimes
only available on request.
- business_associate_agreement_url: the HIPAA BAA. Almost always gated behind
sales or an enterprise plan.
- subprocessors_list_url: the sub-processors list page.
- status_page_url: the uptime/status page (commonly status.<domain> or a
hosted statuspage.io / instatus / better-uptime page).
- security_page_url: the security overview page.
- trust_page_url: the trust center / trust portal. Many vendors host this on
Vanta, SafeBase, Drata, Conveyor, or a /trust path.
- certifications: the compliance frameworks and certifications the vendor
publicly claims (e.g. SOC 2 Type II, ISO 27001, ISO 27701, PCI DSS, HIPAA,
GDPR, FedRAMP). Read these from the trust or security page.
Method:
- Start from the vendor's <website> when provided. Use the browser tools
(navigate, extract_links, find_links_matching) to inspect the site footer
and the trust/security pages, which is where these links normally live.
- Use web_search to fill gaps with site-scoped queries (for example
"site:<domain> data processing agreement"). Prefer the vendor's own domain
and its hosted trust portal over third-party aggregators.
- Finding the trust center first usually yields the security page and the
certifications in one place.
Rules:
- Return the most specific canonical URL. Prefer a direct document/page URL
over a generic legal-index page.
- Never guess or fabricate a URL. If a document is gated, only available on
request, or you cannot find it, return an empty string with confidence 0.
Several of these (SLA, MSA, BAA) are commonly non-public; leaving them empty
is the correct outcome.
- confidence is your own 0.0-1.0 estimate that the URL is correct and current.
Reserve values above 0.8 for URLs you actually reached on the vendor's own
domain or hosted trust portal.
- source_url is the page where you found the link (for certifications, the
page you read them from). Leave it empty when nothing was found.
- Do not include commentary; return only the structured fields.

View File

@@ -70,8 +70,14 @@ func ResolveOrCreateCommonThirdParty(
Slug: partySlug,
Category: category,
Certifications: []string{},
CreatedAt: now,
UpdatedAt: now,
// Request enrichment at creation: a freshly resolved catalog row
// carries only name/slug/category, so the enrichment worker fills
// the rest (URLs, address, certifications, logo). Curated seed
// rows are inserted via Upsert without this flag, so a full
// re-seed does not trigger an enrichment storm.
EnrichmentRequestedAt: &now,
CreatedAt: now,
UpdatedAt: now,
}
// Insert inside a savepoint so a concurrent transaction that created

180
pkg/webinspect/logo.go Normal file
View File

@@ -0,0 +1,180 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package webinspect
import (
"fmt"
"strconv"
"strings"
"golang.org/x/net/html"
)
func FindLogoURL(info *PageInfo) (string, error) {
head := findElement(info.Root, "head")
if head == nil {
return "", fmt.Errorf("cannot find logo: no head element")
}
var (
svgIcon string
appleTouchIcon string
appleTouchSize int
largestIcon string
largestSize int
msTileImage string
)
for _, n := range findAllIn(head, "link") {
rel := strings.ToLower(attrVal(n, "rel"))
href := attrVal(n, "href")
if href == "" {
continue
}
switch {
case strings.Contains(rel, "icon") && !strings.Contains(rel, "apple-touch-icon") && attrVal(n, "type") == "image/svg+xml":
svgIcon = href
case strings.Contains(rel, "apple-touch-icon"):
size := parseSizeAttr(attrVal(n, "sizes"))
if appleTouchIcon == "" || size > appleTouchSize {
appleTouchIcon = href
appleTouchSize = size
}
case strings.Contains(rel, "icon") && !strings.Contains(rel, "apple-touch-icon"):
size := parseSizeAttr(attrVal(n, "sizes"))
if largestIcon == "" || size > largestSize {
largestIcon = href
largestSize = size
}
}
}
for _, n := range findAllIn(head, "meta") {
name := strings.ToLower(attrVal(n, "name"))
content := attrVal(n, "content")
if name == "msapplication-tileimage" && content != "" {
msTileImage = content
}
}
candidates := []string{
svgIcon,
appleTouchIcon,
largestIcon,
msTileImage,
}
for _, href := range candidates {
if href != "" {
return info.ResolveHref(href), nil
}
}
return "", fmt.Errorf("cannot find logo")
}
func parseSizeAttr(sizes string) int {
if sizes == "" || strings.EqualFold(sizes, "any") {
return 0
}
best := 0
for token := range strings.FieldsSeq(sizes) {
token = strings.ToLower(token)
parts := strings.SplitN(token, "x", 2)
if len(parts) != 2 {
continue
}
w, err := strconv.Atoi(parts[0])
if err != nil {
continue
}
if w > best {
best = w
}
}
return best
}
func ExtensionForMIME(contentType string) string {
ct := strings.ToLower(contentType)
if idx := strings.Index(ct, ";"); idx != -1 {
ct = ct[:idx]
}
ct = strings.TrimSpace(ct)
switch ct {
case "image/svg+xml":
return ".svg"
case "image/png":
return ".png"
case "image/jpeg":
return ".jpg"
case "image/gif":
return ".gif"
case "image/webp":
return ".webp"
case "image/x-icon", "image/vnd.microsoft.icon":
return ".ico"
default:
return ".png"
}
}
// HeadLinks returns all <link> nodes from <head> whose rel attribute
// contains the given value (case-insensitive partial match).
func (p *PageInfo) HeadLinks(rel string) []*html.Node {
head := findElement(p.Root, "head")
if head == nil {
return nil
}
rel = strings.ToLower(rel)
var matches []*html.Node
for _, n := range findAllIn(head, "link") {
if strings.Contains(strings.ToLower(attrVal(n, "rel")), rel) {
matches = append(matches, n)
}
}
return matches
}
// HeadMeta returns the content attribute of the first <meta> tag in <head>
// whose name attribute matches (case-insensitive).
func (p *PageInfo) HeadMeta(name string) (string, bool) {
head := findElement(p.Root, "head")
if head == nil {
return "", false
}
name = strings.ToLower(name)
for _, n := range findAllIn(head, "meta") {
if strings.EqualFold(attrVal(n, "name"), name) {
content := attrVal(n, "content")
if content != "" {
return content, true
}
}
}
return "", false
}

163
pkg/webinspect/logo_test.go Normal file
View File

@@ -0,0 +1,163 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package webinspect_test
import (
"net/url"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.probo.inc/probo/pkg/webinspect"
)
func parseTestHTML(t *testing.T, rawURL string, body string) *webinspect.PageInfo {
t.Helper()
u, err := url.Parse(rawURL)
require.NoError(t, err)
info, err := webinspect.ParseHTML(u, strings.NewReader(body))
require.NoError(t, err)
return info
}
func TestFindLogoURL_SVGPreferred(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://example.com", `<html><head>
<link rel="icon" type="image/svg+xml" href="/favicon.svg">
<link rel="apple-touch-icon" href="/apple-touch-icon.png">
<link rel="icon" sizes="192x192" href="/icon-192.png">
</head><body></body></html>`)
got, err := webinspect.FindLogoURL(info)
require.NoError(t, err)
assert.Equal(t, "https://example.com/favicon.svg", got)
}
func TestFindLogoURL_AppleTouchIconSecond(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://example.com", `<html><head>
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
<link rel="icon" sizes="32x32" href="/favicon-32.png">
</head><body></body></html>`)
got, err := webinspect.FindLogoURL(info)
require.NoError(t, err)
assert.Equal(t, "https://example.com/apple-touch-icon.png", got)
}
func TestFindLogoURL_AppleTouchIconLargest(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://example.com", `<html><head>
<link rel="apple-touch-icon" sizes="120x120" href="/touch-120.png">
<link rel="apple-touch-icon" sizes="180x180" href="/touch-180.png">
<link rel="apple-touch-icon" sizes="152x152" href="/touch-152.png">
</head><body></body></html>`)
got, err := webinspect.FindLogoURL(info)
require.NoError(t, err)
assert.Equal(t, "https://example.com/touch-180.png", got)
}
func TestFindLogoURL_LargestIcon(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://example.com", `<html><head>
<link rel="icon" sizes="32x32" href="/icon-32.png">
<link rel="icon" sizes="192x192" href="/icon-192.png">
<link rel="icon" sizes="16x16" href="/icon-16.png">
</head><body></body></html>`)
got, err := webinspect.FindLogoURL(info)
require.NoError(t, err)
assert.Equal(t, "https://example.com/icon-192.png", got)
}
func TestFindLogoURL_MsTileImage(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://example.com", `<html><head>
<meta name="msapplication-TileImage" content="/mstile-144.png">
</head><body></body></html>`)
got, err := webinspect.FindLogoURL(info)
require.NoError(t, err)
assert.Equal(t, "https://example.com/mstile-144.png", got)
}
func TestFindLogoURL_RelativeHrefResolved(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://cdn.example.com/app/", `<html><head>
<link rel="icon" type="image/svg+xml" href="../assets/logo.svg">
</head><body></body></html>`)
got, err := webinspect.FindLogoURL(info)
require.NoError(t, err)
assert.Equal(t, "https://cdn.example.com/assets/logo.svg", got)
}
func TestFindLogoURL_NoHead(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://example.com", `<html><body><p>no head</p></body></html>`)
_, err := webinspect.FindLogoURL(info)
assert.Error(t, err)
}
func TestFindLogoURL_NothingFound(t *testing.T) {
t.Parallel()
info := parseTestHTML(t, "https://example.com", `<html><head></head><body></body></html>`)
_, err := webinspect.FindLogoURL(info)
assert.Error(t, err)
}
func TestExtensionForMIME(t *testing.T) {
t.Parallel()
tests := []struct {
contentType string
expected string
}{
{"image/svg+xml", ".svg"},
{"image/png", ".png"},
{"image/jpeg", ".jpg"},
{"image/gif", ".gif"},
{"image/webp", ".webp"},
{"image/x-icon", ".ico"},
{"image/vnd.microsoft.icon", ".ico"},
{"image/png; charset=utf-8", ".png"},
{"application/octet-stream", ".png"},
}
for _, tt := range tests {
t.Run(
tt.contentType,
func(t *testing.T) {
t.Parallel()
assert.Equal(t, tt.expected, webinspect.ExtensionForMIME(tt.contentType))
},
)
}
}

117
pkg/webinspect/parse.go Normal file
View File

@@ -0,0 +1,117 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
// Package webinspect parses a web page's static HTML to extract metadata
// from its <head> (icons/logos, meta tags, link relations). It is a pure,
// deterministic helper: callers supply an http.Client (e.g. one with SSRF
// protection) so the package itself makes no policy decisions about which
// hosts are reachable. The logo discovery is reused by the common
// third-party enricher to populate logo_file_id without an LLM.
package webinspect
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"golang.org/x/net/html"
)
type PageInfo struct {
URL *url.URL
Root *html.Node
}
func Parse(ctx context.Context, client *http.Client, websiteURL string) (*PageInfo, error) {
parsed, err := url.Parse(websiteURL)
if err != nil {
return nil, fmt.Errorf("cannot parse website URL: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, websiteURL, nil)
if err != nil {
return nil, fmt.Errorf("cannot create request: %w", err)
}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("cannot fetch page: %w", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("cannot fetch page: status %d", resp.StatusCode)
}
const maxHTMLSize = 10 << 20 // 10 MiB
return ParseHTML(parsed, io.LimitReader(resp.Body, maxHTMLSize))
}
func ParseHTML(baseURL *url.URL, r io.Reader) (*PageInfo, error) {
root, err := html.Parse(r)
if err != nil {
return nil, fmt.Errorf("cannot parse HTML: %w", err)
}
return &PageInfo{URL: baseURL, Root: root}, nil
}
func (p *PageInfo) ResolveHref(href string) string {
ref, err := url.Parse(href)
if err != nil {
return href
}
return p.URL.ResolveReference(ref).String()
}
func findElement(n *html.Node, tag string) *html.Node {
if n.Type == html.ElementNode && n.Data == tag {
return n
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
if found := findElement(c, tag); found != nil {
return found
}
}
return nil
}
func findAllIn(parent *html.Node, tag string) []*html.Node {
var nodes []*html.Node
for c := parent.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.ElementNode && c.Data == tag {
nodes = append(nodes, c)
}
nodes = append(nodes, findAllIn(c, tag)...)
}
return nodes
}
func attrVal(n *html.Node, key string) string {
for _, a := range n.Attr {
if a.Key == key {
return a.Val
}
}
return ""
}