diff --git a/.cursor/rules/prompt-style.mdc b/.cursor/rules/prompt-style.mdc new file mode 100644 index 000000000..c4a1db90b --- /dev/null +++ b/.cursor/rules/prompt-style.mdc @@ -0,0 +1,44 @@ +--- +description: Agent prompt template structure (role/task/instructions XML style) +globs: "**/prompts/**/*.tmpl" +alwaysApply: false +--- + +# Agent prompt style + +See full guide: `contrib/claude/prompt-style.md` + +Agent prompt templates use an XML-tag structure with three top-level sections, +in this order: + +``` + +One short paragraph: who the agent is and its single objective. + + + +What the agent is given and the fields it must return. Describe the structured +output here as a bullet list (one bullet per field). + + + +1. A numbered list of directives, most important first. +2. ... + +``` + +## Rules + +- Always use the three sections ``, ``, `` in that order. +- `` is an ordered (numbered) list; put the most decisive rule first. +- Keep `` to one short paragraph stating the objective. +- Describe every output field in `` as a bullet, mirroring the typed result struct. +- Never hardcode enum values or source-of-truth lists; use a `{{.Placeholder}}` + and substitute at runtime (see `template-files.mdc`). +- No commentary outside the tags; the prompt body is the system instruction. + +## Examples + +- `pkg/thirdparty/prompts/disambiguation.txt.tmpl` +- `pkg/cookiebanner/prompts/tracker_identification.txt.tmpl` +- `pkg/thirdparty/prompts/common_third_party_company_profile.txt.tmpl` diff --git a/AGENTS.md b/AGENTS.md index ffcad1f57..229da17aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,8 @@ Detailed guides for specific subsystems live in `contrib/claude/`: - [`contrib/claude/ui.md`](contrib/claude/ui.md) — @probo/ui, Tailwind, tailwind-variants, folders, skeletons, compound components - [`contrib/claude/config.md`](contrib/claude/config.md) — Configuration propagation (all files to update when config changes) - [`contrib/claude/file-naming.md`](contrib/claude/file-naming.md) — File naming conventions (template files, extensions) +- [`contrib/claude/prompt-style.md`](contrib/claude/prompt-style.md) — Agent prompt template structure (role/task/instructions XML style) +- [`contrib/claude/prompt-style.md`](contrib/claude/prompt-style.md) — Agent prompt template structure (role/task/instructions XML style) - [`contrib/claude/commit.md`](contrib/claude/commit.md) — Commit message conventions - [`contrib/claude/license.md`](contrib/claude/license.md) — ISC license header (all file types) - [`contrib/claude/release.md`](contrib/claude/release.md) — Release process (version bump, changelog, tag, push) diff --git a/contrib/claude/prompt-style.md b/contrib/claude/prompt-style.md new file mode 100644 index 000000000..26740e193 --- /dev/null +++ b/contrib/claude/prompt-style.md @@ -0,0 +1,60 @@ +# Agent prompt style + +Agent prompt templates (the `//go:embed`-ed `*.txt.tmpl` files that back an +agent's instructions) follow a consistent XML-tag structure. This keeps prompts +scannable, makes the contract between the prompt and the typed result struct +explicit, and matches what the models in use respond to best. + +## Structure + +Three top-level sections, always in this order: + +``` + +One short paragraph: who the agent is and its single objective. + + + +What the agent is given and the fields it must return. Describe the structured +output as a bullet list, one bullet per field, mirroring the typed result. + + + +1. A numbered list of directives, ordered most-decisive first. +2. Each rule is a single, actionable directive. +3. ... + +``` + +## Rules + +- Use ``, ``, `` in that order. Do not invent other + top-level sections (no loose `Method:` / `Rules:` prose blocks). +- `` is one short paragraph stating who the agent is and its objective. +- `` enumerates the inputs and the output fields. List one bullet per + field so the prompt stays in sync with the typed result struct. +- `` is an ordered (numbered) list. Put the most important rule + first. Keep each item to a single directive. +- Never hardcode enum values or source-of-truth lists. Use a `{{.Placeholder}}` + and substitute at runtime. See [`file-naming.md`](file-naming.md) and the + `template-files` cursor rule. +- No commentary outside the tags; the whole body is the system instruction. + +## Per-row input + +The embedded template is the static system instruction. The dynamic per-row +input (the specific entity being processed) is built separately in Go and sent +as the user message, wrapped in its own descriptive tags: + +```go +fmt.Fprintf(&b, " %s \n", party.Name) +fmt.Fprintf(&b, " %s \n", website) +``` + +## Examples + +- `pkg/thirdparty/prompts/disambiguation.txt.tmpl` +- `pkg/cookiebanner/prompts/tracker_identification.txt.tmpl` +- `pkg/cookiebanner/prompts/tracker_enrichment.txt.tmpl` +- `pkg/thirdparty/prompts/common_third_party_company_profile.txt.tmpl` +- `pkg/thirdparty/prompts/common_third_party_compliance_docs.txt.tmpl` diff --git a/pkg/agent/run.go b/pkg/agent/run.go index d4e90ba2a..0d32d3be1 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -401,6 +401,12 @@ func coreLoop(ctx context.Context, startAgent *Agent, inputMessages []llm.Messag emptyOutputRetries := 0 + // forcedSynthesis records that the turn budget was exhausted while + // the agent was still exploring with a pending structured output, so + // the last turn was spent forcing the schema rather than failing + // outright. It guards against looping past the cap more than once. + forcedSynthesis := false + structuredFormat := resolveStructuredFormat(s.agent) // When the agent has both tools and a structured output request, @@ -440,7 +446,33 @@ func coreLoop(ctx context.Context, startAgent *Agent, inputMessages []llm.Messag } if s.turns >= s.agent.maxTurns { - return s.finishRun(ctx, nil, &MaxTurnsExceededError{MaxTurns: s.agent.maxTurns}) + // The turn budget is exhausted. If the agent is still + // exploring with tools and owes a structured output, spend + // one final turn forcing the schema (ToolChoice=none) so it + // emits the best answer it can from what it has gathered, + // rather than failing with nothing. This runs at most once; + // the next iteration falls through to the error below. + if !exploring || forcedSynthesis || structuredFormat == nil || len(s.toolDefs) == 0 { + return s.finishRun(ctx, nil, &MaxTurnsExceededError{MaxTurns: s.agent.maxTurns}) + } + + forcedSynthesis = true + exploring = false + + s.messages = append( + s.messages, + llm.Message{ + Role: llm.RoleUser, + Parts: []llm.Part{llm.TextPart{Text: synthesisNudge}}, + }, + ) + + s.logger.WarnCtx( + ctx, + "max turns reached while exploring: forcing final synthesis turn", + log.Int("turn", s.turns), + log.Int("max_turns", s.agent.maxTurns), + ) } fullMessages := buildFullMessages(s.systemPrompt, s.messages) @@ -533,7 +565,7 @@ func coreLoop(ctx context.Context, startAgent *Agent, inputMessages []llm.Messag Parts: []llm.Part{llm.TextPart{Text: synthesisNudge}}, }, ) - s.logger.WarnCtx( + s.logger.DebugCtx( ctx, "entering synthesis turn: forcing structured output with tool_choice=none", log.Int("turn", s.turns), diff --git a/pkg/agent/tools/browser/find_links.go b/pkg/agent/tools/browser/find_links.go index f43fb860d..26d4613cd 100644 --- a/pkg/agent/tools/browser/find_links.go +++ b/pkg/agent/tools/browser/find_links.go @@ -62,7 +62,7 @@ func FindLinksMatchingTool(b *Browser) agent.Tool { js := fmt.Sprintf( `(() => { - const pattern = JSON.parse(%s).toLowerCase(); + const pattern = (%s).toLowerCase(); const normalize = s => s.replace(/[-_\s]+/g, ""); const normalizedPattern = normalize(pattern); return Array.from(document.querySelectorAll("a[href]")) diff --git a/pkg/thirdparty/common_third_party_company_profile_agent.go b/pkg/thirdparty/common_third_party_company_profile_agent.go index a04dc4f17..71a9f6606 100644 --- a/pkg/thirdparty/common_third_party_company_profile_agent.go +++ b/pkg/thirdparty/common_third_party_company_profile_agent.go @@ -38,11 +38,17 @@ type CompanyProfileResult struct { WebsiteURL EnrichedField `json:"website_url" jsonschema:"The vendor's canonical primary marketing website URL (https scheme, no tracking query parameters, no trailing path)."` } +// buildCompanyProfileAgent builds Agent A. extraTools carries the +// browser read-only toolset when a headless Chrome endpoint is +// configured; it is empty otherwise, in which case the agent relies on +// web_search alone. The browser lets it read footer/imprint/about/legal +// pages where the legal name and headquarters address live. func buildCompanyProfileAgent( cfg EnrichmentConfig, logger *log.Logger, + extraTools []agent.Tool, ) *agent.Agent { - var tools []agent.Tool + tools := append([]agent.Tool{}, extraTools...) if cfg.FirecrawlAPIKey != "" { tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlAPIKey)) diff --git a/pkg/thirdparty/common_third_party_enrichment.go b/pkg/thirdparty/common_third_party_enrichment.go index 412187812..3ad6b629e 100644 --- a/pkg/thirdparty/common_third_party_enrichment.go +++ b/pkg/thirdparty/common_third_party_enrichment.go @@ -30,10 +30,11 @@ const ( enrichmentSourceEnrichment = "enrichment" enrichmentSourceExternal = "external" - enrichmentFieldStatusFound = "found" - enrichmentFieldStatusNotFound = "not_found" - enrichmentFieldStatusLowConfidence = "low_confidence" - enrichmentFieldStatusExternal = "exists_external" + enrichmentFieldStatusFound = "found" + enrichmentFieldStatusNotFound = "not_found" + enrichmentFieldStatusLowConfidence = "low_confidence" + enrichmentFieldStatusExternal = "exists_external" + enrichmentFieldStatusFallbackDisplayName = "fallback_display_name" // Run-level status recorded at the top of the enrichment payload. enrichmentStatusDone = "done" @@ -230,6 +231,35 @@ func applyCertifications( } } +// applyLegalNameFallback fills legal_name with the catalog display name +// when the merge left the column empty, so it is never blank. A real +// value (agent-resolved or external) already on the row is left +// untouched. The fallback is recorded as enrichment-owned so a later run +// that resolves the real legal entity overwrites it. +func applyLegalNameFallback( + party *coredata.CommonThirdParty, + meta map[string]EnrichmentFieldMeta, + now time.Time, +) { + const name = "legal_name" + + if party.LegalName != nil && strings.TrimSpace(*party.LegalName) != "" { + return + } + + displayName := strings.TrimSpace(party.Name) + if displayName == "" { + return + } + + party.LegalName = &displayName + meta[name] = EnrichmentFieldMeta{ + Status: enrichmentFieldStatusFallbackDisplayName, + Source: enrichmentSourceEnrichment, + UpdatedAt: now, + } +} + // normalizeCertifications trims, drops blanks, and de-duplicates the // certification names returned by the agent, preserving order. func normalizeCertifications(values []string) []string { diff --git a/pkg/thirdparty/common_third_party_enrichment_worker.go b/pkg/thirdparty/common_third_party_enrichment_worker.go index 498313559..42184d5dd 100644 --- a/pkg/thirdparty/common_third_party_enrichment_worker.go +++ b/pkg/thirdparty/common_third_party_enrichment_worker.go @@ -132,11 +132,10 @@ func resolveEnrichmentMaxTokens(configured *int) int { } type enrichmentHandler struct { - pg *pg.Client - logger *log.Logger - cfg EnrichmentConfig - companyAgent *agent.Agent - httpClient *http.Client + pg *pg.Client + logger *log.Logger + cfg EnrichmentConfig + httpClient *http.Client } // NewCommonThirdPartyEnrichmentWorker builds the worker that enriches @@ -158,13 +157,6 @@ func NewCommonThirdPartyEnrichmentWorker( httpClient: newEnrichmentHTTPClient(), } - // Agent A has no browser, so it is built once and reused. Agent B is - // built per Process because it needs a per-run browser bound to the - // process context. - if cfg.LLMClient != nil { - h.companyAgent = buildCompanyProfileAgent(cfg, logger) - } - return worker.New( "common-third-party-enrichment-worker", h, @@ -203,7 +195,7 @@ func (h *enrichmentHandler) Claim(ctx context.Context) (coredata.CommonThirdPart // transaction. Process always writes an enrichment payload, even on a // no-result run, so stale recovery does not re-queue the row. func (h *enrichmentHandler) Process(ctx context.Context, party coredata.CommonThirdParty) error { - if h.companyAgent == nil { + if h.cfg.LLMClient == nil { return nil } @@ -226,6 +218,33 @@ func (h *enrichmentHandler) Process(ctx context.Context, party coredata.CommonTh } website := effectiveWebsiteURL(party, company, h.cfg.ConfidenceThreshold) + + meta := make(map[string]EnrichmentFieldMeta) + + // Website is the hard precondition: Agent B and the logo step both + // depend on it, and an Agent B run without a domain scope produces + // inconsistent cross-domain results. When no website is resolved, + // persist what Agent A found and stop here rather than running Agent + // B blind. + if website == "" { + for _, field := range scalarFields(company, ComplianceDocsResult{}) { + applyScalarField(&party, meta, prior, field, h.cfg.ConfidenceThreshold, now) + } + + applyCertifications(&party, meta, prior, CertificationsField{}, h.cfg.ConfidenceThreshold, now) + applyLegalNameFallback(&party, meta, now) + + runErrors = append(runErrors, "website_url unresolved: skipped compliance docs") + + return h.persist(ctx, party, EnrichmentMetadata{ + Model: h.cfg.Model, + AttemptedAt: now, + Status: enrichmentStatusFailed, + Error: strings.Join(runErrors, "; "), + Fields: meta, + }, nil, now) + } + legalName := effectiveLegalName(party, company, h.cfg.ConfidenceThreshold) // Agent B: compliance documents and trust pages. @@ -241,13 +260,12 @@ func (h *enrichmentHandler) Process(ctx context.Context, party coredata.CommonTh // transaction; the File row is inserted below. logoFile := h.prepareLogo(ctx, party, website) - meta := make(map[string]EnrichmentFieldMeta) - for _, field := range scalarFields(company, compliance) { applyScalarField(&party, meta, prior, field, h.cfg.ConfidenceThreshold, now) } applyCertifications(&party, meta, prior, compliance.Certifications, h.cfg.ConfidenceThreshold, now) + applyLegalNameFallback(&party, meta, now) status := enrichmentStatusDone @@ -266,6 +284,20 @@ func (h *enrichmentHandler) Process(ctx context.Context, party coredata.CommonTh Fields: meta, } + return h.persist(ctx, party, payload, logoFile, now) +} + +// persist marshals the enrichment payload onto the row and writes it in a +// single transaction, inserting the logo File row and linking it when one +// was prepared. It always writes an enrichment payload, even on a +// no-result run, so stale recovery does not re-queue the row. +func (h *enrichmentHandler) persist( + ctx context.Context, + party coredata.CommonThirdParty, + payload EnrichmentMetadata, + logoFile *coredata.File, + now time.Time, +) error { raw, err := json.Marshal(payload) if err != nil { return fmt.Errorf("cannot marshal enrichment metadata: %w", err) @@ -298,7 +330,7 @@ func (h *enrichmentHandler) Process(ctx context.Context, party coredata.CommonTh "enriched common third party", log.String("common_third_party_id", party.ID.String()), log.String("name", party.Name), - log.String("status", status), + log.String("status", payload.Status), log.Bool("logo_stored", logoFile != nil), ) @@ -323,10 +355,27 @@ func (h *enrichmentHandler) RecoverStale(ctx context.Context) error { ) } +// runCompanyProfile builds Agent A with a per-run browser when a Chrome +// endpoint is configured, then runs it. The browser is closed when the +// run returns. It is not pinned to a domain so the agent can follow a +// product site to the legal entity's corporate domain (where the legal +// name and headquarters address live); SSRF protection still blocks +// non-public hosts. func (h *enrichmentHandler) runCompanyProfile( ctx context.Context, party coredata.CommonThirdParty, ) (CompanyProfileResult, error) { + var browserTools []agent.Tool + + if h.cfg.ChromeAddr != "" { + webBrowser := browser.NewBrowser(ctx, h.cfg.ChromeAddr) + defer webBrowser.Close() + + browserTools = browser.NewReadOnlyToolset(webBrowser).Tools() + } + + companyAgent := buildCompanyProfileAgent(h.cfg, h.logger, browserTools) + prompt := buildCompanyProfilePrompt(party) agentCtx, cancel := context.WithTimeout(ctx, h.cfg.AgentTimeout) @@ -334,7 +383,7 @@ func (h *enrichmentHandler) runCompanyProfile( result, err := agent.RunTyped[CompanyProfileResult]( agentCtx, - h.companyAgent, + companyAgent, []llm.Message{ { Role: llm.RoleUser, diff --git a/pkg/thirdparty/prompts/common_third_party_company_profile.txt.tmpl b/pkg/thirdparty/prompts/common_third_party_company_profile.txt.tmpl index 8a84864bd..8aeeec5a6 100644 --- a/pkg/thirdparty/prompts/common_third_party_company_profile.txt.tmpl +++ b/pkg/thirdparty/prompts/common_third_party_company_profile.txt.tmpl @@ -1,33 +1,30 @@ -You are a research agent that builds a factual company profile for a software -vendor or service provider. You are given the vendor's name (and sometimes a -known website). Return only verifiable identity facts. + +You are a research agent that builds a factual company profile for a software vendor or service provider. You are given the vendor's name and sometimes a known website. You return only verifiable identity facts. + -Resolve these fields: + +Resolve these fields for the vendor: +- legal_name: the full legal entity name, including the suffix (Inc., Ltd., GmbH, S.A.S., etc.). +- headquarter_address: the postal address of the company's headquarters (street, city, region, country). +- website_url: the canonical primary marketing website. Use the https scheme, drop tracking query parameters and trailing paths, and prefer the apex or www host the vendor uses for its homepage. -- legal_name: the full legal entity name, including the suffix (Inc., Ltd., - GmbH, S.A.S., etc.). Prefer the name as it appears in the vendor's own legal - documents (privacy policy footer, terms of service) or an official business - registry. -- headquarter_address: the postal address of the company's headquarters. -- website_url: the canonical primary marketing website. Use the https scheme, - drop tracking query parameters and trailing paths, and prefer the apex or - www host the vendor uses for its homepage. +Each field carries a value, a 0.0-1.0 confidence, and the source_url where you verified it. + -Method: + +0. Work within a tight tool budget. You have a limited number of turns, so do not exhaust them browsing exhaustively. Read the homepage and the one or two pages most likely to carry identity facts (footer, imprint/impressum, about, legal, contact), then produce the structured output. Leaving a field empty is acceptable; running out of turns before producing the structured output is not. -- Use the web_search tool when it is available to confirm facts. Prefer the - vendor's own website and official registries over third-party aggregators. -- The website_url is the most important field: downstream steps depend on it. - Resolve it carefully and with high confidence when the vendor clearly owns a - primary domain. +1. website_url is the gating field: downstream steps (compliance-document discovery, logo) only run when it is resolved. Resolve one canonical primary marketing domain with high confidence. If you cannot confidently identify the vendor's own primary domain, return an empty website_url with confidence 0 rather than guessing. -Rules: +2. When the browser tools (navigate, extract_links, find_links_matching) are available, use them to read the vendor's own site. The legal name and headquarters address usually live in the footer, imprint/impressum, about, legal, or contact pages, not on the homepage. -- Never guess. If you cannot verify a field, return an empty string with a - confidence of 0. -- confidence is your own 0.0-1.0 estimate that the value is correct. Reserve - values above 0.8 for facts you verified from the vendor's own site or an - official registry. -- source_url is the page where you verified the value. Leave it empty when the - value was not found. -- Do not include commentary; return only the structured fields. +3. The legal entity often lives on a different domain from the product or marketing site. A product site (for example a .org or .io) frequently links to a corporate domain (for example a .ltd or .com) in its footer, imprint, or terms. Follow those links to the corporate domain to confirm the legal_name and headquarter_address. + +4. Use the web_search tool to confirm facts and to find the corporate domain or an official business registry. Prefer the vendor's own website and official registries over third-party aggregators. + +5. Never guess. If you cannot verify a field, return an empty string with a confidence of 0. confidence is your own 0.0-1.0 estimate that the value is correct; reserve values above 0.8 for facts you verified on the vendor's own site or an official registry. + +6. source_url is the page where you verified the value. Leave it empty when the value was not found. + +7. Do not include commentary; return only the structured fields. + diff --git a/pkg/thirdparty/prompts/common_third_party_compliance_docs.txt.tmpl b/pkg/thirdparty/prompts/common_third_party_compliance_docs.txt.tmpl index c8073481d..aa6cbf1ee 100644 --- a/pkg/thirdparty/prompts/common_third_party_compliance_docs.txt.tmpl +++ b/pkg/thirdparty/prompts/common_third_party_compliance_docs.txt.tmpl @@ -1,51 +1,40 @@ -You are a research agent that locates a software vendor's public compliance -documents and trust pages. You are given the vendor's name and, usually, its -website. Return canonical URLs for each document, plus the certifications the -vendor publicly claims. + +You are a research agent that locates a software vendor's public compliance documents and trust pages. You are given the vendor's name and, usually, its website. You return canonical URLs for each document, plus the certifications the vendor publicly claims. + + Resolve these fields (each is a single URL unless noted): - - privacy_policy_url: the privacy policy. - terms_of_service_url: the terms of service / terms of use. - service_level_agreement_url: the public SLA. Frequently gated behind sales. -- service_software_agreement_url: the master software/subscription agreement - (MSA). Often gated, or the same document as the terms of service. -- data_processing_agreement_url: the DPA. Often a downloadable PDF; sometimes - only available on request. -- business_associate_agreement_url: the HIPAA BAA. Almost always gated behind - sales or an enterprise plan. +- service_software_agreement_url: the master software/subscription agreement (MSA). Often gated, or the same document as the terms of service. +- data_processing_agreement_url: the DPA. Often a downloadable PDF; sometimes only available on request. +- business_associate_agreement_url: the HIPAA BAA. Almost always gated behind sales or an enterprise plan. - subprocessors_list_url: the sub-processors list page. -- status_page_url: the uptime/status page (commonly status. or a - hosted statuspage.io / instatus / better-uptime page). +- status_page_url: the uptime/status page (commonly status. or a hosted statuspage.io / instatus / better-uptime page). - security_page_url: the security overview page. -- trust_page_url: the trust center / trust portal. Many vendors host this on - Vanta, SafeBase, Drata, Conveyor, or a /trust path. -- certifications: the compliance frameworks and certifications the vendor - publicly claims (e.g. SOC 2 Type II, ISO 27001, ISO 27701, PCI DSS, HIPAA, - GDPR, FedRAMP). Read these from the trust or security page. +- trust_page_url: the trust center / trust portal. Many vendors host this on Vanta, SafeBase, Drata, Conveyor, or a /trust path. +- certifications: the compliance frameworks and certifications the vendor publicly claims (e.g. SOC 2 Type II, ISO 27001, ISO 27701, PCI DSS, HIPAA, GDPR, FedRAMP). Read these from the trust or security page. -Method: +Each field carries a value, a 0.0-1.0 confidence, and the source_url where you found it. + -- Start from the vendor's when provided. Use the browser tools - (navigate, extract_links, find_links_matching) to inspect the site footer - and the trust/security pages, which is where these links normally live. -- Use web_search to fill gaps with site-scoped queries (for example - "site: data processing agreement"). Prefer the vendor's own domain - and its hosted trust portal over third-party aggregators. -- Finding the trust center first usually yields the security page and the - certifications in one place. + +0. Work within a tight tool budget. You have a limited number of turns, so do not exhaust them browsing exhaustively. Prefer extract_links / find_links_matching over navigating many pages one at a time, fetch the footer and trust/security pages first, and stop as soon as you have the core documents. Leaving a field empty is acceptable; running out of turns before producing the structured output is not. Once you have what you can reasonably find, produce the final structured output instead of continuing to search. -Rules: +1. Start from the vendor's when provided. Use the browser tools (navigate, extract_links, find_links_matching) to inspect the site footer and the trust/security pages, which is where these links normally live. Finding the trust center first usually yields the security page and the certifications in one place. -- Return the most specific canonical URL. Prefer a direct document/page URL - over a generic legal-index page. -- Never guess or fabricate a URL. If a document is gated, only available on - request, or you cannot find it, return an empty string with confidence 0. - Several of these (SLA, MSA, BAA) are commonly non-public; leaving them empty - is the correct outcome. -- confidence is your own 0.0-1.0 estimate that the URL is correct and current. - Reserve values above 0.8 for URLs you actually reached on the vendor's own - domain or hosted trust portal. -- source_url is the page where you found the link (for certifications, the - page you read them from). Leave it empty when nothing was found. -- Do not include commentary; return only the structured fields. +2. Use web_search to fill gaps with site-scoped queries (for example "site: data processing agreement"). Prefer the vendor's own domain and its hosted trust portal over third-party aggregators. + +3. Keep document URLs on the primary host. A vendor may operate several linked domains (for example a product site and a corporate site); when the same document is reachable on more than one of them, choose the one served from the host and verify it loads. Do not mix hosts across fields when one canonical host serves them all. + +4. Return the most specific canonical URL. Prefer a direct document/page URL over a generic legal-index page. + +5. Never guess or fabricate a URL. If a document is gated, only available on request, or you cannot find it, return an empty string with confidence 0. Several of these (SLA, MSA, BAA) are commonly non-public; leaving them empty is the correct outcome. + +6. confidence is your own 0.0-1.0 estimate that the URL is correct and current. Reserve values above 0.8 for URLs you actually reached on the vendor's own domain or hosted trust portal. + +7. source_url is the page where you found the link (for certifications, the page you read them from). Leave it empty when nothing was found. + +8. Do not include commentary; return only the structured fields. +