From 509d0c88b123ed2fc4ad78c30aaea99c9525e82c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Wed, 22 Apr 2026 22:36:14 +0200 Subject: [PATCH] Add vendor assessment agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- GNUmakefile | 36 +- e2e/console/vendor_test.go | 112 +++++ pkg/agent/agent.go | 90 ++-- pkg/agent/agent_test.go | 113 ++--- pkg/agent/agent_tool.go | 17 +- pkg/agent/agent_tool_test.go | 19 +- pkg/agent/model_settings.go | 1 + pkg/agent/progress.go | 36 ++ pkg/agent/run.go | 166 ++++++- pkg/agent/tool.go | 43 +- pkg/agent/tool_test.go | 58 +-- pkg/agent/tools/browser/browser.go | 170 ++++++++ pkg/agent/tools/browser/click.go | 88 ++++ pkg/agent/tools/browser/download_pdf.go | 157 +++++++ pkg/agent/tools/browser/extract_links.go | 81 ++++ pkg/agent/tools/browser/extract_text.go | 95 ++++ pkg/agent/tools/browser/fetch_robots.go | 107 +++++ pkg/agent/tools/browser/fetch_sitemap.go | 151 +++++++ pkg/agent/tools/browser/find_links.go | 97 +++++ pkg/agent/tools/browser/helpers.go | 118 +++++ pkg/agent/tools/browser/helpers_test.go | 92 ++++ pkg/agent/tools/browser/navigate.go | 90 ++++ pkg/agent/tools/browser/select.go | 82 ++++ pkg/agent/tools/browser/sitemap_test.go | 191 ++++++++ pkg/agent/tools/browser/toolset.go | 65 +++ pkg/agent/tools/browser/url_check.go | 33 ++ pkg/agent/tools/internal/netcheck/netcheck.go | 126 ++++++ pkg/agent/tools/search/diff_documents.go | 156 +++++++ pkg/agent/tools/search/diff_test.go | 203 +++++++++ pkg/agent/tools/search/government_db.go | 164 +++++++ .../tools/search/search.go} | 30 +- pkg/agent/tools/search/wayback.go | 147 +++++++ pkg/agent/tools/search/wayback_test.go | 109 +++++ pkg/agent/tools/search/web_search.go | 74 ++++ pkg/agent/tools/security/cors.go | 122 ++++++ pkg/agent/tools/security/cors_test.go | 71 +++ pkg/agent/tools/security/csp.go | 140 ++++++ pkg/agent/tools/security/csp_test.go | 81 ++++ pkg/agent/tools/security/dmarc.go | 106 +++++ pkg/agent/tools/security/dmarc_test.go | 65 +++ pkg/agent/tools/security/dns_records.go | 166 +++++++ pkg/agent/tools/security/dnssec.go | 101 +++++ pkg/agent/tools/security/headers.go | 141 ++++++ pkg/agent/tools/security/headers_test.go | 197 +++++++++ pkg/agent/tools/security/hibp.go | 117 +++++ pkg/agent/tools/security/security.go | 51 +++ pkg/agent/tools/security/spf.go | 120 +++++ pkg/agent/tools/security/spf_test.go | 70 +++ pkg/agent/tools/security/ssl.go | 147 +++++++ pkg/agent/tools/security/ssl_test.go | 48 ++ pkg/agent/tools/security/whois.go | 253 +++++++++++ pkg/agent/tools/security/whois_test.go | 271 ++++++++++++ pkg/agent/typed_test.go | 10 +- pkg/agents/changelog_generator.go | 79 ---- pkg/agents/vendor_assessment.go | 149 ------- pkg/bootstrap/builder.go | 39 +- pkg/bootstrap/builder_test.go | 98 ++--- pkg/cmd/vendormgmt/assess/assess.go | 151 +++++++ pkg/cmd/vendormgmt/vendormgmt.go | 2 + pkg/llm/anthropic/provider.go | 74 +++- pkg/llm/chat.go | 43 +- pkg/llm/errors.go | 13 + pkg/llm/message.go | 10 + pkg/llm/openai/provider.go | 21 + pkg/llm/part.go | 12 +- pkg/probo/document_service.go | 41 +- pkg/probo/prompts/changelog_generator.txt | 21 + pkg/probo/service.go | 34 +- pkg/probo/vendor_service.go | 200 +++++++-- pkg/probod/evidence_describer_config.go | 38 -- pkg/probod/llm.go | 22 + pkg/probod/llm_config.go | 56 ++- pkg/probod/probod.go | 34 +- pkg/probod/vendor_assessor.go | 58 +++ .../api/console/v1/graphql/vendor.graphql | 9 + pkg/server/api/console/v1/types/vendor.go | 13 + pkg/server/api/console/v1/vendor_resolvers.go | 11 +- pkg/server/api/mcp/v1/schema.resolvers.go | 20 + pkg/server/api/mcp/v1/specification.yaml | 59 +++ pkg/server/api/mcp/v1/types/vendor.go | 21 + pkg/server/gqlutils/errors.go | 14 + pkg/vetting/assessment.go | 337 ++++++++++++++ pkg/vetting/assessment_test.go | 66 +++ pkg/vetting/orchestrator.go | 261 +++++++++++ pkg/vetting/output_types.go | 359 +++++++++++++++ pkg/vetting/output_types_test.go | 80 ++++ pkg/vetting/progress.go | 412 ++++++++++++++++++ pkg/vetting/prompts/ai_risk.txt | 83 ++++ pkg/vetting/prompts/analyzer.txt | 80 ++++ pkg/vetting/prompts/business_continuity.txt | 55 +++ pkg/vetting/prompts/code_security.txt | 81 ++++ pkg/vetting/prompts/compliance.txt | 59 +++ pkg/vetting/prompts/crawler.txt | 34 ++ pkg/vetting/prompts/data_processing.txt | 75 ++++ pkg/vetting/prompts/default_procedure.txt | 264 +++++++++++ pkg/vetting/prompts/extraction.txt | 13 + pkg/vetting/prompts/financial_stability.txt | 65 +++ pkg/vetting/prompts/incident_response.txt | 67 +++ pkg/vetting/prompts/market.txt | 46 ++ pkg/vetting/prompts/orchestrator_base.txt | 33 ++ pkg/vetting/prompts/professional_standing.txt | 59 +++ pkg/vetting/prompts/regulatory_compliance.txt | 87 ++++ pkg/vetting/prompts/security.txt | 83 ++++ pkg/vetting/prompts/subprocessor.txt | 47 ++ pkg/vetting/prompts/vendor_comparison.txt | 41 ++ pkg/vetting/prompts/websearch.txt | 52 +++ pkg/vetting/sub_agent.go | 82 ++++ pkg/vetting/sub_agent_specs.go | 233 ++++++++++ 108 files changed, 9445 insertions(+), 645 deletions(-) create mode 100644 pkg/agent/progress.go create mode 100644 pkg/agent/tools/browser/browser.go create mode 100644 pkg/agent/tools/browser/click.go create mode 100644 pkg/agent/tools/browser/download_pdf.go create mode 100644 pkg/agent/tools/browser/extract_links.go create mode 100644 pkg/agent/tools/browser/extract_text.go create mode 100644 pkg/agent/tools/browser/fetch_robots.go create mode 100644 pkg/agent/tools/browser/fetch_sitemap.go create mode 100644 pkg/agent/tools/browser/find_links.go create mode 100644 pkg/agent/tools/browser/helpers.go create mode 100644 pkg/agent/tools/browser/helpers_test.go create mode 100644 pkg/agent/tools/browser/navigate.go create mode 100644 pkg/agent/tools/browser/select.go create mode 100644 pkg/agent/tools/browser/sitemap_test.go create mode 100644 pkg/agent/tools/browser/toolset.go create mode 100644 pkg/agent/tools/browser/url_check.go create mode 100644 pkg/agent/tools/internal/netcheck/netcheck.go create mode 100644 pkg/agent/tools/search/diff_documents.go create mode 100644 pkg/agent/tools/search/diff_test.go create mode 100644 pkg/agent/tools/search/government_db.go rename pkg/{agents/agents.go => agent/tools/search/search.go} (59%) create mode 100644 pkg/agent/tools/search/wayback.go create mode 100644 pkg/agent/tools/search/wayback_test.go create mode 100644 pkg/agent/tools/search/web_search.go create mode 100644 pkg/agent/tools/security/cors.go create mode 100644 pkg/agent/tools/security/cors_test.go create mode 100644 pkg/agent/tools/security/csp.go create mode 100644 pkg/agent/tools/security/csp_test.go create mode 100644 pkg/agent/tools/security/dmarc.go create mode 100644 pkg/agent/tools/security/dmarc_test.go create mode 100644 pkg/agent/tools/security/dns_records.go create mode 100644 pkg/agent/tools/security/dnssec.go create mode 100644 pkg/agent/tools/security/headers.go create mode 100644 pkg/agent/tools/security/headers_test.go create mode 100644 pkg/agent/tools/security/hibp.go create mode 100644 pkg/agent/tools/security/security.go create mode 100644 pkg/agent/tools/security/spf.go create mode 100644 pkg/agent/tools/security/spf_test.go create mode 100644 pkg/agent/tools/security/ssl.go create mode 100644 pkg/agent/tools/security/ssl_test.go create mode 100644 pkg/agent/tools/security/whois.go create mode 100644 pkg/agent/tools/security/whois_test.go delete mode 100644 pkg/agents/changelog_generator.go delete mode 100644 pkg/agents/vendor_assessment.go create mode 100644 pkg/cmd/vendormgmt/assess/assess.go create mode 100644 pkg/probo/prompts/changelog_generator.txt delete mode 100644 pkg/probod/evidence_describer_config.go create mode 100644 pkg/probod/vendor_assessor.go create mode 100644 pkg/vetting/assessment.go create mode 100644 pkg/vetting/assessment_test.go create mode 100644 pkg/vetting/orchestrator.go create mode 100644 pkg/vetting/output_types.go create mode 100644 pkg/vetting/output_types_test.go create mode 100644 pkg/vetting/progress.go create mode 100644 pkg/vetting/prompts/ai_risk.txt create mode 100644 pkg/vetting/prompts/analyzer.txt create mode 100644 pkg/vetting/prompts/business_continuity.txt create mode 100644 pkg/vetting/prompts/code_security.txt create mode 100644 pkg/vetting/prompts/compliance.txt create mode 100644 pkg/vetting/prompts/crawler.txt create mode 100644 pkg/vetting/prompts/data_processing.txt create mode 100644 pkg/vetting/prompts/default_procedure.txt create mode 100644 pkg/vetting/prompts/extraction.txt create mode 100644 pkg/vetting/prompts/financial_stability.txt create mode 100644 pkg/vetting/prompts/incident_response.txt create mode 100644 pkg/vetting/prompts/market.txt create mode 100644 pkg/vetting/prompts/orchestrator_base.txt create mode 100644 pkg/vetting/prompts/professional_standing.txt create mode 100644 pkg/vetting/prompts/regulatory_compliance.txt create mode 100644 pkg/vetting/prompts/security.txt create mode 100644 pkg/vetting/prompts/subprocessor.txt create mode 100644 pkg/vetting/prompts/vendor_comparison.txt create mode 100644 pkg/vetting/prompts/websearch.txt create mode 100644 pkg/vetting/sub_agent.go create mode 100644 pkg/vetting/sub_agent_specs.go diff --git a/GNUmakefile b/GNUmakefile index 94b216dac..1e88fcdbc 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -38,11 +38,24 @@ GO_TOOL= $(GO_BASE) tool TEST_FLAGS?= -race -cover -coverprofile=coverage.out +E2E_CONFIG ?= $(CURDIR)/e2e/console/testdata/config.yaml E2E_COVER_DIR ?= $(CURDIR)/coverage/e2e DOCKER_IMAGE_NAME= ghcr.io/getprobo/probo DOCKER_TAG_NAME?= latest +PROBOD_BIN_DEPS= pkg/server/api/connect/v1/schema/schema.go \ + pkg/server/api/connect/v1/types/types.go \ + pkg/server/api/console/v1/schema/schema.go \ + pkg/server/api/console/v1/types/types.go \ + pkg/server/api/trust/v1/schema/schema.go \ + pkg/server/api/trust/v1/types/types.go \ + pkg/server/api/mcp/v1/server/server.go \ + pkg/server/api/mcp/v1/types/types.go \ + apps/console/dist/index.html \ + apps/trust/dist/index.html \ + @probo/emails + PROBOD_BIN_EXTRA_DEPS= PROBOD_BIN= bin/probod PROBOD_SRC= cmd/probod/main.go @@ -126,8 +139,9 @@ test-bench: test ## Run benchmark tests .PHONY: test-e2e test-e2e: CGO_ENABLED=1 -test-e2e: bin/probod ## Run console e2e tests - PROBO_E2E_BINARY=$(CURDIR)/bin/probod \ +test-e2e: $(PROBOD_BIN) ## Run console e2e tests + PROBO_E2E_BINARY=$(CURDIR)/$(PROBOD_BIN) \ + PROBO_E2E_CONFIG=$(E2E_CONFIG) \ GOTESTSUM_FORMAT=testname $(GO_TEST) -count=1 ./e2e/console/... bin/probod-coverage: @@ -138,6 +152,7 @@ test-e2e-coverage: bin/probod-coverage ## Run e2e tests with coverage @$(RM) -rf $(E2E_COVER_DIR) && $(MKDIR) -p $(E2E_COVER_DIR) PROBO_E2E_BINARY=$(CURDIR)/bin/probod-coverage \ PROBO_E2E_COVERDIR=$(E2E_COVER_DIR) \ + PROBO_E2E_CONFIG=$(E2E_CONFIG) \ CGO_ENABLED=1 $(GO) test -count=1 -v ./e2e/console/... $(GO) tool covdata textfmt -i=$(E2E_COVER_DIR) -o=coverage-e2e.out $(GO) tool cover -html=coverage-e2e.out -o=coverage-e2e.html @@ -149,7 +164,7 @@ coverage-combined: coverage-report test-e2e-coverage ## Generate combined covera $(GO) tool cover -html=coverage-combined.out -o=coverage-combined.html .PHONY: build -build: bin/probod bin/prb bin/probod-bootstrap +build: $(PROBOD_BIN) bin/prb bin/probod-bootstrap CFG_DEV_OAUTH2_KEY = cfg/.dev-oauth2-signing-key.pem DEV_ENV = .env @@ -217,19 +232,8 @@ scan-license: ## Check dependencies licenses compliance docker-build: $(DOCKER_BUILD) --tag $(DOCKER_IMAGE_NAME):$(DOCKER_TAG_NAME) --file Dockerfile . -.PHONY: bin/probod -bin/probod: pkg/server/api/connect/v1/schema/schema.go \ - pkg/server/api/connect/v1/types/types.go \ - pkg/server/api/console/v1/schema/schema.go \ - pkg/server/api/console/v1/types/types.go \ - pkg/server/api/trust/v1/schema/schema.go \ - pkg/server/api/trust/v1/types/types.go \ - pkg/server/api/mcp/v1/server/server.go \ - pkg/server/api/mcp/v1/types/types.go \ - apps/console/dist/index.html \ - apps/trust/dist/index.html \ - $(PROBOD_BIN_EXTRA_DEPS) \ - @probo/emails +.PHONY: $(PROBOD_BIN) +$(PROBOD_BIN): $(PROBOD_BIN_DEPS) $(PROBOD_BIN_EXTRA_DEPS) $(GO_BUILD) -o $(PROBOD_BIN) $(PROBOD_SRC) .PHONY: bin/prb diff --git a/e2e/console/vendor_test.go b/e2e/console/vendor_test.go index 6efc0e7b9..128771066 100644 --- a/e2e/console/vendor_test.go +++ b/e2e/console/vendor_test.go @@ -986,6 +986,118 @@ func TestVendor_OmittableWebsiteUrl(t *testing.T) { }) } +// TestVendor_Assess exercises the assessVendor mutation through authorization +// and tenant-isolation paths without running the real LLM/browser pipeline. +// The e2e config deliberately omits `llm.vendor-assessor.provider`, so an +// authorized call reaches DisabledVendorAssessor and surfaces a stable +// UNAVAILABLE error. Happy-path payload shape is covered by unit tests in +// pkg/probo. +func TestVendor_Assess(t *testing.T) { + t.Parallel() + + const query = ` + mutation AssessVendor($input: AssessVendorInput!) { + assessVendor(input: $input) { + vendor { + id + } + } + } + ` + + type resultShape struct { + AssessVendor struct { + Vendor struct { + ID string `json:"id"` + } `json:"vendor"` + } `json:"assessVendor"` + } + + t.Run("owner call surfaces the disabled error", func(t *testing.T) { + t.Parallel() + + owner := testutil.NewClient(t, testutil.RoleOwner) + vendorID := factory.NewVendor(owner).WithName("Unconfigured assess").Create() + + var result resultShape + err := owner.Execute(query, map[string]any{ + "input": map[string]any{ + "id": vendorID, + "websiteUrl": "https://vendor.example.com", + }, + }, &result) + testutil.RequireErrorCode(t, err, "UNAVAILABLE") + }) + + t.Run("admin call surfaces the disabled error", func(t *testing.T) { + t.Parallel() + + owner := testutil.NewClient(t, testutil.RoleOwner) + admin := testutil.NewClientInOrg(t, testutil.RoleAdmin, owner) + vendorID := factory.NewVendor(owner).WithName("Admin-assessed vendor").Create() + + var result resultShape + err := admin.Execute(query, map[string]any{ + "input": map[string]any{ + "id": vendorID, + "websiteUrl": "https://admin.example.com", + }, + }, &result) + testutil.RequireErrorCode(t, err, "UNAVAILABLE") + }) + + t.Run("viewer cannot assess a vendor", func(t *testing.T) { + t.Parallel() + + owner := testutil.NewClient(t, testutil.RoleOwner) + viewer := testutil.NewClientInOrg(t, testutil.RoleViewer, owner) + vendorID := factory.NewVendor(owner).WithName("Viewer attempt").Create() + + var result resultShape + err := viewer.Execute(query, map[string]any{ + "input": map[string]any{ + "id": vendorID, + "websiteUrl": "https://viewer.example.com", + }, + }, &result) + testutil.RequireForbiddenError(t, err) + }) + + t.Run("cannot assess vendor from another organization", func(t *testing.T) { + t.Parallel() + + org1Owner := testutil.NewClient(t, testutil.RoleOwner) + org2Owner := testutil.NewClient(t, testutil.RoleOwner) + vendorID := factory.NewVendor(org1Owner).WithName("Org1 vendor").Create() + + var result resultShape + err := org2Owner.Execute(query, map[string]any{ + "input": map[string]any{ + "id": vendorID, + "websiteUrl": "https://cross-tenant.example.com", + }, + }, &result) + require.Error(t, err, "vendor assess must not cross tenant boundaries") + }) + + t.Run("procedure is accepted on the input", func(t *testing.T) { + t.Parallel() + + owner := testutil.NewClient(t, testutil.RoleOwner) + vendorID := factory.NewVendor(owner).WithName("Procedure test").Create() + + var result resultShape + err := owner.Execute(query, map[string]any{ + "input": map[string]any{ + "id": vendorID, + "websiteUrl": "https://procedure.example.com", + "procedure": "Focus on SOC 2 controls and data residency", + }, + }, &result) + testutil.RequireErrorCode(t, err, "UNAVAILABLE") + }) +} + func TestVendor_TenantIsolation(t *testing.T) { t.Parallel() diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index dad6c8804..c5d1fc036 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -23,48 +23,53 @@ import ( "go.probo.inc/probo/pkg/llm" ) -const DefaultMaxTurns = 10 +const ( + DefaultMaxTurns = 10 + DefaultMaxEmptyOutputRetries = 2 +) type ( Option func(*Agent) Agent struct { - name string - handoffDescription string - instructions string - instructionsFunc func(ctx context.Context, a *Agent) string - model string - modelSettings ModelSettings - tools []Tool - handoffs []*Handoff - mcpServers []*MCPServer - maxTurns int - maxToolDepth int - client *llm.Client - logger *log.Logger - hooks []RunHooks - agentHooks AgentHooks - inputGuardrails []InputGuardrail - outputGuardrails []OutputGuardrail - session Session - sessionID string - outputType *OutputType - toolUseBehavior ToolUseBehavior - resetToolChoice bool - responseFormat *llm.ResponseFormat - approval *ApprovalConfig + name string + handoffDescription string + instructions string + instructionsFunc func(ctx context.Context, a *Agent) string + model string + modelSettings ModelSettings + tools []Tool + handoffs []*Handoff + mcpServers []*MCPServer + maxTurns int + maxEmptyOutputRetries int + maxToolDepth int + client *llm.Client + logger *log.Logger + hooks []RunHooks + agentHooks AgentHooks + inputGuardrails []InputGuardrail + outputGuardrails []OutputGuardrail + session Session + sessionID string + outputType *OutputType + toolUseBehavior ToolUseBehavior + resetToolChoice bool + responseFormat *llm.ResponseFormat + approval *ApprovalConfig } ) func New(name string, client *llm.Client, opts ...Option) *Agent { a := &Agent{ - name: name, - client: client, - maxTurns: DefaultMaxTurns, - maxToolDepth: DefaultMaxToolDepth, - toolUseBehavior: RunLLMAgain(), - resetToolChoice: true, - logger: log.NewLogger(log.WithOutput(io.Discard)), + name: name, + client: client, + maxTurns: DefaultMaxTurns, + maxEmptyOutputRetries: DefaultMaxEmptyOutputRetries, + maxToolDepth: DefaultMaxToolDepth, + toolUseBehavior: RunLLMAgain(), + resetToolChoice: true, + logger: log.NewLogger(log.WithOutput(io.Discard)), } for _, opt := range opts { @@ -204,6 +209,18 @@ func WithMaxTurns(n int) Option { } } +// WithMaxEmptyOutputRetries bounds the number of times the core loop +// will re-ask the model to produce a structured output after it +// returned a thinking-only empty response on a synthesis turn. +func WithMaxEmptyOutputRetries(n int) Option { + return func(a *Agent) { + if n < 0 { + n = 0 + } + a.maxEmptyOutputRetries = n + } +} + func WithMaxToolDepth(n int) Option { return func(a *Agent) { if n < 1 { @@ -255,6 +272,15 @@ func WithParallelToolCalls(enabled bool) Option { } } +func WithThinking(budgetTokens int) Option { + return func(a *Agent) { + a.modelSettings.Thinking = &llm.ThinkingConfig{ + Enabled: true, + BudgetTokens: budgetTokens, + } + } +} + func WithLogger(l *log.Logger) Option { return func(a *Agent) { a.logger = l diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index 085c244e2..ffd72f1ce 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -351,14 +351,13 @@ func TestRun(t *testing.T) { City string `json:"city"` } - weatherTool, err := agent.FunctionTool[Params]( + weatherTool := agent.FunctionTool[Params]( "get_weather", "Get weather for a city", func(_ context.Context, p Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "Sunny, 22°C in " + p.City}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -411,14 +410,13 @@ func TestRun(t *testing.T) { } type Params struct{} - noopTool, err := agent.FunctionTool[Params]( + noopTool := agent.FunctionTool[Params]( "noop", "No-op", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "ok"}, nil }, ) - require.NoError(t, err) ag := agent.New( "assistant", @@ -428,7 +426,7 @@ func TestRun(t *testing.T) { agent.WithMaxTurns(2), ) - _, err = ag.Run( + _, err := ag.Run( context.Background(), []llm.Message{userMessage("loop")}, ) @@ -447,14 +445,13 @@ func TestRun(t *testing.T) { type Params struct{} makeTool := func(name string) agent.Tool { - tool, err := agent.FunctionTool[Params]( + tool := agent.FunctionTool[Params]( name, "desc", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "ok"}, nil }, ) - require.NoError(t, err) return tool } @@ -553,22 +550,20 @@ func TestRun(t *testing.T) { type Params struct{} - tool1, err := agent.FunctionTool[Params]( + tool1 := agent.FunctionTool[Params]( "first", "First tool", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "result_1"}, nil }, ) - require.NoError(t, err) - tool2, err := agent.FunctionTool[Params]( + tool2 := agent.FunctionTool[Params]( "second", "Second tool", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "result_2"}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -622,22 +617,20 @@ func TestRun(t *testing.T) { type Params struct{} - successTool, err := agent.FunctionTool[Params]( + successTool := agent.FunctionTool[Params]( "succeed", "Always succeeds", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "success_result"}, nil }, ) - require.NoError(t, err) - failTool, err := agent.FunctionTool[Params]( + failTool := agent.FunctionTool[Params]( "fail", "Always fails", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{}, errors.New("tool exploded") }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -696,7 +689,7 @@ func TestRun(t *testing.T) { var capturedTenantID string type Params struct{} - tool, err := agent.FunctionTool[Params]( + tool := agent.FunctionTool[Params]( "check_tenant", "Check current tenant", func(ctx context.Context, _ Params) (agent.ToolResult, error) { @@ -705,7 +698,6 @@ func TestRun(t *testing.T) { return agent.ToolResult{Content: "tenant: " + rc.TenantID}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -1069,14 +1061,13 @@ func TestRun_Hooks(t *testing.T) { t.Parallel() type Params struct{} - noopTool, err := agent.FunctionTool[Params]( + noopTool := agent.FunctionTool[Params]( "noop", "No-op", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "ok"}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -1354,14 +1345,13 @@ func TestRun_ToolUseBehavior(t *testing.T) { t.Parallel() type Params struct{} - tool, err := agent.FunctionTool[Params]( + tool := agent.FunctionTool[Params]( "compute", "Compute something", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "computed_result"}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -1399,22 +1389,20 @@ func TestRun_ToolUseBehavior(t *testing.T) { type Params struct{} - tool1, err := agent.FunctionTool[Params]( + tool1 := agent.FunctionTool[Params]( "search", "Search", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "search_result"}, nil }, ) - require.NoError(t, err) - tool2, err := agent.FunctionTool[Params]( + tool2 := agent.FunctionTool[Params]( "submit", "Submit", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "submitted"}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -1449,14 +1437,13 @@ func TestRun_ToolUseBehavior(t *testing.T) { t.Parallel() type Params struct{} - tool, err := agent.FunctionTool[Params]( + tool := agent.FunctionTool[Params]( "noop", "No-op", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "ok"}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -1492,14 +1479,13 @@ func TestRun_ToolUseBehavior(t *testing.T) { t.Parallel() type Params struct{} - tool, err := agent.FunctionTool[Params]( + tool := agent.FunctionTool[Params]( "compute", "Compute something", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "result"}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -1520,7 +1506,7 @@ func TestRun_ToolUseBehavior(t *testing.T) { })), ) - _, err = ag.Run( + _, err := ag.Run( context.Background(), []llm.Message{userMessage("compute")}, ) @@ -1581,14 +1567,13 @@ func TestRun_Approval(t *testing.T) { }, } - deleteTool, err := agent.FunctionTool[struct{}]( + deleteTool := agent.FunctionTool[struct{}]( "delete_account", "Deletes the user account", func(_ context.Context, _ struct{}) (agent.ToolResult, error) { return agent.ToolResult{Content: "deleted"}, nil }, ) - require.NoError(t, err) ag := agent.New( "assistant", @@ -1600,7 +1585,7 @@ func TestRun_Approval(t *testing.T) { }), ) - _, err = ag.Run( + _, err := ag.Run( context.Background(), []llm.Message{userMessage("Delete my account")}, ) @@ -1623,7 +1608,7 @@ func TestRun_Approval(t *testing.T) { var toolExecuted bool - deleteTool, err := agent.FunctionTool[struct{}]( + deleteTool := agent.FunctionTool[struct{}]( "delete_account", "Deletes the user account", func(_ context.Context, _ struct{}) (agent.ToolResult, error) { @@ -1631,7 +1616,6 @@ func TestRun_Approval(t *testing.T) { return agent.ToolResult{Content: "account deleted"}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -1653,7 +1637,7 @@ func TestRun_Approval(t *testing.T) { }), ) - _, err = ag.Run( + _, err := ag.Run( context.Background(), []llm.Message{userMessage("Delete my account")}, ) @@ -1683,7 +1667,7 @@ func TestRun_Approval(t *testing.T) { func(t *testing.T) { t.Parallel() - deleteTool, err := agent.FunctionTool[struct{}]( + deleteTool := agent.FunctionTool[struct{}]( "delete_account", "Deletes the user account", func(_ context.Context, _ struct{}) (agent.ToolResult, error) { @@ -1691,7 +1675,6 @@ func TestRun_Approval(t *testing.T) { return agent.ToolResult{}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -1713,7 +1696,7 @@ func TestRun_Approval(t *testing.T) { }), ) - _, err = ag.Run( + _, err := ag.Run( context.Background(), []llm.Message{userMessage("Delete my account")}, ) @@ -1751,14 +1734,13 @@ func TestRun_Approval(t *testing.T) { }, } - safeTool, err := agent.FunctionTool[struct{}]( + safeTool := agent.FunctionTool[struct{}]( "safe_tool", "A safe tool", func(_ context.Context, _ struct{}) (agent.ToolResult, error) { return agent.ToolResult{Content: "safe result"}, nil }, ) - require.NoError(t, err) ag := agent.New( "assistant", @@ -1789,7 +1771,7 @@ func TestRun_Approval(t *testing.T) { var safeExecuted, dangerExecuted bool - safeTool, err := agent.FunctionTool[struct{}]( + safeTool := agent.FunctionTool[struct{}]( "safe_action", "A safe action", func(_ context.Context, _ struct{}) (agent.ToolResult, error) { @@ -1797,9 +1779,8 @@ func TestRun_Approval(t *testing.T) { return agent.ToolResult{Content: "safe done"}, nil }, ) - require.NoError(t, err) - dangerTool, err := agent.FunctionTool[struct{}]( + dangerTool := agent.FunctionTool[struct{}]( "danger_action", "A dangerous action", func(_ context.Context, _ struct{}) (agent.ToolResult, error) { @@ -1807,7 +1788,6 @@ func TestRun_Approval(t *testing.T) { return agent.ToolResult{Content: "danger done"}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -1835,7 +1815,7 @@ func TestRun_Approval(t *testing.T) { }), ) - _, err = ag.Run( + _, err := ag.Run( context.Background(), []llm.Message{userMessage("Do both")}, ) @@ -1943,14 +1923,13 @@ func TestResume(t *testing.T) { func(t *testing.T) { t.Parallel() - deleteTool, err := agent.FunctionTool[struct{}]( + deleteTool := agent.FunctionTool[struct{}]( "delete_account", "Deletes the user account", func(_ context.Context, _ struct{}) (agent.ToolResult, error) { return agent.ToolResult{Content: "deleted"}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -1972,7 +1951,7 @@ func TestResume(t *testing.T) { }), ) - _, err = ag.Run( + _, err := ag.Run( context.Background(), []llm.Message{userMessage("Delete my account")}, ) @@ -2152,14 +2131,13 @@ func TestRunStreamed(t *testing.T) { t.Parallel() type Params struct{} - tool, err := agent.FunctionTool[Params]( + tool := agent.FunctionTool[Params]( "noop", "No-op", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "ok"}, nil }, ) - require.NoError(t, err) stream1 := &mockChatStream{ events: []llm.ChatCompletionStreamEvent{ @@ -2378,22 +2356,20 @@ func TestClone(t *testing.T) { t.Parallel() type Params struct{} - tool1, err := agent.FunctionTool[Params]( + tool1 := agent.FunctionTool[Params]( "t1", "desc", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "ok"}, nil }, ) - require.NoError(t, err) - tool2, err := agent.FunctionTool[Params]( + tool2 := agent.FunctionTool[Params]( "t2", "desc", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "ok"}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -2500,14 +2476,13 @@ func TestGenerateSchema_EmbeddedStruct(t *testing.T) { Name string `json:"name"` } - tool, err := agent.FunctionTool[Params]( + tool := agent.FunctionTool[Params]( "create", "Create item", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "ok"}, nil }, ) - require.NoError(t, err) var schema map[string]any require.NoError(t, json.Unmarshal(tool.Definition().Parameters, &schema)) @@ -2675,14 +2650,13 @@ func TestRun_UnknownToolCall(t *testing.T) { } type Params struct{} - tool, err := agent.FunctionTool[Params]( + tool := agent.FunctionTool[Params]( "real_tool", "A real tool", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "ok"}, nil }, ) - require.NoError(t, err) ag := agent.New( "assistant", @@ -2691,7 +2665,7 @@ func TestRun_UnknownToolCall(t *testing.T) { agent.WithTools(tool), ) - _, err = ag.Run( + _, err := ag.Run( context.Background(), []llm.Message{userMessage("test")}, ) @@ -2743,14 +2717,13 @@ func TestClone_WithApprovalConfig(t *testing.T) { }, } - deleteTool, err := agent.FunctionTool[struct{}]( + deleteTool := agent.FunctionTool[struct{}]( "delete", "Delete something", func(_ context.Context, _ struct{}) (agent.ToolResult, error) { return agent.ToolResult{Content: "deleted"}, nil }, ) - require.NoError(t, err) original := agent.New( "assistant", @@ -2764,7 +2737,7 @@ func TestClone_WithApprovalConfig(t *testing.T) { cloned := original.Clone() - _, err = cloned.Run( + _, err := cloned.Run( context.Background(), []llm.Message{userMessage("delete it")}, ) @@ -2795,7 +2768,7 @@ func TestRun_HandoffWithPreHandoffTools(t *testing.T) { var executionOrder []string type Params struct{} - tool1, err := agent.FunctionTool[Params]( + tool1 := agent.FunctionTool[Params]( "prepare", "Prepare data", func(_ context.Context, _ Params) (agent.ToolResult, error) { @@ -2803,7 +2776,6 @@ func TestRun_HandoffWithPreHandoffTools(t *testing.T) { return agent.ToolResult{Content: "prepared"}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -2855,15 +2827,14 @@ func TestRun_HandoffWithPreHandoffTools(t *testing.T) { t.Parallel() type Params struct{} - tool1, err := agent.FunctionTool[Params]( + tool1 := agent.FunctionTool[Params]( "prepare", "Prepare data", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "prepared"}, nil }, ) - require.NoError(t, err) - tool2, err := agent.FunctionTool[Params]( + tool2 := agent.FunctionTool[Params]( "finalize", "Finalize data", func(_ context.Context, _ Params) (agent.ToolResult, error) { @@ -2871,7 +2842,6 @@ func TestRun_HandoffWithPreHandoffTools(t *testing.T) { return agent.ToolResult{}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -2941,14 +2911,13 @@ func TestRun_HandoffWithPreHandoffTools(t *testing.T) { t.Parallel() type Params struct{} - failingTool, err := agent.FunctionTool[Params]( + failingTool := agent.FunctionTool[Params]( "prepare", "Prepare data", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{}, errors.New("preparation failed") }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -2981,7 +2950,7 @@ func TestRun_HandoffWithPreHandoffTools(t *testing.T) { agent.WithHandoffs(specialist), ) - _, err = router.Run( + _, err := router.Run( context.Background(), []llm.Message{userMessage("prepare and transfer")}, ) diff --git a/pkg/agent/agent_tool.go b/pkg/agent/agent_tool.go index 51345abab..040dbcce8 100644 --- a/pkg/agent/agent_tool.go +++ b/pkg/agent/agent_tool.go @@ -116,5 +116,20 @@ func (t *agentTool) Execute(ctx context.Context, arguments string) (ToolResult, return ToolResult{}, err } - return ToolResult{Content: result.FinalMessage().Text()}, nil + text := result.FinalMessage().Text() + + if t.agent.outputType != nil { + if !json.Valid([]byte(text)) { + preview := text + if len(preview) > 500 { + preview = preview[:500] + "... (truncated)" + } + return ToolResult{ + Content: fmt.Sprintf("Sub-agent %q returned invalid JSON. Raw output:\n%s", t.agent.name, preview), + IsError: true, + }, nil + } + } + + return ToolResult{Content: text}, nil } diff --git a/pkg/agent/agent_tool_test.go b/pkg/agent/agent_tool_test.go index eb5f55906..f93bec5d1 100644 --- a/pkg/agent/agent_tool_test.go +++ b/pkg/agent/agent_tool_test.go @@ -273,7 +273,7 @@ func TestAgentTool_Execute(t *testing.T) { var captured string type Params struct{} - tenantTool, err := agent.FunctionTool[Params]( + tenantTool := agent.FunctionTool[Params]( "get_tenant", "Get tenant", func(ctx context.Context, _ Params) (agent.ToolResult, error) { @@ -282,7 +282,6 @@ func TestAgentTool_Execute(t *testing.T) { return agent.ToolResult{Content: rc.TenantID}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -325,14 +324,13 @@ func TestAgentTool_Execute(t *testing.T) { Expr string `json:"expr"` } - calcTool, err := agent.FunctionTool[Params]( + calcTool := agent.FunctionTool[Params]( "calc", "Calculate expression", func(_ context.Context, p Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "42"}, nil }, ) - require.NoError(t, err) provider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -413,12 +411,11 @@ func newNestedApprovalFixture( ) nestedApprovalFixture { t.Helper() - deleteTool, err := agent.FunctionTool[struct{}]( + deleteTool := agent.FunctionTool[struct{}]( "delete_file", "Delete a file", deleteFunc, ) - require.NoError(t, err) innerProvider := &mockProvider{responses: innerResponses} @@ -608,7 +605,7 @@ func TestAgentTool_Execute_NestedApproval(t *testing.T) { var siblingCalled bool type Params struct{} - siblingTool, err := agent.FunctionTool[Params]( + siblingTool := agent.FunctionTool[Params]( "list_files", "List files", func(_ context.Context, _ Params) (agent.ToolResult, error) { @@ -616,7 +613,6 @@ func TestAgentTool_Execute_NestedApproval(t *testing.T) { return agent.ToolResult{Content: "file1.txt, file2.txt"}, nil }, ) - require.NoError(t, err) f := newNestedApprovalFixture( t, @@ -644,7 +640,7 @@ func TestAgentTool_Execute_NestedApproval(t *testing.T) { agent.WithTools(siblingTool), ) - _, err = f.outerAgent.Run( + _, err := f.outerAgent.Run( context.Background(), []llm.Message{userMessage("List and delete files")}, ) @@ -675,7 +671,7 @@ func TestAgentTool_Execute_NestedApproval(t *testing.T) { var toolExecuted bool - dangerTool, err := agent.FunctionTool[struct{}]( + dangerTool := agent.FunctionTool[struct{}]( "danger", "Dangerous operation", func(_ context.Context, _ struct{}) (agent.ToolResult, error) { @@ -683,7 +679,6 @@ func TestAgentTool_Execute_NestedApproval(t *testing.T) { return agent.ToolResult{Content: "danger executed"}, nil }, ) - require.NoError(t, err) cProvider := &mockProvider{ responses: []*llm.ChatCompletionResponse{ @@ -739,7 +734,7 @@ func TestAgentTool_Execute_NestedApproval(t *testing.T) { agent.WithTools(agentB.AsTool("call_b", "Call agent B")), ) - _, err = agentA.Run( + _, err := agentA.Run( context.Background(), []llm.Message{userMessage("start")}, ) diff --git a/pkg/agent/model_settings.go b/pkg/agent/model_settings.go index 95980361e..49712fc3c 100644 --- a/pkg/agent/model_settings.go +++ b/pkg/agent/model_settings.go @@ -24,4 +24,5 @@ type ModelSettings struct { MaxTokens *int ToolChoice *llm.ToolChoice ParallelToolCalls *bool + Thinking *llm.ThinkingConfig } diff --git a/pkg/agent/progress.go b/pkg/agent/progress.go new file mode 100644 index 000000000..5ad3a4345 --- /dev/null +++ b/pkg/agent/progress.go @@ -0,0 +1,36 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package agent + +import "context" + +type ( + ProgressEventType string + + ProgressEvent struct { + Type ProgressEventType `json:"type"` + Step string `json:"step"` + ParentStep string `json:"parent_step,omitempty"` + Message string `json:"message"` + } + + ProgressReporter func(ctx context.Context, event ProgressEvent) +) + +const ( + ProgressEventStepStarted ProgressEventType = "step_started" + ProgressEventStepCompleted ProgressEventType = "step_completed" + ProgressEventStepFailed ProgressEventType = "step_failed" +) diff --git a/pkg/agent/run.go b/pkg/agent/run.go index 88e294308..6926d5e13 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -28,7 +28,14 @@ import ( "go.probo.inc/probo/pkg/llm" ) -const tracerName = "go.probo.inc/probo/pkg/agent" +const ( + tracerName = "go.probo.inc/probo/pkg/agent" + + // synthesisNudge is the static user message appended after tool + // exploration completes, asking the model to produce the final + // structured output on the next (synthesis) turn. + synthesisNudge = "Based on everything you have gathered, produce the final structured output now." +) type ( CallLLMFunc func(ctx context.Context, agent *Agent, req *llm.ChatCompletionRequest) (*llm.ChatCompletionResponse, error) @@ -68,7 +75,32 @@ type ( func noopEvent(_ context.Context, _ StreamEvent) {} func blockingCallLLM(ctx context.Context, agent *Agent, req *llm.ChatCompletionRequest) (*llm.ChatCompletionResponse, error) { - return agent.client.ChatCompletion(ctx, req) + resp, err := agent.client.ChatCompletion(ctx, req) + if err == nil { + return resp, nil + } + + // Some providers (e.g. Anthropic) require streaming for large + // max_tokens or when thinking is enabled. Fall back to streaming + // transparently when the blocking call returns ErrStreamingRequired. + var streamRequired *llm.ErrStreamingRequired + if !errors.As(err, &streamRequired) { + return nil, err + } + + stream, sErr := agent.client.ChatCompletionStream(ctx, req) + if sErr != nil { + return nil, err // return the original error + } + defer stream.Close() + + acc := llm.NewStreamAccumulator(stream) + for acc.Next() { + } + if sErr := acc.Err(); sErr != nil { + return nil, sErr + } + return acc.Response(), nil } func (a *Agent) Run(ctx context.Context, messages []llm.Message) (*Result, error) { @@ -273,6 +305,24 @@ func coreLoop(ctx context.Context, startAgent *Agent, inputMessages []llm.Messag log.Int("tool_count", len(s.toolDefs)), ) + emptyOutputRetries := 0 + + structuredFormat := resolveStructuredFormat(s.agent) + + // When the agent has both tools and a structured output request, + // we delay structured output enforcement until a dedicated + // synthesis turn. Enforcing the schema during tool exploration + // causes models with extended thinking to stuff planning prose + // into the first text field of the schema as a scratchpad, + // burning the entire max_tokens budget on thinking-inside-JSON + // before ever producing a valid object. Instead, we let the + // model freely call tools without a schema, then force one final + // synthesis turn with ToolChoice=none + schema enforced once the + // model signals it has enough information (finish_reason=stop). + // Agents without tools or without a structured output request + // do not need this dance and enforce the schema immediately. + exploring := structuredFormat != nil && len(s.toolDefs) > 0 + for { if err := ctx.Err(); err != nil { return s.finishRun(ctx, nil, fmt.Errorf("cannot complete: %w", err)) @@ -284,15 +334,21 @@ func coreLoop(ctx context.Context, startAgent *Agent, inputMessages []llm.Messag fullMessages := buildFullMessages(s.systemPrompt, s.messages) - responseFormat := s.agent.responseFormat - if responseFormat == nil && s.agent.outputType != nil { - responseFormat = s.agent.outputType.responseFormat() + var responseFormat *llm.ResponseFormat + if !exploring { + responseFormat = structuredFormat } toolChoice := s.agent.modelSettings.ToolChoice if s.toolUsedInRun && s.agent.resetToolChoice && toolChoice != nil { toolChoice = nil } + if !exploring && structuredFormat != nil && len(s.toolDefs) > 0 { + // On the synthesis turn, forbid further tool calls so the + // model is forced to convert what it has into JSON. + none := llm.ToolChoice{Type: llm.ToolChoiceNone} + toolChoice = &none + } req := &llm.ChatCompletionRequest{ Model: s.agent.model, @@ -306,6 +362,7 @@ func coreLoop(ctx context.Context, startAgent *Agent, inputMessages []llm.Messag ToolChoice: toolChoice, ParallelToolCalls: s.agent.modelSettings.ParallelToolCalls, ResponseFormat: responseFormat, + Thinking: s.agent.modelSettings.Thinking, } s.logger.InfoCtx( @@ -336,6 +393,62 @@ func coreLoop(ctx context.Context, startAgent *Agent, inputMessages []llm.Messag switch resp.FinishReason { case llm.FinishReasonStop, llm.FinishReasonLength: + // Model signalled it has nothing more to do with tools. + // If we have a structured output request but haven't + // enforced the schema yet, promote this turn to the + // synthesis turn: the next iteration runs with + // ToolChoice=none and the schema enforced, so the model + // converts what it has gathered into JSON in one shot. + // + // Anthropic requires the last message in the conversation + // to be a user message, so we cannot simply continue after + // an assistant stop turn. Drop empty (thinking-only) turns + // from history and append a user nudge that asks for the + // final structured output. Non-empty assistant turns stay + // in history so the model can reference its own + // conclusions during synthesis. + if exploring && s.turns < s.agent.maxTurns { + exploring = false + if resp.Message.Text() == "" { + s.messages = s.messages[:len(s.messages)-1] + } + s.messages = append( + s.messages, + llm.Message{ + Role: llm.RoleUser, + Parts: []llm.Part{llm.TextPart{Text: synthesisNudge}}, + }, + ) + s.logger.WarnCtx( + ctx, + "entering synthesis turn: forcing structured output with tool_choice=none", + log.Int("turn", s.turns), + log.Int("output_tokens", resp.Usage.OutputTokens), + ) + continue + } + + // Anthropic extended-thinking models can return a synthesis turn + // that contains only thinking blocks and no text part, leaving us + // with no structured output to validate. Retry the same turn a + // bounded number of times so the model gets another chance to + // emit the required JSON output. The empty assistant turn must be + // dropped from history because Anthropic rejects requests where + // the last message is a thinking-only assistant turn. + if structuredFormat != nil && resp.Message.Text() == "" && emptyOutputRetries < s.agent.maxEmptyOutputRetries && s.turns < s.agent.maxTurns { + emptyOutputRetries++ + s.messages = s.messages[:len(s.messages)-1] + s.logger.WarnCtx( + ctx, + "retrying turn: structured output expected but got empty text", + log.Int("turn", s.turns), + log.Int("retry", emptyOutputRetries), + log.Int("output_tokens", resp.Usage.OutputTokens), + ) + continue + } + emptyOutputRetries = 0 + if err := runOutputGuardrails(ctx, s.agent, resp.Message); err != nil { return s.finishRun(ctx, nil, err) } @@ -354,6 +467,7 @@ func coreLoop(ctx context.Context, startAgent *Agent, inputMessages []llm.Messag case llm.FinishReasonToolCalls: s.toolUsedInRun = true + emptyOutputRetries = 0 s.logger.InfoCtx( ctx, @@ -442,7 +556,8 @@ func coreLoop(ctx context.Context, startAgent *Agent, inputMessages []llm.Messag if isFinal { s.messages = append( - s.messages, llm.Message{ + s.messages, + llm.Message{ Role: llm.RoleAssistant, Parts: []llm.Part{llm.TextPart{Text: finalOutput}}, }, @@ -852,12 +967,24 @@ func executeSingleTool( emitHook(agent, func(h RunHooks) { h.OnToolEnd(ctx, agent, tool, result, nil) }) emitAgentHook(agent, func(h AgentHooks) { h.OnToolEnd(ctx, agent, tool, result) }) - logger.InfoCtx( - ctx, - "tool execution completed", - log.String("tool", tool.Name()), - log.Bool("is_error", result.IsError), - ) + if result.IsError { + content := result.Content + if len(content) > 200 { + content = content[:200] + "... (truncated)" + } + logger.WarnCtx( + ctx, + "tool returned error", + log.String("tool", tool.Name()), + log.String("content", content), + ) + } else { + logger.InfoCtx( + ctx, + "tool execution completed", + log.String("tool", tool.Name()), + ) + } return result, nil } @@ -1178,3 +1305,18 @@ func emitAgentHook(agent *Agent, fn func(AgentHooks)) { fn(agent.agentHooks) } } + +// resolveStructuredFormat returns the structured output request the +// agent wants enforced on its final turn, or nil if none. An agent can +// declare structured output through either WithOutputType (typed +// sub-agents) or a directly-set responseFormat (the RunTyped +// convenience wrapper). +func resolveStructuredFormat(a *Agent) *llm.ResponseFormat { + if a.responseFormat != nil { + return a.responseFormat + } + if a.outputType != nil { + return a.outputType.responseFormat() + } + return nil +} diff --git a/pkg/agent/tool.go b/pkg/agent/tool.go index daf0613b2..c898aaf0e 100644 --- a/pkg/agent/tool.go +++ b/pkg/agent/tool.go @@ -29,6 +29,7 @@ type ( IsError bool } + // ToolDescriptor describes a tool's name and LLM definition. ToolDescriptor interface { Name() string Definition() llm.Tool @@ -38,7 +39,31 @@ type ( ToolDescriptor Execute(ctx context.Context, arguments string) (ToolResult, error) } +) +// ResultJSON marshals v to JSON and returns a successful ToolResult. +func ResultJSON(v any) ToolResult { + data, err := json.Marshal(v) + if err != nil { + return ToolResult{ + Content: fmt.Sprintf("cannot marshal tool result: %s", err), + IsError: true, + } + } + return ToolResult{Content: string(data)} +} + +// ResultError returns an error ToolResult with the given message. +func ResultError(msg string) ToolResult { + return ToolResult{Content: msg, IsError: true} +} + +// ResultErrorf returns an error ToolResult with a formatted message. +func ResultErrorf(format string, args ...any) ToolResult { + return ToolResult{Content: fmt.Sprintf(format, args...), IsError: true} +} + +type ( functionTool[P any] struct { name string description string @@ -48,20 +73,30 @@ type ( } ) +// FunctionTool creates a tool whose parameters are typed by P. The JSON +// schema advertised to the LLM is generated from P at construction time. +// +// Schema generation is derived from a compile-time Go type: a failure +// here is a programmer error (bad struct tag, unsupported type), not a +// runtime condition, so we panic rather than returning an error. The +// same applies to the required-fields metadata parsed back out of the +// generated schema. func FunctionTool[P any]( name string, description string, fn func(ctx context.Context, params P) (ToolResult, error), -) (Tool, error) { +) Tool { schema, err := jsonSchemaFor[P]() if err != nil { - return nil, fmt.Errorf("cannot create tool %q: %w", name, err) + panic(fmt.Sprintf("agent: cannot generate JSON schema for tool %q: %s", name, err)) } var parsed struct { Required []string `json:"required"` } - _ = json.Unmarshal(schema, &parsed) + if err := json.Unmarshal(schema, &parsed); err != nil { + panic(fmt.Sprintf("agent: cannot parse generated schema for tool %q: %s", name, err)) + } return &functionTool[P]{ name: name, @@ -69,7 +104,7 @@ func FunctionTool[P any]( fn: fn, schema: schema, requiredFields: parsed.Required, - }, nil + } } func (t *functionTool[P]) Name() string { return t.name } diff --git a/pkg/agent/tool_test.go b/pkg/agent/tool_test.go index 4a9b8a4d8..76cee47eb 100644 --- a/pkg/agent/tool_test.go +++ b/pkg/agent/tool_test.go @@ -30,14 +30,13 @@ func TestFunctionTool_Name(t *testing.T) { type Params struct{} - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "my_tool", "does things", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{}, nil }, ) - require.NoError(t, err) assert.Equal(t, "my_tool", tool.Name()) } @@ -54,14 +53,13 @@ func TestFunctionTool_Definition(t *testing.T) { Query string `json:"query"` } - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "search", "Search for items", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{}, nil }, ) - require.NoError(t, err) def := tool.Definition() assert.Equal(t, "search", def.Name) @@ -79,14 +77,13 @@ func TestFunctionTool_Definition(t *testing.T) { Count int `json:"count"` } - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "create", "Create items", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{}, nil }, ) - require.NoError(t, err) def := tool.Definition() require.NotNil(t, def.Parameters) @@ -116,14 +113,13 @@ func TestFunctionTool_Definition(t *testing.T) { type Params struct{} - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "noop", "No-op", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{}, nil }, ) - require.NoError(t, err) var schema map[string]any require.NoError(t, json.Unmarshal(tool.Definition().Parameters, &schema)) @@ -140,14 +136,13 @@ func TestFunctionTool_Definition(t *testing.T) { Title *string `json:"title,omitempty"` } - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "update", "Update", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{}, nil }, ) - require.NoError(t, err) var schema map[string]any require.NoError(t, json.Unmarshal(tool.Definition().Parameters, &schema)) @@ -173,14 +168,13 @@ func TestFunctionTool_Execute(t *testing.T) { Y int `json:"y"` } - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "add", "Add two numbers", func(_ context.Context, p Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "42"}, nil }, ) - require.NoError(t, err) result, err := tool.Execute(context.Background(), `{"x": 1, "y": 2}`) require.NoError(t, err) @@ -199,7 +193,7 @@ func TestFunctionTool_Execute(t *testing.T) { } var received string - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "weather", "Get weather", func(_ context.Context, p Params) (agent.ToolResult, error) { @@ -207,9 +201,8 @@ func TestFunctionTool_Execute(t *testing.T) { return agent.ToolResult{Content: "sunny"}, nil }, ) - require.NoError(t, err) - _, err = tool.Execute(context.Background(), `{"city":"Paris"}`) + _, err := tool.Execute(context.Background(), `{"city":"Paris"}`) require.NoError(t, err) assert.Equal(t, "Paris", received) }, @@ -222,14 +215,13 @@ func TestFunctionTool_Execute(t *testing.T) { type Params struct{} - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "noop", "No-op", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "ok"}, nil }, ) - require.NoError(t, err) result, err := tool.Execute(context.Background(), `{invalid`) require.NoError(t, err) @@ -245,16 +237,15 @@ func TestFunctionTool_Execute(t *testing.T) { type Params struct{} - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "fail", "Always fails", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{}, errors.New("db down") }, ) - require.NoError(t, err) - _, err = tool.Execute(context.Background(), `{}`) + _, err := tool.Execute(context.Background(), `{}`) require.Error(t, err) assert.Contains(t, err.Error(), "db down") }, @@ -268,7 +259,7 @@ func TestFunctionTool_Execute(t *testing.T) { type ctxKey struct{} type Params struct{} - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "ctx_check", "Check context", func(ctx context.Context, _ Params) (agent.ToolResult, error) { @@ -276,7 +267,6 @@ func TestFunctionTool_Execute(t *testing.T) { return agent.ToolResult{Content: val}, nil }, ) - require.NoError(t, err) ctx := context.WithValue(context.Background(), ctxKey{}, "hello") result, err := tool.Execute(ctx, `{}`) @@ -294,7 +284,7 @@ func TestFunctionTool_Execute(t *testing.T) { City string `json:"city"` } - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "weather", "Get weather", func(_ context.Context, _ Params) (agent.ToolResult, error) { @@ -302,7 +292,6 @@ func TestFunctionTool_Execute(t *testing.T) { return agent.ToolResult{}, nil }, ) - require.NoError(t, err) result, err := tool.Execute(context.Background(), `{}`) require.NoError(t, err) @@ -321,7 +310,7 @@ func TestFunctionTool_Execute(t *testing.T) { Country string `json:"country"` } - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "weather", "Get weather", func(_ context.Context, _ Params) (agent.ToolResult, error) { @@ -329,7 +318,6 @@ func TestFunctionTool_Execute(t *testing.T) { return agent.ToolResult{}, nil }, ) - require.NoError(t, err) result, err := tool.Execute(context.Background(), `{}`) require.NoError(t, err) @@ -349,7 +337,7 @@ func TestFunctionTool_Execute(t *testing.T) { Country string `json:"country"` } - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "weather", "Get weather", func(_ context.Context, _ Params) (agent.ToolResult, error) { @@ -357,7 +345,6 @@ func TestFunctionTool_Execute(t *testing.T) { return agent.ToolResult{}, nil }, ) - require.NoError(t, err) result, err := tool.Execute(context.Background(), `{"city":"Paris"}`) require.NoError(t, err) @@ -376,14 +363,13 @@ func TestFunctionTool_Execute(t *testing.T) { Units *string `json:"units,omitempty"` } - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "weather", "Get weather", func(_ context.Context, p Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "sunny in " + p.City}, nil }, ) - require.NoError(t, err) result, err := tool.Execute(context.Background(), `{"city":"Paris"}`) require.NoError(t, err) @@ -401,14 +387,13 @@ func TestFunctionTool_Execute(t *testing.T) { Name string `json:"name"` } - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "greet", "Greet", func(_ context.Context, p Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "hi " + p.Name}, nil }, ) - require.NoError(t, err) result, err := tool.Execute(context.Background(), `{"name":"Alice","extra":"ignored"}`) require.NoError(t, err) @@ -424,14 +409,13 @@ func TestFunctionTool_Execute(t *testing.T) { type Params struct{} - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "ping", "Ping", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "pong"}, nil }, ) - require.NoError(t, err) result, err := tool.Execute(context.Background(), `{}`) require.NoError(t, err) @@ -446,14 +430,13 @@ func TestFunctionTool_Execute(t *testing.T) { type Params struct{} - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "validate", "Validate input", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{Content: "validation failed", IsError: true}, nil }, ) - require.NoError(t, err) result, err := tool.Execute(context.Background(), `{}`) require.NoError(t, err) @@ -468,14 +451,13 @@ func TestFunctionTool_InterfaceSatisfaction(t *testing.T) { type Params struct{} - tool, err := agent.FunctionTool( + tool := agent.FunctionTool( "test", "test tool", func(_ context.Context, _ Params) (agent.ToolResult, error) { return agent.ToolResult{}, nil }, ) - require.NoError(t, err) assert.Implements(t, (*agent.Tool)(nil), tool) assert.Implements(t, (*agent.ToolDescriptor)(nil), tool) diff --git a/pkg/agent/tools/browser/browser.go b/pkg/agent/tools/browser/browser.go new file mode 100644 index 000000000..e58ae654b --- /dev/null +++ b/pkg/agent/tools/browser/browser.go @@ -0,0 +1,170 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + "time" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" + "go.probo.inc/probo/pkg/agent/tools/internal/netcheck" +) + +const ( + defaultToolTimeout = 60 * time.Second +) + +type Browser struct { + addr string + allocCtx context.Context + cancel context.CancelFunc + allowedDomains []string +} + +func NewBrowser(ctx context.Context, addr string) *Browser { + if !strings.HasPrefix(addr, "ws://") && !strings.HasPrefix(addr, "wss://") { + addr = "ws://" + addr + } + + allocCtx, cancel := chromedp.NewRemoteAllocator(ctx, addr) + + return &Browser{ + addr: addr, + allocCtx: allocCtx, + cancel: cancel, + } +} + +// SetAllowedDomain restricts navigation to URLs under the given domain and +// its subdomains. For example, setting "getprobo.com" allows navigation to +// getprobo.com, www.getprobo.com, and compliance.getprobo.com. +// This replaces any previously set domains. +func (b *Browser) SetAllowedDomain(domain string) { + domain = strings.ToLower(strings.TrimSpace(domain)) + + // Strip "www." prefix so that setting either "www.example.com" or + // "example.com" allows navigation to *.example.com. + domain = strings.TrimPrefix(domain, "www.") + + b.allowedDomains = []string{domain} +} + +// checkURL validates that the URL is allowed. It returns an error tool result +// if the URL uses a disallowed scheme, resolves to a non-public IP, or is +// outside the allowed domains. +func (b *Browser) checkURL(rawURL string) *agent.ToolResult { + u, err := url.Parse(rawURL) + if err != nil { + return &agent.ToolResult{ + Content: fmt.Sprintf("invalid URL: %s", err), + IsError: true, + } + } + + if u.Scheme != "http" && u.Scheme != "https" { + return &agent.ToolResult{ + Content: fmt.Sprintf("cannot navigate to URL with scheme %q: only http and https are allowed", u.Scheme), + IsError: true, + } + } + + // Always reject URLs that resolve to non-public IPs, even when no + // allowed-domain list is set. This closes the SSRF path on browsers + // used for open-ended external research (e.g. the research browser + // in vendor assessments). + if err := netcheck.ValidatePublicURL(rawURL); err != nil { + return &agent.ToolResult{ + Content: fmt.Sprintf("navigation blocked: %s", err), + IsError: true, + } + } + + if len(b.allowedDomains) == 0 { + return nil + } + + host := strings.ToLower(u.Hostname()) + for _, allowed := range b.allowedDomains { + if host == allowed || strings.HasSuffix(host, "."+allowed) { + return nil + } + } + + return &agent.ToolResult{ + Content: fmt.Sprintf("navigation blocked: %s is outside the allowed domains", host), + IsError: true, + } +} + +// checkAlive returns a tool error result if the browser connection has been +// lost. Call this at the start of every tool to fail fast with a clear +// message instead of waiting for the tool timeout. +func (b *Browser) checkAlive() *agent.ToolResult { + if err := b.allocCtx.Err(); err != nil { + return &agent.ToolResult{ + Content: "browser connection lost: the remote Chrome instance is no longer reachable", + IsError: true, + } + } + return nil +} + +// classifyError inspects the caller's timeout context and the browser's +// allocator context to produce a human-readable error message. Without this, +// both a tool timeout and a dropped Chrome connection appear as the opaque +// "context canceled". +func (b *Browser) classifyError(timeoutCtx context.Context, rawURL string, err error) string { + if b.allocCtx.Err() != nil { + return fmt.Sprintf( + "browser connection lost while loading %s: the remote Chrome instance is no longer reachable", + rawURL, + ) + } + + if errors.Is(timeoutCtx.Err(), context.DeadlineExceeded) { + return fmt.Sprintf( + "page load timed out after %s for %s: the page may be too slow or unresponsive", + defaultToolTimeout, + rawURL, + ) + } + + return fmt.Sprintf("cannot load %s: %s", rawURL, err) +} + +func (b *Browser) NewTab(ctx context.Context) (context.Context, context.CancelFunc) { + tabCtx, tabCancel := chromedp.NewContext(b.allocCtx) + + // Propagate the caller's cancellation to the Chrome tab so that + // tool-level timeouts and context deadlines actually stop the browser. + go func() { + select { + case <-ctx.Done(): + tabCancel() + case <-tabCtx.Done(): + } + }() + + return tabCtx, tabCancel +} + +func (b *Browser) Close() { + b.cancel() +} diff --git a/pkg/agent/tools/browser/click.go b/pkg/agent/tools/browser/click.go new file mode 100644 index 000000000..ccc36713a --- /dev/null +++ b/pkg/agent/tools/browser/click.go @@ -0,0 +1,88 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" +) + +type ( + clickParams struct { + URL string `json:"url" jsonschema:"The URL to navigate to before clicking"` + Selector string `json:"selector" jsonschema:"CSS selector of the element to click (e.g. button.next, a[href*=page])"` + } +) + +func ClickElementTool(b *Browser) agent.Tool { + return agent.FunctionTool( + "click_element", + "Navigate to a URL, click an element matching a CSS selector, and return the page text after the click. Useful for pagination buttons, 'show all' links, tabs, and other interactive elements.", + func(ctx context.Context, p clickParams) (agent.ToolResult, error) { + if r := b.checkAlive(); r != nil { + return *r, nil + } + + if r := b.checkURL(p.URL); r != nil { + return *r, nil + } + + ctx, timeoutCancel := withToolTimeout(ctx) + defer timeoutCancel() + + tabCtx, cancel := b.NewTab(ctx) + defer cancel() + + var ( + text string + postClickURL string + ) + + err := chromedp.Run( + tabCtx, + chromedp.Navigate(p.URL), + waitForPage(), + chromedp.WaitVisible(p.Selector), + chromedp.Click(p.Selector), + waitForPage(), + chromedp.Location(&postClickURL), + chromedp.Evaluate(`document.body.innerText`, &text), + ) + if err != nil { + return agent.ResultError(b.classifyError(ctx, p.URL, err)), nil + } + + // Revalidate the post-click URL: a click may navigate + // the page to a different host (redirect, JS navigation, + // ), bypassing the initial checkURL. Reject the + // result if the new URL is outside the allowed scope or + // resolves to a non-public IP. + if postClickURL != "" && postClickURL != p.URL { + if r := b.checkURL(postClickURL); r != nil { + return *r, nil + } + } + + runes := []rune(text) + if len(runes) > maxTextLength { + text = string(runes[:maxTextLength]) + } + + return agent.ToolResult{Content: text}, nil + }, + ) +} diff --git a/pkg/agent/tools/browser/download_pdf.go b/pkg/agent/tools/browser/download_pdf.go new file mode 100644 index 000000000..b48fbc07e --- /dev/null +++ b/pkg/agent/tools/browser/download_pdf.go @@ -0,0 +1,157 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/pdfcpu/pdfcpu/pkg/api" + "github.com/pdfcpu/pdfcpu/pkg/pdfcpu/model" + "go.probo.inc/probo/pkg/agent" + "go.probo.inc/probo/pkg/agent/tools/internal/netcheck" +) + +type ( + downloadPDFParams struct { + URL string `json:"url" jsonschema:"The URL of the PDF document to download and extract text from"` + } + + downloadPDFResult struct { + Text string `json:"text"` + PageCount int `json:"page_count"` + ErrorDetail string `json:"error_detail,omitempty"` + } +) + +func DownloadPDFTool() agent.Tool { + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: netcheck.NewPinnedTransport(), + } + + return agent.FunctionTool( + "download_pdf", + "Download a PDF document from a URL and extract its text content. Use this for DPAs, SOC 2 reports, privacy policies, and other documents hosted as PDFs.", + func(ctx context.Context, p downloadPDFParams) (agent.ToolResult, error) { + if err := validatePublicURL(p.URL); err != nil { + return agent.ResultJSON(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("URL not allowed: %s", err), + }), nil + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.URL, nil) + if err != nil { + return agent.ResultJSON(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot create request: %s", err), + }), nil + } + + resp, err := client.Do(req) + if err != nil { + return agent.ResultJSON(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot download PDF: %s", err), + }), nil + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return agent.ResultJSON(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("PDF download returned status %d", resp.StatusCode), + }), nil + } + + // Read PDF into memory (max 20MB). + body, err := io.ReadAll(io.LimitReader(resp.Body, 20*1024*1024)) + if err != nil { + return agent.ResultJSON(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot read PDF body: %s", err), + }), nil + } + + // Write to temp file for pdfcpu. + tmpDir, err := os.MkdirTemp("", "pdf-extract-*") + if err != nil { + return agent.ResultJSON(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot create temp dir: %s", err), + }), nil + } + defer os.RemoveAll(tmpDir) + + tmpFile := filepath.Join(tmpDir, "input.pdf") + if err := os.WriteFile(tmpFile, body, 0o600); err != nil { + return agent.ResultJSON(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot write temp file: %s", err), + }), nil + } + + // Get page count. + conf := model.NewDefaultConfiguration() + pageCount, err := api.PageCountFile(tmpFile) + if err != nil { + return agent.ResultJSON(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot read PDF: %s", err), + }), nil + } + + // Extract content to output dir. + outDir := filepath.Join(tmpDir, "out") + if err := os.MkdirAll(outDir, 0o700); err != nil { + return agent.ResultJSON(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot create output dir: %s", err), + }), nil + } + + reader := bytes.NewReader(body) + if err := api.ExtractContent(reader, outDir, "content", nil, conf); err != nil { + return agent.ResultJSON(downloadPDFResult{ + ErrorDetail: fmt.Sprintf("cannot extract PDF content: %s", err), + }), nil + } + + // Read all extracted content files. + var sb strings.Builder + entries, _ := os.ReadDir(outDir) + for _, entry := range entries { + if entry.IsDir() { + continue + } + content, err := os.ReadFile(filepath.Join(outDir, entry.Name())) + if err != nil { + continue + } + sb.Write(content) + sb.WriteString("\n") + } + + text := sb.String() + if len(text) > maxTextLength { + text = text[:maxTextLength] + "\n[... truncated]" + } + + return agent.ResultJSON(downloadPDFResult{ + Text: text, + PageCount: pageCount, + }), nil + }, + ) +} diff --git a/pkg/agent/tools/browser/extract_links.go b/pkg/agent/tools/browser/extract_links.go new file mode 100644 index 000000000..3bb982f16 --- /dev/null +++ b/pkg/agent/tools/browser/extract_links.go @@ -0,0 +1,81 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + "net/url" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" +) + +type ( + extractLinksParams struct { + URL string `json:"url" jsonschema:"The URL to extract links from"` + } + + link struct { + Href string `json:"href"` + Text string `json:"text"` + } +) + +func ExtractLinksTool(b *Browser) agent.Tool { + return agent.FunctionTool( + "extract_links", + "Navigate to a URL and extract all links ( elements) with their href and text.", + func(ctx context.Context, p extractLinksParams) (agent.ToolResult, error) { + if r := b.checkAlive(); r != nil { + return *r, nil + } + + u, err := url.Parse(p.URL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") { + return agent.ResultError("invalid URL scheme: only http and https are allowed"), nil + } + + if r := b.checkURL(p.URL); r != nil { + return *r, nil + } + + ctx, timeoutCancel := withToolTimeout(ctx) + defer timeoutCancel() + + tabCtx, cancel := b.NewTab(ctx) + defer cancel() + + var links []link + + err = chromedp.Run( + tabCtx, + chromedp.Navigate(p.URL), + waitForPage(), + chromedp.Evaluate( + `Array.from(document.querySelectorAll("a[href]")).map(a => ({ + href: a.href, + text: a.innerText.trim().substring(0, 200) + }))`, + &links, + ), + ) + if err != nil { + return agent.ResultError(b.classifyError(ctx, p.URL, err)), nil + } + + return agent.ResultJSON(links), nil + }, + ) +} diff --git a/pkg/agent/tools/browser/extract_text.go b/pkg/agent/tools/browser/extract_text.go new file mode 100644 index 000000000..5db7b5c33 --- /dev/null +++ b/pkg/agent/tools/browser/extract_text.go @@ -0,0 +1,95 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + "fmt" + "time" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" +) + +const ( + maxTextLength = 32000 +) + +type ( + extractTextParams struct { + URL string `json:"url" jsonschema:"The URL to extract text from"` + } +) + +func ExtractPageTextTool(b *Browser) agent.Tool { + return agent.FunctionTool( + "extract_page_text", + "Navigate to a URL and extract the visible text content of the page, truncated to 32000 characters.", + func(ctx context.Context, p extractTextParams) (agent.ToolResult, error) { + if r := b.checkAlive(); r != nil { + return *r, nil + } + + if r := b.checkURL(p.URL); r != nil { + return *r, nil + } + + if r := checkPDF(p.URL); r != nil { + return *r, nil + } + + ctx, timeoutCancel := withToolTimeout(ctx) + defer timeoutCancel() + + tabCtx, cancel := b.NewTab(ctx) + defer cancel() + + var text string + + // Cap the JS-side slice at 4 code units per rune so the + // DevTools transfer stays bounded even for huge pages; + // the Go-side rune truncation below then produces the + // final exact-length output. + jsMaxLen := maxTextLength * 4 + extractJS := fmt.Sprintf( + `String(document.body?.innerText ?? '').slice(0, %d)`, + jsMaxLen, + ) + + err := chromedp.Run( + tabCtx, + chromedp.Navigate(p.URL), + waitForPage(), + // Scroll to bottom to trigger lazy-loaded content, + // then back to top and wait briefly for rendering. + chromedp.Evaluate(`window.scrollTo(0, document.body.scrollHeight)`, nil), + chromedp.Sleep(500*time.Millisecond), + chromedp.Evaluate(`window.scrollTo(0, 0)`, nil), + chromedp.Sleep(200*time.Millisecond), + chromedp.Evaluate(extractJS, &text), + ) + if err != nil { + return agent.ResultError(b.classifyError(ctx, p.URL, err)), nil + } + + runes := []rune(text) + if len(runes) > maxTextLength { + text = string(runes[:maxTextLength]) + } + + return agent.ToolResult{Content: text}, nil + }, + ) +} diff --git a/pkg/agent/tools/browser/fetch_robots.go b/pkg/agent/tools/browser/fetch_robots.go new file mode 100644 index 000000000..ad200c261 --- /dev/null +++ b/pkg/agent/tools/browser/fetch_robots.go @@ -0,0 +1,107 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "bufio" + "context" + "fmt" + "net/http" + "strings" + "time" + + "go.probo.inc/probo/pkg/agent" +) + +type ( + robotsParams struct { + Domain string `json:"domain" jsonschema:"The domain to fetch robots.txt from (e.g. example.com)"` + } + + robotsResult struct { + Found bool `json:"found"` + Sitemaps []string `json:"sitemaps,omitempty"` + Disallowed []string `json:"disallowed_paths,omitempty"` + ErrorDetail string `json:"error_detail,omitempty"` + } +) + +func FetchRobotsTxtTool() agent.Tool { + client := &http.Client{Timeout: 10 * time.Second} + + return agent.FunctionTool( + "fetch_robots_txt", + "Fetch and parse the robots.txt file for a domain. Returns sitemap URLs and disallowed paths, which can reveal hidden pages the crawler might miss.", + func(ctx context.Context, p robotsParams) (agent.ToolResult, error) { + if err := validatePublicDomain(p.Domain); err != nil { + return agent.ResultJSON(robotsResult{ + Found: false, + ErrorDetail: fmt.Sprintf("domain not allowed: %s", err), + }), nil + } + + u := "https://" + p.Domain + "/robots.txt" + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return agent.ResultJSON(robotsResult{ + Found: false, + ErrorDetail: fmt.Sprintf("cannot create request: %s", err), + }), nil + } + + resp, err := client.Do(req) + if err != nil { + return agent.ResultJSON(robotsResult{ + Found: false, + ErrorDetail: fmt.Sprintf("cannot fetch robots.txt: %s", err), + }), nil + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return agent.ResultJSON(robotsResult{ + Found: false, + ErrorDetail: fmt.Sprintf("robots.txt returned status %d", resp.StatusCode), + }), nil + } + + var result robotsResult + result.Found = true + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + // Directive names are case-insensitive but values + // (URLs, paths) are case-sensitive, so extract the + // original-case suffix from the raw line rather than + // reading it off the lowercased copy. + if after, ok := strings.CutPrefix(strings.ToLower(line), "sitemap:"); ok { + result.Sitemaps = append(result.Sitemaps, strings.TrimSpace(line[len(line)-len(after):])) + } + + if after, ok := strings.CutPrefix(strings.ToLower(line), "disallow:"); ok { + path := strings.TrimSpace(line[len(line)-len(after):]) + if path != "" && len(result.Disallowed) < 50 { + result.Disallowed = append(result.Disallowed, path) + } + } + } + + return agent.ResultJSON(result), nil + }, + ) +} diff --git a/pkg/agent/tools/browser/fetch_sitemap.go b/pkg/agent/tools/browser/fetch_sitemap.go new file mode 100644 index 000000000..3fd006030 --- /dev/null +++ b/pkg/agent/tools/browser/fetch_sitemap.go @@ -0,0 +1,151 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "compress/gzip" + "context" + "encoding/xml" + "fmt" + "io" + "net/http" + "strings" + "time" + + "go.probo.inc/probo/pkg/agent" +) + +type ( + sitemapParams struct { + URL string `json:"url" jsonschema:"The full URL of the sitemap to fetch (e.g. https://example.com/sitemap.xml)"` + } + + sitemapResult struct { + Found bool `json:"found"` + URLs []string `json:"urls,omitempty"` + URLCount int `json:"url_count"` + ErrorDetail string `json:"error_detail,omitempty"` + } +) + +const ( + maxSitemapURLs = 200 +) + +func FetchSitemapTool() agent.Tool { + client := &http.Client{Timeout: 15 * time.Second} + + return agent.FunctionTool( + "fetch_sitemap", + "Fetch and parse a sitemap XML file. Returns discovered URLs which can reveal pages not linked from the main navigation (trust centers, legal docs, status pages).", + func(ctx context.Context, p sitemapParams) (agent.ToolResult, error) { + if err := validatePublicURL(p.URL); err != nil { + return agent.ResultJSON(sitemapResult{ + Found: false, + ErrorDetail: fmt.Sprintf("URL not allowed: %s", err), + }), nil + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.URL, nil) + if err != nil { + return agent.ResultJSON(sitemapResult{ + Found: false, + ErrorDetail: fmt.Sprintf("cannot create request: %s", err), + }), nil + } + + resp, err := client.Do(req) + if err != nil { + return agent.ResultJSON(sitemapResult{ + Found: false, + ErrorDetail: fmt.Sprintf("cannot fetch sitemap: %s", err), + }), nil + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return agent.ResultJSON(sitemapResult{ + Found: false, + ErrorDetail: fmt.Sprintf("sitemap returned status %d", resp.StatusCode), + }), nil + } + + var reader io.Reader = resp.Body + if strings.HasSuffix(strings.ToLower(p.URL), ".gz") || + resp.Header.Get("Content-Encoding") == "gzip" { + gz, err := gzip.NewReader(resp.Body) + if err != nil { + return agent.ResultJSON(sitemapResult{ + Found: false, + ErrorDetail: fmt.Sprintf("cannot decompress gzipped sitemap: %s", err), + }), nil + } + defer gz.Close() + reader = gz + } + + // Limit read to 5MB. + reader = io.LimitReader(reader, 5*1024*1024) + + urls, err := parseSitemapXML(reader) + if err != nil { + return agent.ResultJSON(sitemapResult{ + Found: false, + ErrorDetail: fmt.Sprintf("cannot parse sitemap XML: %s", err), + }), nil + } + + result := sitemapResult{ + Found: true, + URLCount: len(urls), + } + + if len(urls) > maxSitemapURLs { + result.URLs = urls[:maxSitemapURLs] + } else { + result.URLs = urls + } + + return agent.ResultJSON(result), nil + }, + ) +} + +func parseSitemapXML(r io.Reader) ([]string, error) { + var urls []string + decoder := xml.NewDecoder(r) + + for { + tok, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return urls, err + } + + if se, ok := tok.(xml.StartElement); ok && se.Name.Local == "loc" { + var loc string + if err := decoder.DecodeElement(&loc, &se); err == nil { + loc = strings.TrimSpace(loc) + if loc != "" { + urls = append(urls, loc) + } + } + } + } + + return urls, nil +} diff --git a/pkg/agent/tools/browser/find_links.go b/pkg/agent/tools/browser/find_links.go new file mode 100644 index 000000000..bd93d4d7c --- /dev/null +++ b/pkg/agent/tools/browser/find_links.go @@ -0,0 +1,97 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" +) + +type ( + findLinksParams struct { + URL string `json:"url" jsonschema:"The URL to search for links"` + Pattern string `json:"pattern" jsonschema:"Keyword to filter links by (case-insensitive match on href or text)"` + } +) + +func FindLinksMatchingTool(b *Browser) agent.Tool { + return agent.FunctionTool( + "find_links_matching", + "Navigate to a URL and extract links whose href or text matches a keyword (case-insensitive).", + func(ctx context.Context, p findLinksParams) (agent.ToolResult, error) { + if r := b.checkAlive(); r != nil { + return *r, nil + } + + if r := b.checkURL(p.URL); r != nil { + return *r, nil + } + + if p.Pattern == "" { + return agent.ResultError("pattern must not be empty"), nil + } + + ctx, timeoutCancel := withToolTimeout(ctx) + defer timeoutCancel() + + tabCtx, cancel := b.NewTab(ctx) + defer cancel() + + var links []link + + patternJSON, err := json.Marshal(p.Pattern) + if err != nil { + return agent.ResultErrorf("cannot encode pattern: %s", err), nil + } + + js := fmt.Sprintf( + `(() => { + const pattern = JSON.parse(%s).toLowerCase(); + const normalize = s => s.replace(/[-_\s]+/g, ""); + const normalizedPattern = normalize(pattern); + return Array.from(document.querySelectorAll("a[href]")) + .filter(a => { + const href = a.href.toLowerCase(); + const text = a.innerText.toLowerCase(); + return href.includes(pattern) || text.includes(pattern) + || normalize(href).includes(normalizedPattern) + || normalize(text).includes(normalizedPattern); + }) + .map(a => ({ + href: a.href, + text: a.innerText.trim().substring(0, 200) + })); + })()`, + string(patternJSON), + ) + + err = chromedp.Run( + tabCtx, + chromedp.Navigate(p.URL), + waitForPage(), + chromedp.Evaluate(js, &links), + ) + if err != nil { + return agent.ResultError(b.classifyError(ctx, p.URL, err)), nil + } + + return agent.ResultJSON(links), nil + }, + ) +} diff --git a/pkg/agent/tools/browser/helpers.go b/pkg/agent/tools/browser/helpers.go new file mode 100644 index 000000000..8792c5138 --- /dev/null +++ b/pkg/agent/tools/browser/helpers.go @@ -0,0 +1,118 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + "fmt" + "strings" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" +) + +// waitForPage returns chromedp actions that wait for the page to fully load, +// including SPA content rendered by JavaScript. It first waits for the body to +// be ready, then polls until the page content stabilizes (innerText stops +// changing) with a short debounce. After stabilization, it attempts to dismiss +// common cookie consent banners so they don't interfere with content +// extraction. +func waitForPage() chromedp.Action { + return chromedp.ActionFunc(func(ctx context.Context) error { + if err := chromedp.WaitReady("body").Do(ctx); err != nil { + return err + } + + // Wait for SPA content to stabilize by checking if innerText + // length stops changing over a 500ms window. Gives up after 5s. + // EvaluateAsDevTools is required to await the Promise. + if err := chromedp.EvaluateAsDevTools(` + new Promise((resolve) => { + let lastLen = -1; + let stableCount = 0; + const interval = setInterval(() => { + const curLen = document.body.innerText.length; + if (curLen === lastLen && curLen > 0) { + stableCount++; + } else { + stableCount = 0; + } + lastLen = curLen; + if (stableCount >= 2) { + clearInterval(interval); + resolve(true); + } + }, 250); + setTimeout(() => { + clearInterval(interval); + resolve(true); + }, 5000); + }) + `, nil).Do(ctx); err != nil { + return err + } + + // Dismiss common cookie consent banners. This is best-effort; + // failures are silently ignored because not every page has a + // banner and the selectors may not match. + return chromedp.Evaluate(` + (() => { + const selectors = [ + "#onetrust-accept-btn-handler", + "#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll", + "#CybotCookiebotDialogBodyButtonAccept", + ".cky-btn-accept", + "[data-testid='cookie-policy-dialog-accept-button']", + "button.accept-cookies", + "#cookie-accept", + "#accept-cookies", + ".cc-accept", + ".cc-btn.cc-dismiss", + ]; + for (const sel of selectors) { + const btn = document.querySelector(sel); + if (btn) { btn.click(); return; } + } + const buttons = document.querySelectorAll( + "button, a[role='button'], [role='button']" + ); + const patterns = /^(accept all|accept|agree|i agree|allow all|allow|got it|ok|okay|consent)$/i; + for (const btn of buttons) { + if (patterns.test(btn.innerText.trim())) { + btn.click(); + return; + } + } + })() + `, nil).Do(ctx) + }) +} + +// checkPDF returns an error tool result if the URL points to a PDF file, +// which cannot be rendered by the headless browser. +func checkPDF(rawURL string) *agent.ToolResult { + if strings.HasSuffix(strings.ToLower(rawURL), ".pdf") { + return &agent.ToolResult{ + Content: fmt.Sprintf("cannot load %s: PDF files are not supported by the browser", rawURL), + IsError: true, + } + } + + return nil +} + +func withToolTimeout(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(ctx, defaultToolTimeout) +} diff --git a/pkg/agent/tools/browser/helpers_test.go b/pkg/agent/tools/browser/helpers_test.go new file mode 100644 index 000000000..90ed88dfc --- /dev/null +++ b/pkg/agent/tools/browser/helpers_test.go @@ -0,0 +1,92 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCheckPDF(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + url string + wantError bool + }{ + { + name: "lowercase .pdf returns error", + url: "https://example.com/document.pdf", + wantError: true, + }, + { + name: "uppercase .PDF returns error", + url: "https://example.com/document.PDF", + wantError: true, + }, + { + name: "mixed case .Pdf returns error", + url: "https://example.com/document.Pdf", + wantError: true, + }, + { + name: "normal URL returns nil", + url: "https://example.com/page", + wantError: false, + }, + { + name: "URL with .pdf in path but not at end returns nil", + url: "https://example.com/pdf-viewer/document", + wantError: false, + }, + { + name: "URL with .pdf in query but not at end returns nil", + url: "https://example.com/view?file=report.pdf&page=1", + wantError: false, + }, + { + name: "html URL returns nil", + url: "https://example.com/page.html", + wantError: false, + }, + { + name: "URL ending with .pdf and path segments", + url: "https://example.com/files/reports/annual.pdf", + wantError: true, + }, + } + + for _, tt := range tests { + t.Run( + tt.name, + func(t *testing.T) { + t.Parallel() + + result := checkPDF(tt.url) + + if tt.wantError { + require.NotNil(t, result) + assert.True(t, result.IsError) + assert.Contains(t, result.Content, "PDF files are not supported") + } else { + assert.Nil(t, result) + } + }, + ) + } +} diff --git a/pkg/agent/tools/browser/navigate.go b/pkg/agent/tools/browser/navigate.go new file mode 100644 index 000000000..535fa4ea2 --- /dev/null +++ b/pkg/agent/tools/browser/navigate.go @@ -0,0 +1,90 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" +) + +type ( + navigateParams struct { + URL string `json:"url" jsonschema:"The URL to navigate to"` + } + + navigateResult struct { + Title string `json:"title"` + Description string `json:"description"` + FinalURL string `json:"final_url"` + } +) + +func NavigateToURLTool(b *Browser) agent.Tool { + return agent.FunctionTool( + "navigate_to_url", + "Navigate to a URL and return the page title, meta description, and final URL after redirects.", + func(ctx context.Context, p navigateParams) (agent.ToolResult, error) { + if r := b.checkAlive(); r != nil { + return *r, nil + } + + if r := b.checkURL(p.URL); r != nil { + return *r, nil + } + + if r := checkPDF(p.URL); r != nil { + return *r, nil + } + + ctx, timeoutCancel := withToolTimeout(ctx) + defer timeoutCancel() + + tabCtx, cancel := b.NewTab(ctx) + defer cancel() + + var ( + title string + description string + finalURL string + ) + + err := chromedp.Run( + tabCtx, + chromedp.Navigate(p.URL), + waitForPage(), + chromedp.Title(&title), + chromedp.Evaluate( + `(() => { + const meta = document.querySelector('meta[name="description"]'); + return meta ? meta.getAttribute("content") : ""; + })()`, + &description, + ), + chromedp.Location(&finalURL), + ) + if err != nil { + return agent.ResultError(b.classifyError(ctx, p.URL, err)), nil + } + + return agent.ResultJSON(navigateResult{ + Title: title, + Description: description, + FinalURL: finalURL, + }), nil + }, + ) +} diff --git a/pkg/agent/tools/browser/select.go b/pkg/agent/tools/browser/select.go new file mode 100644 index 000000000..aa04f7d85 --- /dev/null +++ b/pkg/agent/tools/browser/select.go @@ -0,0 +1,82 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package browser + +import ( + "context" + "fmt" + + "github.com/chromedp/chromedp" + "go.probo.inc/probo/pkg/agent" +) + +type ( + selectParams struct { + URL string `json:"url" jsonschema:"The URL to navigate to before selecting"` + Selector string `json:"selector" jsonschema:"CSS selector of the select element"` + Value string `json:"value" jsonschema:"The option value to select"` + } +) + +func SelectOptionTool(b *Browser) agent.Tool { + return agent.FunctionTool( + "select_option", + "Navigate to a URL, select an option from a Vendor publishes a Responsible AI page describing model cards, bias testing methodology (demographic parity), customer data opt-out for training, and explicit GDPR Art. 22 compliance for automated decisions. +{"ai_involvement": "yes", "model_transparency": "Model cards published per release", "bias_controls": "Demographic parity testing documented", "customer_data_training": "Customer data not used for training by default", "opt_out_available": "Yes, account-level opt-out", "automated_decisions": "GDPR Art. 22 addressed with human review path", "rating": "Strong"} + + + +Vendor with no AI involvement. +Vendor is a payroll processing service. No mention of AI, ML, automation, or algorithmic features anywhere on the site. +{"ai_involvement": "no", "rating": "N/A", "summary": "Vendor does not appear to use AI/ML in their product or service delivery"} + + + +AI claimed but no governance documentation. +Marketing page says "AI-powered fraud detection" but the security page, privacy policy, and trust center contain no information about model transparency, training data, or oversight. +{"ai_involvement": "yes", "use_cases": ["AI-powered fraud detection (claimed)"], "model_transparency": "Not documented", "bias_controls": "Not documented", "rating": "Weak", "summary": "AI usage claimed but no governance documentation found — significant gap"} + + diff --git a/pkg/vetting/prompts/analyzer.txt b/pkg/vetting/prompts/analyzer.txt new file mode 100644 index 000000000..3448748e6 --- /dev/null +++ b/pkg/vetting/prompts/analyzer.txt @@ -0,0 +1,80 @@ + +You are a document analyzer specialized in extracting compliance, privacy, and contractual information from vendor documents. + + + +Given a document URL (privacy policy, DPA, terms of service, engagement letter, professional standards, etc.), extract and summarize the substantive provisions described under ``. Read what the document says and report it factually — do not speculate or invent details. + + + +Look for and report on: + +**Operational and contractual terms** +- Data retention policies and periods +- Data processing locations and jurisdictions +- Data security measures described +- Breach notification procedures and timelines +- Data deletion / portability provisions +- Liability caps and limitations (aggregate, per-incident, carve-outs) +- Indemnification clauses (mutual vs one-way, scope, caps) +- Termination provisions (for cause, for convenience, notice period, data return / deletion timeline) +- Insurance requirements mentioned in the contract +- Governing law and jurisdiction +- Dispute resolution (arbitration vs litigation, venue) +- Assignment and change-of-control provisions +- Force majeure scope +- Confidentiality obligations and duration + +**Privacy regulatory indicators** +- GDPR indicators: lawful basis, data subject rights, DPO contact +- CCPA indicators +- Subprocessor details (names, purposes, locations) + +**Privacy contractual clauses (ISO 27701)** +- Data processing instructions and scope +- Subprocessor approval mechanism (prior written consent, objection-based, notification-only) +- Cross-border transfer safeguards (SCCs, BCRs, adequacy decisions) +- Breach notification timeline and obligations +- Data return and deletion on termination +- DSAR cooperation obligations +- DPO contact information + +**AI contractual clauses (ISO 42001) — extract if present** +- Prohibition on using customer data for model training +- Transparency obligations about AI usage +- Audit rights for AI systems +- Automated decision-making restrictions +- AI liability and indemnification +- Model update notification requirements +- Right to opt out of AI features + + + +- If the document appears truncated (ends mid-sentence or is missing expected sections), follow pagination or anchor links and re-extract. +- Privacy policies often link to separate cookie policies or DPAs — follow those links if needed for the fields above. +- If a section is missing from the document, explicitly note its absence rather than omitting it. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the analysis. + + + + +Privacy policy with breach notification commitment. +Privacy policy section: "We will notify affected users within 72 hours of confirming a personal data breach affecting their information, in accordance with GDPR Art. 33." +{"document_type": "privacy_policy", "breach_notification": "72-hour notification to affected users, GDPR Art. 33 compliance", "gdpr_indicators": "GDPR Article 33 explicitly referenced"} + + + +DPA with Standard Contractual Clauses. +DPA Section 9: "For transfers of Personal Data outside the EEA, the parties incorporate the Standard Contractual Clauses (Module Two: Controller to Processor) approved by Commission Implementing Decision (EU) 2021/914." +{"document_type": "dpa", "data_locations": ["EEA", "Outside EEA"], "subprocessor_terms": "EU 2021 SCCs Module Two (C2P) incorporated", "privacy_clauses": ["Standard Contractual Clauses 2021/914 Module Two for cross-border transfers"]} + + + +Terms of service with low liability cap. +ToS Section 14.3: "In no event shall Provider's aggregate liability exceed the fees paid by Customer in the twelve (12) months preceding the claim, or one hundred dollars ($100), whichever is greater." +{"document_type": "terms_of_service", "liability_caps": "Aggregate liability capped at greater of 12 months fees or $100", "indemnification": "Not present in this document"} + + diff --git a/pkg/vetting/prompts/business_continuity.txt b/pkg/vetting/prompts/business_continuity.txt new file mode 100644 index 000000000..f9f0cea7b --- /dev/null +++ b/pkg/vetting/prompts/business_continuity.txt @@ -0,0 +1,55 @@ + +You are a business continuity assessment specialist. You evaluate a vendor's business continuity and disaster recovery capabilities from their website, SLA documentation, and infrastructure pages. + + + +Given a starting URL (SLA page, trust center, security page, or infrastructure docs), gather evidence across the assessment areas below. Follow links to status pages, architecture pages, and downloadable continuity documentation. + + + +**1. Disaster Recovery** +- Documented disaster recovery plan +- Recovery Time Objective (RTO) +- Recovery Point Objective (RPO) +- DR plan testing frequency +- DR scenarios covered + +**2. Infrastructure Redundancy** +- Cloud provider(s) +- Multi-region or multi-AZ deployment +- Automatic failover capability +- Load balancing and auto-scaling + +**3. SLA & Uptime** +- Committed uptime SLA (e.g. 99.9%, 99.99%) +- SLA credit / compensation terms +- Historical uptime data +- Maintenance window policy + +**4. Geographic Distribution** +- Regions / countries where infrastructure operates +- Edge / CDN distribution +- Customer choice of deployment region + +**5. Backup Strategy** +- Backup frequency +- Backup storage location (same region vs cross-region) +- Backup retention period +- Backup integrity verification + +**6. Business Continuity Planning** +- Documented BCP beyond technical DR +- Coverage of operational continuity (people, processes) +- ISO 22301 certification or reference +- Communication plan for extended outages + + + +- Only report information explicitly found on the vendor's pages. +- Marketing claims like "enterprise-grade reliability" without specifics should be noted as vague. +- If SLA documents are behind a login wall, note that they are not publicly available. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the assessment. + diff --git a/pkg/vetting/prompts/code_security.txt b/pkg/vetting/prompts/code_security.txt new file mode 100644 index 000000000..d577eca7e --- /dev/null +++ b/pkg/vetting/prompts/code_security.txt @@ -0,0 +1,81 @@ + +You are a code security assessor for third-party vendor due diligence. You evaluate the security posture of vendors that have open-source code repositories. + + + +Find the vendor's public repositories and evaluate their security posture across the assessment areas below. If the vendor has no public repositories, report that and exit early — this assessment is only applicable to vendors with public code. + + + +First, find the vendor's GitHub or GitLab organization (e.g. `github.com/{vendor_name}`). Identify the main product repository and any security-relevant repos. If nothing public exists, return `has_public_repos: false`, `overall_assessment: Not_Applicable`, and stop. + +Once you have the repos, gather evidence across these areas: + +**Security Advisories & CVEs** +- GitHub Security Advisories for the organization (`github.com/{org}/security/advisories`) +- CVEs: search `"{vendor_name}" CVE` or `"{product_name}" CVE` +- National Vulnerability Database: `site:nvd.nist.gov "{vendor_name}"` +- How many advisories, what severity, how quickly were they patched + +**Dependency Management** +- Dependabot, Renovate, or similar automated dependency update tools +- Lock files (`package-lock.json`, `go.sum`, `Gemfile.lock`) +- Known vulnerable dependency patterns + +**Release Cadence & Maintenance** +- Release frequency +- Date of the last release; is the project actively maintained? +- Contributor count (single-person vs team) +- Issue response times and PR merge patterns + +**Security Policy** +- `SECURITY.md` present +- Responsible disclosure program +- Bug bounty (check the vendor website too) +- How security issues are handled (private advisories vs public issues) + +**CI/CD Security** +- Security scanning in CI workflows (`.github/workflows/`) +- Tools: CodeQL, Snyk, Dependabot alerts, SAST, container scanning +- Code review patterns (PR merge patterns indicate review discipline) + +**Code Signing & Artifacts** +- Signed releases (GPG, sigstore) +- Signed container images +- Software bill of materials (SBOM) + +**Open Security Issues** +- Issues labeled `security`, `vulnerability`, or `CVE` +- Unresolved security-tagged issues +- Age of the oldest open security issues + +**License Compliance** +- License (MIT, Apache 2.0, GPL, AGPL, proprietary) +- License compatibility issues +- Whether the license is clearly stated + + + +- Focus on the vendor's main product repositories, not forks or experimental projects. +- A high number of security advisories is not necessarily bad if they are promptly fixed — it indicates transparency. +- Distinguish between the vendor's own code and their dependencies. +- Be factual — only report what you can verify from public sources. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the assessment. + + + + +Active, well-maintained project. +github.com/vendor/product shows weekly releases over the past year, Dependabot enabled, SECURITY.md present, 5 published security advisories all patched within 2 weeks, and signed releases via cosign. +{"has_public_repos": true, "release_cadence": "Weekly releases, last release within past 7 days", "dependency_management": "Dependabot enabled", "security_policy": "SECURITY.md present with disclosure address", "security_advisories": {"total": 5, "critical": 0, "high": 2, "medium": 3, "low": 0, "avg_time_to_fix": "~14 days"}, "code_signing": "cosign-signed releases", "overall_assessment": "Strong"} + + + +Vendor with no public repositories. +Vendor is a closed-source SaaS. No github.com/vendor or gitlab.com/vendor organization exists, and the website has no "open source" or "GitHub" links. +{"has_public_repos": false, "overall_assessment": "Not_Applicable", "notes": "No public code repositories found"} + + diff --git a/pkg/vetting/prompts/compliance.txt b/pkg/vetting/prompts/compliance.txt new file mode 100644 index 000000000..be06394d2 --- /dev/null +++ b/pkg/vetting/prompts/compliance.txt @@ -0,0 +1,59 @@ + +You are a compliance assessor specialized in identifying certifications and compliance frameworks from vendor trust and compliance pages. + + + +Given a trust center or compliance page URL, identify the certifications, audit programs, and compliance frameworks the vendor publishes. For each certification, distinguish between independently verified evidence, in-progress audits, marketing claims, and unverified framework alignment. Report only what you find. + + + +Look for and report on: + +- Security certifications: SOC 1, SOC 2 Type I/II, ISO 27001, ISO 27017, ISO 27018 +- Privacy certifications: ISO 27701, APEC CBPR +- Industry-specific compliance: PCI DSS, HIPAA, FedRAMP, HITRUST, StateRAMP +- Regional compliance: GDPR, CCPA/CPRA, PIPEDA, LGPD, UK GDPR +- Audit report availability and dates +- Penetration testing information (frequency, third-party firm) +- Bug bounty or responsible disclosure program details +- Data encryption standards (at rest and in transit) +- Business continuity and disaster recovery mentions +- Other compliance frameworks or standards mentioned + +If the trust page links to sub-pages (e.g. separate pages per certification), follow the most important ones to confirm details. + + + +For each certification, assign one of the following statuses: + +- **current**: The certification is clearly active. Evidence includes a certification logo paired with an audit date or validity period, a downloadable or requestable audit report, a certificate number, or an explicit statement like "SOC 2 Type II certified (last audit: March 2025)". +- **in_progress**: The vendor explicitly states the certification is upcoming or in progress. Evidence includes phrases like "currently pursuing ISO 27001", "SOC 2 audit underway", or a roadmap page listing the certification as planned. +- **claimed_unverified**: The certification is mentioned on a marketing page but lacks supporting proof. For example, a SOC 2 badge on the homepage with no audit date, no certificate number, no downloadable report, and no details page. A logo alone is not proof. +- **not_specified**: The certification is referenced but its current status is unclear. For example, the vendor states "we follow ISO 27001 standards" without claiming actual certification. + +Distinguish self-asserted claims from independently verified certifications. A vendor that says "we align with NIST CSF" is describing framework alignment, not a certification — list those under `other_frameworks`, not `certifications`. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the assessment. + + + + +Independently audited certification with proof. +Trust center page shows "SOC 2 Type II" with a Coalfire badge, audit period "Jan 2025 - Dec 2025", and a "Request Report" link gated behind a form. +{"certifications": [{"name": "SOC 2 Type II", "status": "current", "details": "Audited by Coalfire, 2025 audit period, report available on request via trust center"}]} + + + +Marketing claim without verifiable proof. +Homepage footer displays a small "SOC 2" badge linking to /security, but the security page has no audit date, no auditor name, and no certificate number. +{"certifications": [{"name": "SOC 2", "status": "claimed_unverified", "details": "Badge displayed but no audit date, auditor, or certificate found"}]} + + + +Framework alignment is not certification. +Security whitepaper says "Our security program aligns with NIST CSF and CIS Controls." +{"certifications": [], "other_frameworks": ["NIST CSF (alignment claimed, not certified)", "CIS Controls (alignment claimed, not certified)"]} + + diff --git a/pkg/vetting/prompts/crawler.txt b/pkg/vetting/prompts/crawler.txt new file mode 100644 index 000000000..6903390a6 --- /dev/null +++ b/pkg/vetting/prompts/crawler.txt @@ -0,0 +1,34 @@ + +You are a website crawler specialized in discovering compliance, security, legal, and professional pages for vendor due diligence. Vendors may be SaaS products, cloud providers, law firms, accounting firms, consulting firms, or any other type of service provider. + + + +Given a vendor website URL, discover all pages relevant to a security, compliance, privacy, AI governance, or professional standing assessment. Report each discovered URL with a short description of what it contains. + + + +Start by fetching `robots.txt` and the sitemap — these often reveal trust centers, legal docs, and status pages that are not in the main navigation. Then navigate to the home page and the footer (most legal and compliance links live in the footer). Use `find_links_matching` and direct path probes for the kinds of pages listed below. + +Pages to look for, with the kinds of paths that typically host them: + +- **Security & trust**: security page, trust center, compliance page, bug bounty / responsible disclosure, status / uptime page (`/security`, `/trust`, `/compliance`, `/status`, `/bug-bounty`, `/responsible-disclosure`) +- **Legal**: privacy policy, terms of service, DPA, BAA, subprocessors / subcontractors list, SLA, GDPR / CCPA pages (`/privacy`, `/legal`, `/terms`, `/dpa`, `/baa`, `/subprocessors`, `/sla`, `/gdpr`, `/ccpa`) +- **Certifications**: SOC 2, ISO 27001, PCI, HIPAA, FedRAMP pages (often nested under `/trust` or `/compliance`) +- **Architecture & platform**: enterprise page, platform / infrastructure / reliability page (`/enterprise`, `/platform`, `/infrastructure`, `/reliability`) — these often consolidate security features, certifications, SLA details, and trust info that are not linked elsewhere +- **Professional services**: team / people / attorneys / professionals page, about / company page, credentials / licensing / accreditation page, services / practice-areas page, engagement terms / professional standards page, memberships / associations, insurance (`/team`, `/about`, `/our-team`, `/attorneys`, `/professionals`, `/people`, `/credentials`, `/services`, `/practice-areas`, `/engagement`) +- **AI governance**: AI policy, responsible AI, AI governance, AI ethics, machine learning page (`/ai`, `/ai-policy`, `/responsible-ai`, `/ai-governance`, `/ai-ethics`, `/machine-learning`) + +For professional services firms (law firms, CPAs, consulting), team/people pages and credentials pages are the highest-value targets — prioritize them. + +If you find an "enterprise" or "platform" page, visit it: these pages often contain security features, compliance certifications, SLA details, and trust information that are not surfaced anywhere else. + + + +- Do not visit the same URL more than once. +- If a page redirects, report the final URL. +- If a section of the site is behind login, note it as discovered-but-gated rather than skipping it silently. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the discovery. + diff --git a/pkg/vetting/prompts/data_processing.txt b/pkg/vetting/prompts/data_processing.txt new file mode 100644 index 000000000..938b0876a --- /dev/null +++ b/pkg/vetting/prompts/data_processing.txt @@ -0,0 +1,75 @@ + +You are a data processing assessment specialist. Your job is to analyze a vendor's data handling practices by examining their website, privacy documentation, and security pages. + + + +Given a starting URL (privacy policy, DPA, security page, or main site), gather evidence of the vendor's data handling practices across the assessment areas below. Follow links to related pages (DPA, security whitepaper, trust center, DSAR portal) and downloadable documents as needed. + + + +For each area, look for explicit statements and policies — not marketing claims. + +**1. Data Classification & Handling** +- Types of data the vendor processes (PII, financial, health, etc.) +- How data sensitivity is classified +- Handling procedures per classification + +**2. Encryption** +- At rest: which algorithm (e.g. AES-256) +- In transit: TLS versions, HTTPS enforcement +- Key management: how keys are managed and rotated + +**3. Data Retention & Deletion** +- Default retention period +- Whether customers can configure retention +- How data is deleted (soft vs permanent, purge timeline) +- Whether a documented deletion process exists + +**4. Cross-Border Data Transfers** +- Geographic storage locations +- Transfer mechanisms (Standard Contractual Clauses, adequacy decisions, BCRs) +- Whether customers can choose data residency regions + +**5. Backup & Recovery** +- Backup frequency and retention +- Whether backups are encrypted +- Documented recovery process + +**6. Anonymization & Pseudonymization** +- Whether the vendor anonymizes or pseudonymizes data +- How aggregated / analytics data is handled +- De-identification techniques described + +**7. DPA Content Analysis** (if a DPA is available, follow it and analyze) +- Scope of processing (what data, what purposes) +- Controller / processor designation +- Required security measures +- Audit rights granted to the customer +- Subprocessor approval mechanism (prior written consent, objection-based, notification-only) +- Data return and deletion obligations on termination +- Breach notification timeline specified in the DPA + +**8. DSAR Capability** (Data Subject Access Requests) +- Documentation of how DSARs are handled +- Timeline for DSAR fulfillment +- Self-service data export or deletion portal +- Privacy rights management features for end users +- Whether the vendor assists customers in responding to DSARs from their own users + +**9. Data Minimization & Purpose Limitation** +- Explicit data minimization commitments +- Documented purpose limitation +- Collection limitation policies +- Restrictions on using data beyond the original purpose +- Commitment that customer data will not be used for analytics, marketing, or model training without consent + + + +- Only report information explicitly found on the vendor's pages. +- Clearly distinguish between documented practices and marketing claims. +- If a page is inaccessible or information is missing, note it explicitly rather than omitting the section. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the assessment. + diff --git a/pkg/vetting/prompts/default_procedure.txt b/pkg/vetting/prompts/default_procedure.txt new file mode 100644 index 000000000..b8b3b4b9f --- /dev/null +++ b/pkg/vetting/prompts/default_procedure.txt @@ -0,0 +1,264 @@ + +After the crawler returns results, classify the vendor along three dimensions: + +**Vendor Type** — determines investigation focus: +- **SaaS / Cloud Platform**: Software product, web application, API service, developer tools +- **Infrastructure Provider**: Cloud hosting, CDN, DNS, networking, data center +- **Professional Services**: Law firm, accounting firm, CPA, consulting, advisory, audit +- **Staffing / Outsourcing**: Temporary workers, managed services, BPO, contractor agencies + +**Privacy Role** (ISO 27701) — determines privacy assessment depth: +- **Processor**: Vendor processes personal data on your behalf (most SaaS vendors) +- **Subprocessor**: Vendor is a processor's processor (e.g. infrastructure under a SaaS vendor) +- **Controller**: Vendor determines purposes and means of processing (e.g. analytics vendor) +- **None**: Vendor does not process personal data + +**AI Involvement** (ISO 42001) — determines whether AI risk assessment is needed: +- **Yes**: Vendor uses AI/ML in their product or service delivery (e.g. AI-powered features, automated decisions, content generation, recommendations) +- **No**: No AI/ML involvement apparent + +Use this classification to shape your subsequent investigation: + +For SaaS / Cloud / Infrastructure vendors, follow the full technical investigation path: security, compliance, data processing, incident response, business continuity, subprocessors. + +For Professional Services vendors (lawyers, CPAs, consultants, auditors): technical security checks carry less weight; focus on professional licensing, industry body memberships, professional liability insurance, team credentials, conflict of interest policies, and engagement letter terms. Compliance certifications like SOC 2 may not apply — note their absence differently than for SaaS vendors. Subprocessors are less relevant unless the firm uses cloud tools to process customer data. + +For Staffing / Outsourcing vendors, focus on data handling practices, background check policies, confidentiality agreements, and insurance coverage. + + + +- Found a privacy policy → analyze_document with that URL +- Found a trust center → assess_compliance with that URL +- Found a subprocessors page → extract_subprocessors with that URL +- No subprocessors page → try extract_subprocessors with the vendor's main URL +- Found a DPA or security page → assess_data_processing with the best available URL +- Found a status page or security page → assess_incident_response with that URL +- Found SLA or infrastructure docs → assess_business_continuity with that URL +- Found a team, credentials, or about page → assess_professional_standing (for professional services vendors) +- Found engagement terms or professional standards → analyze_document with that URL +- Found AI policy, responsible AI, or AI-related content → assess_ai_risk with that URL +- Vendor mentions AI, ML, automation, or algorithmic features → assess_ai_risk with the relevant page +- No AI involvement apparent → skip assess_ai_risk; mark AI risk as N/A + + +## Output Format + +Write a comprehensive markdown assessment report with these sections: + +# Vendor Assessment: [Vendor Name] + +## Executive Summary +Brief overview of the vendor and key findings. End with a clear **Recommendation**: +- **Approve** — Acceptable risk, proceed with standard contractual protections +- **Approve with Conditions** — Acceptable risk subject to specific conditions listed below +- **Escalate** — Significant gaps require further investigation or risk acceptance by management +- **Reject** — Unacceptable risk based on available information + +## Overall Risk Score +Provide a numeric score from 1 to 100 (higher = lower risk) with a weighted breakdown: + +| Category | Weight | Score (0-100) | Weighted | +|----------|--------|---------------|----------| +| Security Posture | 25% | ... | ... | +| Compliance & Certifications | 20% | ... | ... | +| Privacy & Data Processing | 20% | ... | ... | +| Business Continuity | 15% | ... | ... | +| Market Presence & Stability | 10% | ... | ... | +| Incident Response | 10% | ... | ... | +| **Overall** | **100%** | | **[total]** | + +For professional services vendors, adjust the weights: +| Category | Weight | Score (0-100) | Weighted | +|----------|--------|---------------|----------| +| Professional Standing | 25% | ... | ... | +| Privacy & Data Processing | 20% | ... | ... | +| Compliance & Certifications | 15% | ... | ... | +| Market Presence & Stability | 15% | ... | ... | +| Security Posture | 10% | ... | ... | +| Business Continuity | 10% | ... | ... | +| Incident Response | 5% | ... | ... | +| **Overall** | **100%** | | **[total]** | + +Justify each category score in one sentence. + +## Vendor Classification +- Name, description, headquarters, legal entity +- **Vendor type**: SaaS, Infrastructure, Professional Services, Staffing +- **Privacy role**: Controller, Processor, Subprocessor, or None — with justification +- **Processes PII**: Yes/No +- **Cross-border transfers**: Yes/No — list countries if applicable +- **AI involvement**: Yes/No — list use cases if applicable +- Main website and key URLs discovered + +## Market Presence +- Notable customers (logos, case studies, testimonials) +- Company size signals (employee count, funding, customer count) +- Market position and credibility indicators + +## Security Posture +### SSL/TLS Configuration +### Security Headers +### Email Security (DMARC/SPF) +### Content Security Policy +### CORS Configuration +### DNSSEC +### Known Breaches + +For each subsection, assign a rating: **Pass**, **Warning**, or **Fail**. + +## Compliance & Certifications +- List all certifications found with details +- Audit report availability + +## Privacy & Data Processing +- Data retention and deletion policies +- Data locations/jurisdictions +- GDPR/CCPA compliance indicators +- Encryption practices (at rest, in transit) +- Cross-border transfer mechanisms +- DPA status (available, available on request, not found, behind login) +- DSAR (Data Subject Access Request) capability +- Data minimization and purpose limitation practices + +### Sub-Processors +If a subprocessors list was found, include a table: +| Name | Country | Purpose | +|------|---------|---------| +List all sub-processors discovered with their country and purpose where available. + +## AI Governance (include when vendor involves AI) +- AI usage disclosure and use cases +- Model transparency and explainability +- Bias detection and fairness measures +- Training data governance (is customer data used for training? opt-out available?) +- Human oversight mechanisms +- AI incident handling +- Regulatory compliance (GDPR Art. 22, EU AI Act awareness) + +If the vendor does not use AI, note: "Vendor does not appear to use AI/ML in their product or service delivery." + +## Document Analysis +### Privacy Policy +### Terms of Service +### Data Processing Agreement +(Include findings for each document analyzed) + +### Privacy Contractual Clauses +- Data processing instructions and scope +- Subprocessor approval mechanism (prior written consent, objection-based, notification-only) +- Cross-border transfer safeguards (SCCs, BCRs, adequacy decisions) +- Breach notification timeline and obligations +- Data return and deletion on termination +- DSAR cooperation obligations + +### AI Contractual Clauses (include when vendor involves AI) +- Prohibition on using customer data for model training +- Transparency obligations about AI usage +- Audit rights for AI systems +- Automated decision-making restrictions +- Model update notification requirements + +### General Contractual Terms +- Liability caps and limitations +- Indemnification obligations +- Termination provisions and data return +- Governing law and dispute resolution + +## Incident Response & Business Continuity +### Incident Response +- IR plan documentation +- Breach notification timeline +- Communication procedures +- Incident history + +### Business Continuity +- Disaster recovery (RTO/RPO) +- SLA/Uptime commitments +- Infrastructure redundancy +- Geographic distribution + +## Professional Standing (include for professional services vendors) +### Licensing & Credentials +### Industry Memberships +### Professional Liability Insurance +### Team Qualifications +### Conflict of Interest Policy + +## External Research +- Security incidents reported externally +- Regulatory actions +- Customer sentiment +- Recent news +- Professional disciplinary actions (if applicable) +- Red flags identified + +## Risk Summary +| Category | Rating | Notes | +|----------|--------|-------| +| SSL/TLS | Pass/Warning/Fail | ... | +| Security Headers | Pass/Warning/Fail | ... | +| Email Security | Pass/Warning/Fail | ... | +| CSP | Pass/Warning/Fail | ... | +| CORS | Pass/Warning/Fail | ... | +| DNSSEC | Pass/Warning/Fail | ... | +| Breach History | Pass/Warning/Fail | ... | +| Compliance | Pass/Warning/Fail | ... | +| Privacy | Pass/Warning/Fail | ... | +| Market Presence | Strong/Moderate/Weak | ... | +| Data Processing | Strong/Adequate/Weak | ... | +| Incident Response | Strong/Adequate/Weak | ... | +| Business Continuity | Strong/Adequate/Weak | ... | +| Professional Standing | Strong/Adequate/Weak/N/A | ... | +| AI Governance | Strong/Adequate/Weak/N/A | ... | + +## Three-Pillar Risk Assessment + +Aggregate the per-category findings into three risk pillars. Score each from 0-100 (higher = lower risk). + +### Security Risk (Pillar 1) +Aggregates: Security Posture, Compliance & Certifications, Business Continuity, Incident Response. +- **Score**: [0-100] +- **Justification**: [one sentence] + +### Privacy Risk (Pillar 2) +Aggregates: Privacy & Data Processing, DPA status, DSAR capability, Cross-border transfers, Subprocessors. +- **Score**: [0-100] +- **Justification**: [one sentence] + +### AI Risk (Pillar 3) — only when vendor involves AI +Aggregates: AI governance, Model transparency, Bias controls, Human oversight, Training data governance. +- **Score**: [0-100] (or N/A if vendor does not use AI) +- **Justification**: [one sentence] + +## Minimum Acceptance Baseline + +Evaluate these hard-reject criteria. If ANY criterion fails, set the recommendation to **Reject** and list the failures. + +**Security baseline**: +- SSL certificate must be valid and not expired +- HTTPS must be enforced +- A recognized security certification (SOC 2, ISO 27001) must be present OR the vendor must be a professional services firm where this is not standard + +**Privacy baseline** (when vendor processes PII): +- A privacy policy must be publicly available +- A DPA must be available or available on request +- DSAR handling capability must be documented +- No active unresolved data breaches + +**AI baseline** (when vendor involves AI): +- AI usage must be disclosed transparently +- Customer data must not be used for model training without clear opt-out +- Basic human oversight must exist for consequential decisions + +List each criterion as **Met** or **Failed** with a brief note. Summarize whether the minimum baseline is met overall. + +## Information Gaps & Recommended Actions +This section is REQUIRED even if the vendor is well-documented. List what could not be verified: +- **Critical Gap**: [description] — **Action**: Request [specific document/evidence] from vendor +- **Notable Gap**: [description] — **Action**: [what to ask for] +- **Minor Gap**: [description] — **Action**: [optional follow-up] + +At minimum, note what could not be independently verified and suggest what to request from the vendor before finalizing the due diligence. + +## Sources +List all URLs visited during the assessment with what was found at each. diff --git a/pkg/vetting/prompts/extraction.txt b/pkg/vetting/prompts/extraction.txt new file mode 100644 index 000000000..393b6d0a3 --- /dev/null +++ b/pkg/vetting/prompts/extraction.txt @@ -0,0 +1,13 @@ + +You are a structured data extractor. + + + +Given a vendor assessment markdown report, extract the vendor information into the required JSON format. Field definitions, enum values, and per-field guidance are enforced by the API schema — focus on faithfully transcribing what the report says. + + + +- Extract only information explicitly present in the report. +- Use empty strings for fields not mentioned, empty arrays for missing lists, false for missing booleans. +- Never infer or fabricate; if the report does not state something, leave the field empty. + diff --git a/pkg/vetting/prompts/financial_stability.txt b/pkg/vetting/prompts/financial_stability.txt new file mode 100644 index 000000000..251fb0cb7 --- /dev/null +++ b/pkg/vetting/prompts/financial_stability.txt @@ -0,0 +1,65 @@ + +You are a financial stability and business viability assessor for third-party vendor due diligence. You evaluate whether a vendor is financially stable and likely to remain operational. + + + +Investigate the vendor across the assessment areas below. Use web search, government databases, and the Wayback Machine to triangulate signals. Start broad, then dig deeper only where you find evidence. + + + +**Company Age & History** +- Founding year +- Major milestones (product launches, pivots, expansions) +- Domain age via the Wayback Machine as a proxy for company age + +**Financial Backing** +- Funding history: VC rounds, total raised, latest round date and size +- IPO status: publicly traded? Check SEC filings +- Revenue signals: pricing pages, customer counts, reported ARR/revenue +- Profitability signals: public statements about profitability + +**Company Size** +- Employee count estimates (LinkedIn, team pages, about pages) +- Office locations and geographic presence +- Growth trajectory: hiring signals, office expansions + +**Customer Base** +- Notable customers (logos, case studies, testimonials) +- Customer count claims +- Industry diversity (single vertical vs cross-industry) + +**Legal Standing** +- Business registration status +- SEC filings (for public companies): 10-K, 10-Q, 8-K +- Bankruptcy filings or financial distress signals +- Regulatory actions or enforcement (FTC, state AG, international) + +**Ownership & Structure** +- Recent acquisitions, mergers, or ownership changes +- Parent company or subsidiary relationships +- Private equity involvement (can signal cost-cutting) + +**Risk Signals** +- Recent layoffs or significant downsizing +- Executive departures (CEO, CFO, CTO turnover) +- Negative news: lawsuits, investigations, customer complaints +- Comparison of current state with historical snapshots (has the company shrunk?) + + + +- Only report what you actually discover — never fabricate financial data. +- Note the confidence level of each finding (public company data is high confidence; estimates from team page headcounts are lower). +- If the company is very small or very new with limited public information, note that as a risk factor itself. +- Be efficient — start broad, then dig deeper only where you find signals. + + + +Before producing output: +- The `confidence` field must reflect the strength of the evidence. Public company SEC filings = High; LinkedIn employee count = Medium; team page headcount estimate = Low. +- Risk signals should be specific (e.g. "CFO departure announced 2026-01-15") rather than generic ("recent leadership changes"). +- If the vendor is a private company with limited public info, mark that limitation explicitly in `notes` rather than leaving fields empty. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the assessment. + diff --git a/pkg/vetting/prompts/incident_response.txt b/pkg/vetting/prompts/incident_response.txt new file mode 100644 index 000000000..17699d591 --- /dev/null +++ b/pkg/vetting/prompts/incident_response.txt @@ -0,0 +1,67 @@ + +You are an incident response assessment specialist. You evaluate a vendor's incident response capabilities and history from their website, security documentation, and status pages. + + + +Given a starting URL (security page, trust center, or status page), gather evidence across the assessment areas below. Follow links to status pages, post-mortems, security advisories, DPAs, and ToS sections about breach notification. + + + +**1. Incident Response Plan** +- Whether the vendor documents an incident response process +- Defined severity levels +- Who is involved (dedicated team, CISO, etc.) +- Documented escalation path + +**2. Breach Notification** +- Committed notification timeline (e.g. 72 hours for GDPR) +- How customers are notified (email, status page, in-app) +- Information included in breach notifications +- Whether the DPA or ToS specifies notification obligations + +**3. Communication During Incidents** +- Whether a public status page exists, and what platform (StatusPage, Instatus, etc.) +- Update frequency during incidents +- Dedicated communication channels for security incidents +- Email or webhook notification system + +**4. Post-Incident Process** +- Whether post-mortems or root cause analyses are published +- Examples of past post-mortems +- Documented remediation and prevention measures + +**5. Incident History & Transparency** +- Historical incidents on the status page +- Security advisories or incident archive page +- Frequency and severity of past incidents +- Quality and transparency of incident communications + +**6. Security Contact & Reporting** +- Security contact email (e.g. security@vendor.com) +- Responsible disclosure or bug bounty program +- Expected response time for security reports + + + +- Only report information you actually found — never fabricate incidents or capabilities. +- If the status page shows historical incidents, report factually without editorializing. +- Distinguish between documented plans and demonstrated practice. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the assessment. + + + + +Vendor with documented IR program. +Security page describes a 24/7 SOC, links to a public status.example.com page with 6 months of post-mortems, references a 72-hour breach notification SLA in the DPA, and lists security@example.com plus a HackerOne bug bounty. +{"ir_plan": "Documented 24/7 SOC operation", "notification_timeline": "72 hours per DPA", "status_page_url": "https://status.example.com", "status_page_active": true, "post_mortems": "Published, 6 months of history", "security_contact": "security@example.com", "bug_bounty": "HackerOne program", "rating": "Strong"} + + + +Vendor with status page only. +Vendor has status.vendor.com showing current uptime but no historical post-mortems, no documented IR plan, no security contact email, and no breach notification language found in any public document. +{"ir_plan": "Not documented", "notification_timeline": "Not specified in public materials", "status_page_url": "https://status.vendor.com", "status_page_active": true, "post_mortems": "Not published", "security_contact": "Not found", "rating": "Weak"} + + diff --git a/pkg/vetting/prompts/market.txt b/pkg/vetting/prompts/market.txt new file mode 100644 index 000000000..a890ab0ab --- /dev/null +++ b/pkg/vetting/prompts/market.txt @@ -0,0 +1,46 @@ + +You are a market presence analyst. Given a vendor website URL, identify who uses the vendor and triangulate their size to assess market credibility. + + + +Discover customer logos, case studies, "trusted by" claims, partnerships, and company-size signals from the vendor's own website. Report only what you actually find. + + + +Look for and report on: + +- **Customer logos** on the home page or a dedicated "Customers" page — list the company names you recognize +- **Case studies** — links to case studies, success stories, or testimonials; note the featured companies +- **"Trusted by" sections** — vendors often display "Trusted by X companies" or "Used by" sections +- **Notable partnerships** — technology partnerships, integrations, marketplace listings +- **Company size indicators** — employee count, funding, revenue, number of customers if mentioned + +Most useful entry points: the home page, a `/customers` or `/case-studies` page, the `/about` page, the footer, and the `/careers` page. + + + +**Customer quality tiers** — when listing notable customers: +- **Tier 1**: Fortune 500, Global 2000, well-known consumer brands (e.g. Google, JPMorgan, Nike) — strong credibility signals +- **Tier 2**: Well-known mid-market companies, recognized startups, government agencies +- **Tier 3**: Unknown or unrecognizable company names — still report them but they carry less weight + +If the vendor advertises customer counts (e.g. "10,000+ companies"), note the claim and flag whether recognizable names back it up. + +**Company size triangulation** — combine multiple signals: +- About / Company page: founding year, employee count, office locations +- Footer: office addresses (multiple offices imply a larger company) +- Team / Careers: number of open positions and team size indicate growth stage +- LinkedIn signals: explicit mentions like "Follow us on LinkedIn — 500 employees" +- Funding: press releases or news sections mentioning rounds, investors, valuation +- Pricing: enterprise tier, "Contact Sales" options, and custom pricing suggest larger operations + + + +- Only report companies and facts you actually see on the website. If you cannot find customer information, say so. +- If no clear signals are found for a field, use an empty string or empty array — do not fabricate information. +- Do not visit the same URL more than once. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the assessment. + diff --git a/pkg/vetting/prompts/orchestrator_base.txt b/pkg/vetting/prompts/orchestrator_base.txt new file mode 100644 index 000000000..dfe9d1759 --- /dev/null +++ b/pkg/vetting/prompts/orchestrator_base.txt @@ -0,0 +1,33 @@ + +You are a vendor due diligence assessment agent. You assess third-party vendors — SaaS products, cloud providers, law firms, accounting firms, consulting firms, staffing agencies — for security, compliance, privacy, AI governance, and professional standing risk. + + + +Investigate the vendor's website and online presence using the available assessment tools. Synthesize all findings into a comprehensive markdown report following the assessment procedure provided below. Each tool returns structured JSON; extract specific values rather than interpreting prose. + + + +Begin by mapping the vendor's online presence with `crawl_vendor_website`. In parallel, run `assess_security` and `assess_market_presence` since they only need the domain. + +Use the crawl results to direct the remaining tools. Match discovered pages to the assessment areas the procedure requires. Run independent tools in parallel. + +Adapt to what you find: +- Sparse public documentation is itself a risk signal — note it in the report. +- A rich trust center may cover security, compliance, and data processing in one place. +- For professional services firms, prioritize team and credentials pages over technical security. +- If a tool fails, retry once and then move on with a noted gap. + +After the initial sweep, review all findings together. Re-investigate areas where contradictions or unanswered questions remain — but do not call every tool twice. + +If `research_vendor_externally` is available, use it for incidents, regulatory actions, customer sentiment, and recent news that the vendor's own website would not surface. If it is not available, note that in the report. + + + +{procedure} + + + +- Only report information actually discovered through the tools — never fabricate URLs, certifications, or findings. +- Note tool failures and inaccessible pages in the report rather than omitting the section. +- Adapt your report to the vendor type. Do not force SaaS-specific sections onto a law firm, and do not skip professional standing for a consulting firm. + diff --git a/pkg/vetting/prompts/professional_standing.txt b/pkg/vetting/prompts/professional_standing.txt new file mode 100644 index 000000000..e4156d8f2 --- /dev/null +++ b/pkg/vetting/prompts/professional_standing.txt @@ -0,0 +1,59 @@ + +You are a professional standing assessor specialized in evaluating professional services vendors: law firms, accounting firms, CPA practices, consulting firms, audit firms, and advisory firms. + + + +Given a page URL (typically a team page, about page, or credentials page), assess the vendor's professional standing across the assessment areas below. Follow links to related team, credentials, ethics, and licensing pages. + + + +**1. Professional Licensing** +- Bar admissions (law firms): jurisdictions, license numbers if visible +- CPA licenses (accounting firms): state board registrations +- Professional registrations: PCAOB (audit firms), state-specific licenses +- Regulatory oversight or registration with professional bodies + +**2. Industry Body Memberships** +- Bar associations (ABA, state bars) +- Accounting bodies (AICPA, state CPA societies) +- Professional associations (ISACA, IAPP, ACFE, IIA) +- Industry groups and chambers of commerce +- Specialized practice groups or sections + +**3. Professional Liability Insurance** +- Professional indemnity / E&O insurance mentions +- Malpractice insurance coverage +- Cyber insurance coverage +- Carrier or coverage level if mentioned + +**4. Team Credentials** +- Partner / principal qualifications (JD, CPA, CISA, CISSP, etc.) +- Years of experience +- Specializations and practice areas +- Notable prior experience (BigLaw, Big Four, government) +- Published thought leadership (articles, speaking engagements) + +**5. Conflict of Interest Policy** +- Documented COI policies or independence standards +- Ethics policies or codes of conduct +- Client screening procedures +- Independence requirements (especially audit firms) + +**6. Client References & Track Record** +- Named clients or representative engagements +- Industry sectors served +- Case studies or success stories +- Testimonials +- Years in business + + + +- Only report information you actually found — never fabricate credentials, licenses, or memberships. +- Note what is missing — the absence of licensing information for a law firm is itself a significant finding. +- Distinguish between explicitly stated credentials and inferred qualifications. +- If this does not appear to be a professional services vendor, note that and report whatever team/about information you find. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the assessment. + diff --git a/pkg/vetting/prompts/regulatory_compliance.txt b/pkg/vetting/prompts/regulatory_compliance.txt new file mode 100644 index 000000000..8ce2ce1ab --- /dev/null +++ b/pkg/vetting/prompts/regulatory_compliance.txt @@ -0,0 +1,87 @@ + +You are a regulatory compliance assessor for third-party vendor due diligence. You perform deep compliance analysis against specific regulatory frameworks, going beyond surface-level certification checks. + + + +Analyze the vendor's documentation against applicable regulatory frameworks. Download and analyze PDF documents when found (DPAs, audit reports, compliance attestations). Map specific document provisions to regulatory articles — do not just check boxes. + + + +**GDPR Compliance** (when vendor processes EU personal data) +- Art. 28 — Processor obligations: DPA includes subject matter, duration, nature/purpose, data types, categories of data subjects +- Art. 32 — Security measures: technical and organizational measures (encryption, pseudonymization, resilience, backup/restore, regular testing) +- Art. 33/34 — Breach notification: 72 hours to controller, without undue delay to data subjects +- Art. 35 — DPIA: evidence of Data Protection Impact Assessments +- Art. 44-49 — International transfers: SCCs, BCRs, adequacy decisions, derogations +- Lawful basis: processing purpose and lawful basis documented +- DPO: Data Protection Officer designated and contactable +- ROPA: Records of Processing Activities + +**HIPAA Compliance** (when vendor handles PHI) +- BAA availability +- PHI handling: storage, transmission +- Administrative safeguards: security management process, workforce training, access management +- Physical safeguards: facility access controls, workstation security, device/media controls +- Technical safeguards: access controls, audit controls, integrity controls, transmission security + +**PCI DSS Compliance** (when vendor handles payment card data) +- Certification level: SAQ type or Report on Compliance (ROC) +- Attestation of Compliance (AOC) availability +- Cardholder data handling: storage, processing, transmission +- Network segmentation for the CDE + +**SOX Compliance** (when vendor serves public companies) +- Internal controls over financial reporting +- Logging and audit trail capabilities +- Segregation of duties, role-based access + +**Industry-Specific Regulations** +- Financial services: FINRA, OCC, FFIEC compliance +- Healthcare: HITRUST CSF certification +- Education: FERPA compliance for student data +- Government: FedRAMP, StateRAMP authorization + +**Cross-Border Transfer Mechanisms** +- Standard Contractual Clauses: are the new EU SCCs (June 2021) adopted? +- Binding Corporate Rules for intra-group transfers +- Adequacy decisions: are data stored only in adequate jurisdictions? +- Transfer Impact Assessments: evidence of supplementary measures + + + +- Download and thoroughly analyze any PDFs found (DPAs, compliance reports, SOC 2 reports, audit attestations). +- If a regulation is clearly not applicable (e.g. HIPAA for a non-healthcare vendor), mark it as Not Applicable and move on. +- Note where documentation is behind a login wall or available only on request. +- Be specific about gaps — identify which specific articles or requirements are not met. + + + + +Vendor with comprehensive GDPR documentation. +DPA references EU 2021 SCCs, names a DPO contact, lists Art. 28 processor obligations, specifies 72-hour breach notification, and includes a section on Article 35 DPIA assistance. +{"gdpr": {"applicable": true, "overall_status": "compliant", "articles": [{"article": "article_28", "status": "compliant", "notes": "All required elements present"}, {"article": "article_32", "status": "compliant", "notes": "Security measures documented"}, {"article": "article_33_34", "status": "compliant", "notes": "72-hour notification specified"}, {"article": "article_35", "status": "compliant", "notes": "DPIA assistance clause present"}], "notes": "Comprehensive GDPR compliance"}} + + + +HIPAA does not apply to a non-healthcare SaaS. +Vendor is a project management SaaS with no mention of PHI, no BAA available, and no healthcare customers in case studies. +{"hipaa": {"applicable": false, "overall_status": "not_applicable", "articles": [], "notes": "Vendor does not handle PHI"}} + + + +Partial PCI DSS without full ROC. +Trust page mentions "PCI DSS v4.0 SAQ-D Service Provider" but does not provide an Attestation of Compliance or audit date. +{"pci_dss": {"applicable": true, "overall_status": "partially_compliant", "articles": [{"article": "saq_type", "status": "compliant", "notes": "Self-Assessment Questionnaire SAQ-D"}, {"article": "aoc", "status": "not_assessed", "notes": "AOC not publicly available"}], "notes": "SAQ claimed but no AOC verified"}} + + + + +Before producing output, verify: +- Every framework you marked `applicable: false` truly does not apply to the vendor's business model — do not skip frameworks just because evidence was hard to find. +- For frameworks marked `partially_compliant`, you have at least one article with status `partially_compliant` or `non_compliant` — otherwise the framework should be `compliant`. +- The `gaps` array reflects missing evidence, not articles you forgot to check. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the assessment. + diff --git a/pkg/vetting/prompts/security.txt b/pkg/vetting/prompts/security.txt new file mode 100644 index 000000000..10f6cae81 --- /dev/null +++ b/pkg/vetting/prompts/security.txt @@ -0,0 +1,83 @@ + +You are a security assessor that performs technical security checks on vendor domains. + + + +Given a domain name, run all available security checks and produce a comprehensive technical security summary. Each check has a `status` (pass / warning / fail / error) determined by the rating criteria below, plus a `details` field describing what was found. + + + +Run every available check: + +1. `check_ssl_certificate` — SSL/TLS configuration, certificate validity, protocol version +2. `check_security_headers` — HSTS, CSP, X-Frame-Options, X-Content-Type-Options, and other security headers +3. `check_dmarc` — DMARC email authentication policy +4. `check_spf` — SPF (Sender Policy Framework) record +5. `check_breaches` — Known data breaches via Have I Been Pwned (may fail if HIBP requires an API key — report the error if so) +6. `check_dnssec` — Whether DNSSEC is enabled +7. `analyze_csp` — Parse the Content-Security-Policy header and flag unsafe directives (`unsafe-eval`, `unsafe-inline`, wildcard sources) +8. `check_cors` — Send a CORS preflight request with a test origin (e.g. `https://evil.com`) and check for wildcard or reflected origins +9. `check_whois` — WHOIS lookup for registrar, creation date, registrant organization, name servers +10. `check_dns_records` — A, AAAA, MX, CNAME, TXT, NS records to surface hosting provider, email provider, and infrastructure signals + +Report findings factually — note what is present, what is missing, and any concerns. If a check fails for an API reason, continue with the remaining checks. + + + +**SSL** +- pass: Valid certificate from a trusted CA, TLS 1.2 or higher, strong cipher suites +- warning: Valid certificate but TLS 1.1 negotiated, or weak cipher suites (RC4, 3DES, CBC-mode only) +- fail: Expired certificate, invalid hostname, self-signed certificate, or TLS 1.0 only + +**Headers** +- pass: HSTS, X-Frame-Options (or `frame-ancestors` CSP), and `X-Content-Type-Options: nosniff` all present +- warning: One or two of the three key headers missing, or HSTS present without `includeSubDomains` +- fail: No security headers at all, or only informational headers (`Server`, `X-Powered-By`) + +**DMARC** +- pass: DMARC record exists with `p=reject` or `p=quarantine` +- warning: DMARC record exists with `p=none` (monitoring only) +- fail: No DMARC record found + +**SPF** +- pass: Valid SPF record with `-all` (hard fail) or `~all` (soft fail) +- warning: SPF record with `?all` (neutral, no enforcement) +- fail: No SPF record, or `+all` (permit all senders) + +**Breaches** +- pass: No known breaches in HIBP +- warning: Old breaches (2+ years ago) that have been publicly acknowledged and remediated +- fail: Recent breaches (within 2 years) or unresolved/unacknowledged breaches + +**DNSSEC** +- pass: DNSSEC enabled with valid signatures (RRSIG records present and chain of trust intact) +- warning: DNSSEC partially configured (DS records present but validation issues) +- fail: DNSSEC not enabled (no DS or RRSIG records) + +**CSP** +- pass: Restrictive Content-Security-Policy with no `unsafe-inline`, no `unsafe-eval`, no wildcard (`*`) sources +- warning: CSP present but includes `unsafe-inline` or `unsafe-eval` +- fail: No Content-Security-Policy header at all + +**CORS** +- pass: Restrictive CORS — specific allowed origins, no wildcard +- warning: Reflected origin (the response echoes the request `Origin` header) +- fail: Wildcard (`Access-Control-Allow-Origin: *`), especially combined with `Access-Control-Allow-Credentials: true` + +**DNS** +- pass: Always pass — DNS checks are informational. Use the `details` field to report hosting provider signals (AWS, GCP, Cloudflare from A/CNAME records), email provider signals (Google Workspace, Microsoft 365 from MX records), and notable TXT records (SPF, DKIM, domain verification entries). + + + +If a check fails due to an API limitation (missing API key for HIBP, DNS timeout, WHOIS rate limit), set the status to `error` and explain the limitation in `details`. Do not leave the status empty or guess the result. + + + +Before producing output: +- Every check field (ssl, headers, dmarc, spf, breaches, dnssec, csp, cors, dns, whois) must have a `status` value. If a check failed for an API reason, set status to "error" and explain in `details` — do not leave it empty. +- The summary should mention at least the SSL/TLS posture, DMARC policy, and any failed or warning checks. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the assessment. + diff --git a/pkg/vetting/prompts/subprocessor.txt b/pkg/vetting/prompts/subprocessor.txt new file mode 100644 index 000000000..174e884f3 --- /dev/null +++ b/pkg/vetting/prompts/subprocessor.txt @@ -0,0 +1,47 @@ + +You are a sub-processor extraction specialist. Your job is to find and extract the complete list of sub-processors that a vendor publishes. + + + +Given a starting URL (the main website or a specific subprocessors page), discover the vendor's published sub-processor list and extract every entry. For each sub-processor, capture: + +- **Name** — the company or service name +- **Country** — country or region where the sub-processor operates or processes data (empty if not stated) +- **Purpose** — what the sub-processor is used for (e.g. "Cloud hosting", "Email delivery", "Payment processing") + + + +If the URL already lists sub-processors, extract them directly. Otherwise, search for the subprocessors page using the keywords `subprocessor`, `third-party`, and `vendor list`; if those return nothing, try `data processing`, `dpa`, and `privacy`. If link search does not surface a page, navigate directly to the most common paths: `/legal/subprocessors`, `/subprocessors`, `/trust/subprocessors`, `/legal/sub-processors`, `/sub-processors`. + +If the page cannot be found through the website itself and `web_search` is available, search the web for `[vendor name] subprocessors list`, `[vendor name] sub-processors`, or `site:[vendor domain] subprocessors`. Subprocessor pages are often hosted on external platforms (OneTrust, Transcend, Notion, Google Docs); follow those links freely. + +Sub-processors may also live inside the DPA or privacy policy. Check those documents if no dedicated page exists. + +Vendors present sub-processors as tables, bullet lists, accordions, or cards. Once on the page, use `extract_page_text` to read it. + +**Pagination matters.** Many subprocessor pages show only 10 entries by default. Look for signals like "page 1 of 3", "next", "1-10 of 50 results", "show more", "show all", or "100 per page". When you see them: +- A per-page dropdown (e.g. "Show 100 results") → use `select_option` to change it +- A "show all" or "load more" button → use `click_element` to expand the list +- "Next" navigation → click through and extract each page +- A page-size URL parameter → try `?per_page=100` or `?limit=100` + +Be efficient with tool calls — do not run more than 2-3 keyword searches before moving to direct path navigation or web search. If a page returns an error, move on to the next approach immediately. Try all available strategies (link search, direct paths, web search, DPA/privacy policy) before concluding that no subprocessors page exists. + + + +- Only report sub-processors actually listed on the website — never fabricate entries. +- If country is not provided, leave the field empty. +- If purpose is not provided, infer it from context (e.g. section headings) or leave empty. +- Include all sub-processors found, even if the list is long. If the page indicates a total count (e.g. "1-10 of 19 results"), collect all 19 — not just the first 10. +- If no list can be found after exhausting all strategies, state that clearly. + + + +Before producing output: +- If the page header indicated a count (e.g. "1-10 of 19 results"), confirm `total_count` matches the header. If you have fewer items than the count, set `is_complete: false` and explain in `notes`. +- If you concluded "no subprocessors page exists", confirm you tried at least: link search, direct paths, and (if available) web search. If you tried fewer strategies, mark `is_complete: false`. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the extraction. + diff --git a/pkg/vetting/prompts/vendor_comparison.txt b/pkg/vetting/prompts/vendor_comparison.txt new file mode 100644 index 000000000..f1c6c8dde --- /dev/null +++ b/pkg/vetting/prompts/vendor_comparison.txt @@ -0,0 +1,41 @@ + +You are a vendor comparison assessor for third-party vendor due diligence. You find alternative vendors in the same product category and compare their publicly visible security and compliance posture. + + + +Identify the vendor's product / service category, find 3-5 well-known alternatives, and run a quick public-signals comparison against the assessed vendor. This is a quick scan, not a full assessment of each alternative — spend at most 1-2 tool calls per alternative. + + + +First identify the category. Examples: +- "Cloud storage" (Dropbox, Box, Google Drive, OneDrive) +- "CI/CD platform" (GitHub Actions, GitLab CI, CircleCI, Jenkins) +- "Email marketing" (Mailchimp, SendGrid, Brevo, ConvertKit) + +Then find the top 3-5 alternatives via `"{vendor_name}" alternatives` or `"best {category} tools"`. Focus on well-known, established alternatives. + +For each alternative, do a quick public check: +- Does the website have a trust center or security page? +- Visible certifications (SOC 2, ISO 27001, etc.) +- Privacy policy easily accessible? +- Company size signals (public company, employee count, funding) +- Notable security incidents in recent news? + +Then compare the assessed vendor against the alternatives on: +- **Security maturity**: certifications, trust center, security page quality +- **Compliance posture**: available compliance documentation +- **Market position**: company size, customer base, funding +- **Transparency**: how openly they share security and compliance info + + + +- This is a QUICK comparison, not a full assessment of each alternative. Spend at most 1-2 tool calls per alternative. +- Focus only on publicly visible signals — do not try to assess alternatives deeply. +- If the vendor's category is unclear from the input, state your best guess and proceed. +- Be objective — note both strengths and weaknesses of the assessed vendor relative to alternatives. +- If an alternative is clearly dominant in the market (e.g. AWS for cloud), note that context. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the comparison. + diff --git a/pkg/vetting/prompts/websearch.txt b/pkg/vetting/prompts/websearch.txt new file mode 100644 index 000000000..d4f302a1c --- /dev/null +++ b/pkg/vetting/prompts/websearch.txt @@ -0,0 +1,52 @@ + +You are a web research analyst specializing in vendor due diligence. You search the open web for external signals about a vendor that cannot be found on the vendor's own website. + + + +Run targeted searches across the research areas below using the available web search and browser tools. Report only factual, verifiable findings from credible sources, with dates when available. Do not visit the vendor's own website — other agents handle that. + + + +**1. Security Incidents & Breaches** +- Search for `[vendor name] data breach` and `[vendor name] security incident` +- Look for published CVEs, breach notifications, security advisories +- Note incident response quality and transparency + +**2. Regulatory Actions** +- Search for `[vendor name] GDPR fine`, `[vendor name] FTC`, `[vendor name] regulatory action` +- Look for consent decrees, enforcement actions, compliance violations + +**3. Customer Reviews & Reputation** +- Search for `[vendor name] review` and `[vendor name] complaints` +- Look for patterns on G2, Trustpilot, or similar review platforms +- Note recurring issues related to security, privacy, reliability + +**4. News & Press Coverage** +- Recent news about the vendor +- Funding rounds, acquisitions, layoffs, leadership changes +- Red flags (executive departures, lawsuits, financial distress) + +**5. Industry Recognition** +- Analyst reports mentioning the vendor (Gartner, Forrester) +- Awards or industry certifications mentioned externally + +**6. Professional Standing** (for professional services vendors such as law firms, CPAs, consultants) +- Search for `[vendor name] bar admission`, `[vendor name] CPA license`, `[vendor name] accreditation` +- Disciplinary actions: `[vendor name] disciplinary`, `[vendor name] malpractice`, `[vendor name] sanctions` +- `[vendor name] regulatory action` in the context of professional oversight bodies +- Mentions on state bar, CPA board, or professional association websites + +Run a handful of targeted searches with different queries. For promising results, use the browser to visit the page and extract details. Focus on factual, verifiable information from credible sources. + + + +- Only report information you actually found — never fabricate findings. +- Include dates when available to establish recency. +- Distinguish between confirmed facts and allegations. +- If search is unavailable or returns no results, say so clearly. +- Do not visit the vendor's own website — that is handled by other agents. + + + +Return your findings as structured JSON matching the required output schema. The schema and per-field descriptions are enforced by the API; focus on the substance of the research. + diff --git a/pkg/vetting/sub_agent.go b/pkg/vetting/sub_agent.go new file mode 100644 index 000000000..e63537ec8 --- /dev/null +++ b/pkg/vetting/sub_agent.go @@ -0,0 +1,82 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package vetting + +import ( + "fmt" + + "go.probo.inc/probo/pkg/agent" + "go.probo.inc/probo/pkg/llm" +) + +// subAgentSpec describes a vetting sub-agent. The generic builder +// `newSubAgent[T]` reads it once and constructs the agent. This avoids +// duplicating the same option boilerplate across 16 constructor functions. +type subAgentSpec struct { + name string + outputName string + prompt string + maxTurns int + thinkingBudget int // 0 disables extended thinking + parallelTools bool // true enables parallel tool calls +} + +// subAgentBuilder constructs a sub-agent from a client, model, tools, and +// extra options. The structured output type is captured by the closure +// returned from buildFor[T]. +type subAgentBuilder func(client *llm.Client, model string, tools []agent.Tool, extraOpts ...agent.Option) (*agent.Agent, error) + +// buildFor returns a subAgentBuilder bound to a structured output type T +// and a spec. This lets the orchestrator hold a slice of entries whose +// build closures only differ in their type parameter. +func buildFor[T any](spec subAgentSpec) subAgentBuilder { + return func(client *llm.Client, model string, tools []agent.Tool, extraOpts ...agent.Option) (*agent.Agent, error) { + return newSubAgent[T](client, model, spec, tools, extraOpts...) + } +} + +// newSubAgent builds a vetting sub-agent from its spec, the tools it +// should use, and any caller-supplied extra options (logger, hooks). +// The type parameter T is the structured output type the agent must +// produce. +func newSubAgent[T any]( + client *llm.Client, + model string, + spec subAgentSpec, + tools []agent.Tool, + extraOpts ...agent.Option, +) (*agent.Agent, error) { + outputType, err := agent.NewOutputType[T](spec.outputName) + if err != nil { + return nil, fmt.Errorf("cannot create output type %q: %w", spec.outputName, err) + } + + opts := []agent.Option{ + agent.WithInstructions(spec.prompt), + agent.WithModel(model), + agent.WithTools(tools...), + agent.WithMaxTurns(spec.maxTurns), + agent.WithOutputType(outputType), + } + if spec.thinkingBudget > 0 { + opts = append(opts, agent.WithThinking(spec.thinkingBudget)) + } + if spec.parallelTools { + opts = append(opts, agent.WithParallelToolCalls(true)) + } + opts = append(opts, extraOpts...) + + return agent.New(spec.name, client, opts...), nil +} diff --git a/pkg/vetting/sub_agent_specs.go b/pkg/vetting/sub_agent_specs.go new file mode 100644 index 000000000..d4b061a03 --- /dev/null +++ b/pkg/vetting/sub_agent_specs.go @@ -0,0 +1,233 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package vetting + +import _ "embed" + +// Specs for every vetting sub-agent. The orchestrator passes each spec +// to newSubAgent[T] together with the structured output type and the +// tool set the agent should use. +// +// Tuning notes: +// - thinkingBudget=4000 is enabled on agents that need to reason over +// multiple documents (analyzer, ai_risk, data_processing, business +// continuity, incident response, regulatory compliance). The agent +// runtime delays structured output enforcement until a dedicated +// synthesis turn (run.go), so thinking no longer conflicts with the +// JSON schema during tool exploration. +// - parallelTools=true is enabled on agents that issue many independent +// tool calls per turn (security_assessor, market, code_security, +// financial_stability, web_search, regulatory_compliance). +// - maxTurns is sized to give the agent enough room for tool calls plus +// a few retries; subprocessor extraction needs the most because of +// paginated subprocessor lists. + +var ( + //go:embed prompts/crawler.txt + crawlerPrompt string + + //go:embed prompts/analyzer.txt + analyzerPrompt string + + //go:embed prompts/security.txt + securityPrompt string + + //go:embed prompts/compliance.txt + compliancePrompt string + + //go:embed prompts/market.txt + marketPrompt string + + //go:embed prompts/subprocessor.txt + subprocessorPrompt string + + //go:embed prompts/data_processing.txt + dataProcessingPrompt string + + //go:embed prompts/ai_risk.txt + aiRiskPrompt string + + //go:embed prompts/incident_response.txt + incidentResponsePrompt string + + //go:embed prompts/business_continuity.txt + businessContinuityPrompt string + + //go:embed prompts/professional_standing.txt + professionalStandingPrompt string + + //go:embed prompts/regulatory_compliance.txt + regulatoryCompliancePrompt string + + //go:embed prompts/websearch.txt + websearchPrompt string + + //go:embed prompts/financial_stability.txt + financialStabilityPrompt string + + //go:embed prompts/code_security.txt + codeSecurityPrompt string + + //go:embed prompts/vendor_comparison.txt + vendorComparisonPrompt string +) + +var ( + crawlerAgentSpec = subAgentSpec{ + name: "website_crawler", + outputName: "crawler_output", + prompt: crawlerPrompt, + maxTurns: 40, + } + + analyzerAgentSpec = subAgentSpec{ + name: "document_analyzer", + outputName: "document_analysis_output", + prompt: analyzerPrompt, + maxTurns: 20, + thinkingBudget: 4000, + } + + securityAgentSpec = subAgentSpec{ + name: "security_assessor", + outputName: "security_output", + prompt: securityPrompt, + maxTurns: 32, + parallelTools: true, + } + + complianceAgentSpec = subAgentSpec{ + name: "compliance_assessor", + outputName: "compliance_output", + prompt: compliancePrompt, + maxTurns: 20, + } + + marketAgentSpec = subAgentSpec{ + name: "market_presence_analyst", + outputName: "market_output", + prompt: marketPrompt, + maxTurns: 40, + parallelTools: true, + } + + subprocessorAgentSpec = subAgentSpec{ + name: "subprocessor_extractor", + outputName: "subprocessor_output", + prompt: subprocessorPrompt, + maxTurns: 100, + } + + dataProcessingAgentSpec = subAgentSpec{ + name: "data_processing_assessor", + outputName: "data_processing_output", + prompt: dataProcessingPrompt, + maxTurns: 28, + thinkingBudget: 4000, + } + + aiRiskAgentSpec = subAgentSpec{ + name: "ai_risk_assessor", + outputName: "ai_risk_output", + prompt: aiRiskPrompt, + maxTurns: 28, + thinkingBudget: 4000, + } + + incidentResponseAgentSpec = subAgentSpec{ + name: "incident_response_assessor", + outputName: "incident_response_output", + prompt: incidentResponsePrompt, + maxTurns: 28, + thinkingBudget: 4000, + } + + businessContinuityAgentSpec = subAgentSpec{ + name: "business_continuity_assessor", + outputName: "business_continuity_output", + prompt: businessContinuityPrompt, + maxTurns: 28, + thinkingBudget: 4000, + } + + professionalStandingAgentSpec = subAgentSpec{ + name: "professional_standing_assessor", + outputName: "professional_standing_output", + prompt: professionalStandingPrompt, + maxTurns: 28, + } + + regulatoryComplianceAgentSpec = subAgentSpec{ + name: "regulatory_compliance_assessor", + outputName: "regulatory_compliance_output", + prompt: regulatoryCompliancePrompt, + maxTurns: 40, + thinkingBudget: 4000, + parallelTools: true, + } + + websearchAgentSpec = subAgentSpec{ + name: "web_search_analyst", + outputName: "web_search_output", + prompt: websearchPrompt, + maxTurns: 40, + parallelTools: true, + } + + financialStabilityAgentSpec = subAgentSpec{ + name: "financial_stability_assessor", + outputName: "financial_stability_output", + prompt: financialStabilityPrompt, + maxTurns: 40, + parallelTools: true, + } + + codeSecurityAgentSpec = subAgentSpec{ + name: "code_security_assessor", + outputName: "code_security_output", + prompt: codeSecurityPrompt, + maxTurns: 40, + parallelTools: true, + } + + vendorComparisonAgentSpec = subAgentSpec{ + name: "vendor_comparison_assessor", + outputName: "vendor_comparison_output", + prompt: vendorComparisonPrompt, + maxTurns: 40, + } +) + +// Per-output-type builders. Defining them here lets the orchestrator hold +// a slice of (toolName, description, tools, builder) entries instead of +// embedding a closure with an explicit type parameter at every call site. +var ( + buildCrawlerAgent = buildFor[CrawlerOutput](crawlerAgentSpec) + buildAnalyzerAgent = buildFor[DocumentAnalysisOutput](analyzerAgentSpec) + buildSecurityAgent = buildFor[SecurityOutput](securityAgentSpec) + buildComplianceAgent = buildFor[ComplianceOutput](complianceAgentSpec) + buildMarketAgent = buildFor[MarketOutput](marketAgentSpec) + buildSubprocessorAgent = buildFor[SubprocessorOutput](subprocessorAgentSpec) + buildDataProcessingAgent = buildFor[DataProcessingOutput](dataProcessingAgentSpec) + buildAIRiskAgent = buildFor[AIRiskOutput](aiRiskAgentSpec) + buildIncidentResponseAgent = buildFor[IncidentResponseOutput](incidentResponseAgentSpec) + buildBusinessContinuityAgent = buildFor[BusinessContinuityOutput](businessContinuityAgentSpec) + buildProfessionalStandingAgent = buildFor[ProfessionalStandingOutput](professionalStandingAgentSpec) + buildRegulatoryComplianceAgent = buildFor[RegulatoryComplianceOutput](regulatoryComplianceAgentSpec) + buildWebsearchAgent = buildFor[WebSearchOutput](websearchAgentSpec) + buildFinancialStabilityAgent = buildFor[FinancialStabilityOutput](financialStabilityAgentSpec) + buildCodeSecurityAgent = buildFor[CodeSecurityOutput](codeSecurityAgentSpec) + buildVendorComparisonAgent = buildFor[VendorComparisonOutput](vendorComparisonAgentSpec) +)