From 2040931405cb7abe31c7b60e63584324fbbdbf1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89mile=20R=C3=A9?= Date: Tue, 19 May 2026 12:32:41 +0400 Subject: [PATCH] Move Firecrawl API key from top-level config into Agents.Tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Firecrawl is a tool used by agents (tracker mapping, third-party assessor), so its configuration belongs under AgentsConfig rather than as a standalone Config field. Adds AgentToolsConfig to hold agent tool credentials and updates all config propagation consumers. Signed-off-by: Émile Ré --- contrib/helm/charts/probo/templates/deployment.yaml | 4 ++-- contrib/helm/charts/probo/templates/secret.yaml | 6 +++--- contrib/helm/charts/probo/values-production.yaml.example | 6 +++--- contrib/helm/charts/probo/values.yaml | 7 ++++--- pkg/bootstrap/builder.go | 6 +++--- pkg/bootstrap/builder_test.go | 8 ++++---- pkg/probod/third_party_assessor.go | 2 +- pkg/probod/tracker_mapping.go | 2 +- pkg/probodconfig/config.go | 6 ------ pkg/probodconfig/llm_config.go | 7 +++++++ 10 files changed, 28 insertions(+), 26 deletions(-) diff --git a/contrib/helm/charts/probo/templates/deployment.yaml b/contrib/helm/charts/probo/templates/deployment.yaml index 1923fd479..bf60e54ab 100644 --- a/contrib/helm/charts/probo/templates/deployment.yaml +++ b/contrib/helm/charts/probo/templates/deployment.yaml @@ -232,8 +232,8 @@ spec: - name: OPENAI_MAX_TOKENS value: {{ .Values.probo.openai.maxTokens | quote }} {{- end }} - # Firecrawl Integration - {{- if .Values.probo.firecrawl.apiKey }} + # Agent Tools + {{- if .Values.probo.agentTools.firecrawlApiKey }} - name: FIRECRAWL_API_KEY valueFrom: secretKeyRef: diff --git a/contrib/helm/charts/probo/templates/secret.yaml b/contrib/helm/charts/probo/templates/secret.yaml index 4638af44e..75933ac77 100644 --- a/contrib/helm/charts/probo/templates/secret.yaml +++ b/contrib/helm/charts/probo/templates/secret.yaml @@ -35,9 +35,9 @@ stringData: openai-api-key: {{ .Values.probo.openai.apiKey | quote }} {{- end }} - # Optional Firecrawl API key - {{- if .Values.probo.firecrawl.apiKey }} - firecrawl-api-key: {{ .Values.probo.firecrawl.apiKey | quote }} + # Optional Firecrawl API key (agent tools) + {{- if .Values.probo.agentTools.firecrawlApiKey }} + firecrawl-api-key: {{ .Values.probo.agentTools.firecrawlApiKey | quote }} {{- end }} # Optional SAML secrets diff --git a/contrib/helm/charts/probo/values-production.yaml.example b/contrib/helm/charts/probo/values-production.yaml.example index 9ec501c00..0c595f7fb 100644 --- a/contrib/helm/charts/probo/values-production.yaml.example +++ b/contrib/helm/charts/probo/values-production.yaml.example @@ -159,9 +159,9 @@ probo: modelName: "gpt-4o" maxTokens: 4096 - # Firecrawl web search (optional, used by tracker mapping agent) - # firecrawl: - # apiKey: "CHANGE_ME_FIRECRAWL_API_KEY" + # Agent tools (optional, shared across agents) + # agentTools: + # firecrawlApiKey: "CHANGE_ME_FIRECRAWL_API_KEY" # Tracker mapping agent (optional, auto-links tracker patterns to vendors) # trackerMapping: diff --git a/contrib/helm/charts/probo/values.yaml b/contrib/helm/charts/probo/values.yaml index 60c37c568..eff51540f 100644 --- a/contrib/helm/charts/probo/values.yaml +++ b/contrib/helm/charts/probo/values.yaml @@ -257,9 +257,10 @@ probo: modelName: "gpt-4o" maxTokens: 4096 - # Firecrawl web search integration (optional, used by tracker mapping agent) - firecrawl: - apiKey: "" + # Agent tools (optional, shared across agents) + agentTools: + # Firecrawl web search API key (used by tracker mapping and third-party assessor agents) + firecrawlApiKey: "" # Tracker mapping agent (optional, requires openai.apiKey or anthropic key) trackerMapping: diff --git a/pkg/bootstrap/builder.go b/pkg/bootstrap/builder.go index 02aeaf53e..b4624c3f5 100644 --- a/pkg/bootstrap/builder.go +++ b/pkg/bootstrap/builder.go @@ -176,9 +176,6 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { CacheTTL: b.getEnvIntOrDefault("WEBHOOK_CACHE_TTL", 86400), }, }, - Firecrawl: probodconfig.FirecrawlConfig{ - APIKey: b.getEnv("FIRECRAWL_API_KEY"), - }, Agents: probodconfig.AgentsConfig{ Providers: map[string]probodconfig.LLMProviderConfig{ "openai": { @@ -214,6 +211,9 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { Temperature: b.getEnvFloatPtr("AGENT_TRACKER_MAPPING_TEMPERATURE"), MaxTokens: b.getEnvIntPtr("AGENT_TRACKER_MAPPING_MAX_TOKENS"), }, + Tools: probodconfig.AgentToolsConfig{ + FirecrawlAPIKey: b.getEnv("FIRECRAWL_API_KEY"), + }, }, CustomDomains: probodconfig.CustomDomainsConfig{ RenewalInterval: b.getEnvIntOrDefault("CUSTOM_DOMAINS_RENEWAL_INTERVAL", 3600), diff --git a/pkg/bootstrap/builder_test.go b/pkg/bootstrap/builder_test.go index 0dcadedf1..cfb62f615 100644 --- a/pkg/bootstrap/builder_test.go +++ b/pkg/bootstrap/builder_test.go @@ -197,8 +197,8 @@ func TestBuilder_Build_Defaults(t *testing.T) { assert.Equal(t, 5, cfg.Probod.Notifications.Webhook.SenderInterval) assert.Equal(t, 86400, cfg.Probod.Notifications.Webhook.CacheTTL) - // Firecrawl — empty by default - assert.Empty(t, cfg.Probod.Firecrawl.APIKey) + // Agents tools — Firecrawl empty by default + assert.Empty(t, cfg.Probod.Agents.Tools.FirecrawlAPIKey) // Agents config — default assert.Equal(t, "openai", cfg.Probod.Agents.Default.Provider) @@ -378,8 +378,8 @@ func TestBuilder_Build_CustomValues(t *testing.T) { assert.Equal(t, "slack-signing-secret", cfg.Probod.Notifications.Slack.SigningSecret) assert.Equal(t, 10, cfg.Probod.Notifications.Webhook.SenderInterval) assert.Equal(t, 3600, cfg.Probod.Notifications.Webhook.CacheTTL) - // Firecrawl - assert.Equal(t, "fc-test-key", cfg.Probod.Firecrawl.APIKey) + // Agents tools — Firecrawl + assert.Equal(t, "fc-test-key", cfg.Probod.Agents.Tools.FirecrawlAPIKey) // Agents — providers assert.Equal(t, "openai", cfg.Probod.Agents.Providers["openai"].Type) assert.Equal(t, "sk-test-key", cfg.Probod.Agents.Providers["openai"].APIKey) diff --git a/pkg/probod/third_party_assessor.go b/pkg/probod/third_party_assessor.go index 443f3758c..cc1628062 100644 --- a/pkg/probod/third_party_assessor.go +++ b/pkg/probod/third_party_assessor.go @@ -52,7 +52,7 @@ func (impl *Implm) buildThirdPartyAssessor( Model: agentCfg.ModelName, MaxTokens: maxTokens, ChromeAddr: impl.cfg.ChromeDPAddr, - FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey, + FirecrawlAPIKey: impl.cfg.Agents.Tools.FirecrawlAPIKey, Logger: l.Named("third-party-assessor"), }), nil } diff --git a/pkg/probod/tracker_mapping.go b/pkg/probod/tracker_mapping.go index f29e0df08..0b13a25b1 100644 --- a/pkg/probod/tracker_mapping.go +++ b/pkg/probod/tracker_mapping.go @@ -49,6 +49,6 @@ func (impl *Implm) buildTrackerMappingConfig( return cookiebanner.TrackerMappingConfig{ LLMClient: llmClient, Model: agentCfg.ModelName, - FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey, + FirecrawlAPIKey: impl.cfg.Agents.Tools.FirecrawlAPIKey, }, nil } diff --git a/pkg/probodconfig/config.go b/pkg/probodconfig/config.go index 852445e9f..9e789abaf 100644 --- a/pkg/probodconfig/config.go +++ b/pkg/probodconfig/config.go @@ -47,11 +47,6 @@ type ( TSAURL string `json:"tsa-url"` } - // FirecrawlConfig contains Firecrawl web scraping API configuration. - FirecrawlConfig struct { - APIKey string `json:"api-key"` - } - // Config represents the probod application configuration. Config struct { BaseURL string `json:"base-url"` @@ -66,7 +61,6 @@ type ( Agents AgentsConfig `json:"llm"` EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"` ChromeDPAddr string `json:"chrome-dp-addr"` - Firecrawl FirecrawlConfig `json:"firecrawl"` CustomDomains CustomDomainsConfig `json:"custom-domains"` SCIMBridge SCIMBridgeConfig `json:"scim-bridge"` ESign ESignConfig `json:"esign"` diff --git a/pkg/probodconfig/llm_config.go b/pkg/probodconfig/llm_config.go index 1f05020d9..3f7dfa912 100644 --- a/pkg/probodconfig/llm_config.go +++ b/pkg/probodconfig/llm_config.go @@ -40,6 +40,12 @@ type ( MaxConcurrency int `json:"max-concurrency"` } + // AgentToolsConfig holds API keys and settings for external tools + // that agents can use (web search, scraping, etc.). + AgentToolsConfig struct { + FirecrawlAPIKey string `json:"firecrawl-api-key"` + } + // AgentsConfig groups LLM provider credentials and per-agent model // settings. Default is used as a fallback when an agent-specific field // is zero-valued. @@ -50,6 +56,7 @@ type ( EvidenceDescriber LLMAgentConfig `json:"evidence-describer"` ThirdPartyAssessor LLMAgentConfig `json:"third-party-assessor"` TrackerMapping LLMAgentConfig `json:"tracker-mapping"` + Tools AgentToolsConfig `json:"tools"` } )