From 76d42ef2db9ae1f7e1bcc3adb4e7cd50c1fc80db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89mile=20R=C3=A9?= Date: Tue, 19 May 2026 10:28:05 +0400 Subject: [PATCH] Hardcode Firecrawl API endpoint, drop FIRECRAWL_ENDPOINT config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Firecrawl has a single public API at https://api.firecrawl.dev/v2. The endpoint was configurable but never varied across environments, so hardcode it as a package-level const and remove the Endpoint field from FirecrawlConfig and all downstream wiring (bootstrap, Helm chart, probod, vetting, cookiebanner). Signed-off-by: Émile Ré --- .../helm/charts/probo/templates/deployment.yaml | 4 ---- .../charts/probo/values-production.yaml.example | 1 - contrib/helm/charts/probo/values.yaml | 1 - pkg/agent/tools/search/firecrawl.go | 13 +++++++------ pkg/agent/tools/search/government_db.go | 4 ++-- pkg/bootstrap/builder.go | 3 +-- pkg/bootstrap/builder_test.go | 3 --- pkg/cookiebanner/tracker_mapping_worker.go | 11 +++++------ pkg/probod/third_party_assessor.go | 13 ++++++------- pkg/probod/tracker_mapping.go | 7 +++---- pkg/probodconfig/config.go | 3 +-- pkg/vetting/assessment.go | 14 ++++++-------- pkg/vetting/orchestrator.go | 9 ++++----- 13 files changed, 35 insertions(+), 51 deletions(-) diff --git a/contrib/helm/charts/probo/templates/deployment.yaml b/contrib/helm/charts/probo/templates/deployment.yaml index 38128ef85..1923fd479 100644 --- a/contrib/helm/charts/probo/templates/deployment.yaml +++ b/contrib/helm/charts/probo/templates/deployment.yaml @@ -233,10 +233,6 @@ spec: value: {{ .Values.probo.openai.maxTokens | quote }} {{- end }} # Firecrawl Integration - {{- if .Values.probo.firecrawl.endpoint }} - - name: FIRECRAWL_ENDPOINT - value: {{ .Values.probo.firecrawl.endpoint | quote }} - {{- end }} {{- if .Values.probo.firecrawl.apiKey }} - name: FIRECRAWL_API_KEY valueFrom: diff --git a/contrib/helm/charts/probo/values-production.yaml.example b/contrib/helm/charts/probo/values-production.yaml.example index 54eff370c..9ec501c00 100644 --- a/contrib/helm/charts/probo/values-production.yaml.example +++ b/contrib/helm/charts/probo/values-production.yaml.example @@ -161,7 +161,6 @@ probo: # Firecrawl web search (optional, used by tracker mapping agent) # firecrawl: - # endpoint: "https://api.firecrawl.dev/v2" # apiKey: "CHANGE_ME_FIRECRAWL_API_KEY" # Tracker mapping agent (optional, auto-links tracker patterns to vendors) diff --git a/contrib/helm/charts/probo/values.yaml b/contrib/helm/charts/probo/values.yaml index bd5445976..60c37c568 100644 --- a/contrib/helm/charts/probo/values.yaml +++ b/contrib/helm/charts/probo/values.yaml @@ -259,7 +259,6 @@ probo: # Firecrawl web search integration (optional, used by tracker mapping agent) firecrawl: - endpoint: "" apiKey: "" # Tracker mapping agent (optional, requires openai.apiKey or anthropic key) diff --git a/pkg/agent/tools/search/firecrawl.go b/pkg/agent/tools/search/firecrawl.go index d3545ea53..7324a47f4 100644 --- a/pkg/agent/tools/search/firecrawl.go +++ b/pkg/agent/tools/search/firecrawl.go @@ -26,6 +26,8 @@ import ( "go.probo.inc/probo/pkg/agent" ) +const firecrawlBaseURL = "https://api.firecrawl.dev/v2" + type ( searchResult struct { Title string `json:"title"` @@ -58,9 +60,8 @@ type ( ) // FirecrawlSearchTool creates a tool that searches the web using the Firecrawl -// API. The endpoint should be the base URL of the Firecrawl instance (e.g. -// "https://api.firecrawl.dev/v2"). The apiKey is used for Bearer authentication. -func FirecrawlSearchTool(endpoint, apiKey string) agent.Tool { +// API. The apiKey is used for Bearer authentication. +func FirecrawlSearchTool(apiKey string) agent.Tool { client := newHTTPClient() return agent.FunctionTool( @@ -75,7 +76,7 @@ func FirecrawlSearchTool(endpoint, apiKey string) agent.Tool { maxResults = 10 } - results, err := firecrawlSearch(ctx, client, endpoint, apiKey, p.Query, maxResults) + results, err := firecrawlSearch(ctx, client, apiKey, p.Query, maxResults) if err != nil { return agent.ResultErrorf("search request failed: %s", err), nil } @@ -88,10 +89,10 @@ func FirecrawlSearchTool(endpoint, apiKey string) agent.Tool { func firecrawlSearch( ctx context.Context, client *http.Client, - endpoint, apiKey, query string, + apiKey, query string, maxResults int, ) ([]searchResult, error) { - u, err := url.JoinPath(endpoint, "search") + u, err := url.JoinPath(firecrawlBaseURL, "search") if err != nil { return nil, fmt.Errorf("cannot build search URL: %w", err) } diff --git a/pkg/agent/tools/search/government_db.go b/pkg/agent/tools/search/government_db.go index 132771d2d..ea1123e45 100644 --- a/pkg/agent/tools/search/government_db.go +++ b/pkg/agent/tools/search/government_db.go @@ -43,7 +43,7 @@ type ( } ) -func CheckGovernmentDBTool(endpoint, apiKey string) agent.Tool { +func CheckGovernmentDBTool(apiKey string) agent.Tool { client := newHTTPClient() return agent.FunctionTool( @@ -87,7 +87,7 @@ func CheckGovernmentDBTool(endpoint, apiKey string) agent.Tool { } for _, s := range searches { - entries, err := firecrawlSearch(ctx, client, endpoint, apiKey, s.query, 3) + entries, err := firecrawlSearch(ctx, client, apiKey, s.query, 3) if err != nil { continue } diff --git a/pkg/bootstrap/builder.go b/pkg/bootstrap/builder.go index cc9787c69..02aeaf53e 100644 --- a/pkg/bootstrap/builder.go +++ b/pkg/bootstrap/builder.go @@ -177,8 +177,7 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { }, }, Firecrawl: probodconfig.FirecrawlConfig{ - Endpoint: b.getEnv("FIRECRAWL_ENDPOINT"), - APIKey: b.getEnv("FIRECRAWL_API_KEY"), + APIKey: b.getEnv("FIRECRAWL_API_KEY"), }, Agents: probodconfig.AgentsConfig{ Providers: map[string]probodconfig.LLMProviderConfig{ diff --git a/pkg/bootstrap/builder_test.go b/pkg/bootstrap/builder_test.go index b93e43664..0dcadedf1 100644 --- a/pkg/bootstrap/builder_test.go +++ b/pkg/bootstrap/builder_test.go @@ -198,7 +198,6 @@ func TestBuilder_Build_Defaults(t *testing.T) { assert.Equal(t, 86400, cfg.Probod.Notifications.Webhook.CacheTTL) // Firecrawl — empty by default - assert.Empty(t, cfg.Probod.Firecrawl.Endpoint) assert.Empty(t, cfg.Probod.Firecrawl.APIKey) // Agents config — default @@ -294,7 +293,6 @@ func TestBuilder_Build_CustomValues(t *testing.T) { env["WEBHOOK_CACHE_TTL"] = "3600" env["CONNECTOR_SLACK_SIGNING_SECRET"] = "slack-signing-secret" // Firecrawl - env["FIRECRAWL_ENDPOINT"] = "https://api.firecrawl.dev/v2" env["FIRECRAWL_API_KEY"] = "fc-test-key" // Agents — providers env["OPENAI_API_KEY"] = "sk-test-key" @@ -381,7 +379,6 @@ func TestBuilder_Build_CustomValues(t *testing.T) { assert.Equal(t, 10, cfg.Probod.Notifications.Webhook.SenderInterval) assert.Equal(t, 3600, cfg.Probod.Notifications.Webhook.CacheTTL) // Firecrawl - assert.Equal(t, "https://api.firecrawl.dev/v2", cfg.Probod.Firecrawl.Endpoint) assert.Equal(t, "fc-test-key", cfg.Probod.Firecrawl.APIKey) // Agents — providers assert.Equal(t, "openai", cfg.Probod.Agents.Providers["openai"].Type) diff --git a/pkg/cookiebanner/tracker_mapping_worker.go b/pkg/cookiebanner/tracker_mapping_worker.go index b8558a899..630df267d 100644 --- a/pkg/cookiebanner/tracker_mapping_worker.go +++ b/pkg/cookiebanner/tracker_mapping_worker.go @@ -51,10 +51,9 @@ type trackerMappingHandler struct { } type TrackerMappingConfig struct { - LLMClient *llm.Client - Model string - FirecrawlEndpoint string - FirecrawlAPIKey string + LLMClient *llm.Client + Model string + FirecrawlAPIKey string } func NewTrackerMappingWorker( @@ -90,8 +89,8 @@ func buildTrackerMappingAgent( searchThirdPartiesTool(pgClient), } - if cfg.FirecrawlEndpoint != "" && cfg.FirecrawlAPIKey != "" { - tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlEndpoint, cfg.FirecrawlAPIKey)) + if cfg.FirecrawlAPIKey != "" { + tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlAPIKey)) } outputType, err := agent.NewOutputType[TrackerIdentification]("tracker_identification") diff --git a/pkg/probod/third_party_assessor.go b/pkg/probod/third_party_assessor.go index 0105ca326..443f3758c 100644 --- a/pkg/probod/third_party_assessor.go +++ b/pkg/probod/third_party_assessor.go @@ -48,12 +48,11 @@ func (impl *Implm) buildThirdPartyAssessor( } return vetting.NewAssessor(vetting.Config{ - Client: llmClient, - Model: agentCfg.ModelName, - MaxTokens: maxTokens, - ChromeAddr: impl.cfg.ChromeDPAddr, - FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint, - FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey, - Logger: l.Named("third-party-assessor"), + Client: llmClient, + Model: agentCfg.ModelName, + MaxTokens: maxTokens, + ChromeAddr: impl.cfg.ChromeDPAddr, + FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey, + Logger: l.Named("third-party-assessor"), }), nil } diff --git a/pkg/probod/tracker_mapping.go b/pkg/probod/tracker_mapping.go index 89a5e4d0a..f29e0df08 100644 --- a/pkg/probod/tracker_mapping.go +++ b/pkg/probod/tracker_mapping.go @@ -47,9 +47,8 @@ func (impl *Implm) buildTrackerMappingConfig( } return cookiebanner.TrackerMappingConfig{ - LLMClient: llmClient, - Model: agentCfg.ModelName, - FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint, - FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey, + LLMClient: llmClient, + Model: agentCfg.ModelName, + FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey, }, nil } diff --git a/pkg/probodconfig/config.go b/pkg/probodconfig/config.go index 50fd1e8f3..852445e9f 100644 --- a/pkg/probodconfig/config.go +++ b/pkg/probodconfig/config.go @@ -49,8 +49,7 @@ type ( // FirecrawlConfig contains Firecrawl web scraping API configuration. FirecrawlConfig struct { - Endpoint string `json:"endpoint"` - APIKey string `json:"api-key"` + APIKey string `json:"api-key"` } // Config represents the probod application configuration. diff --git a/pkg/vetting/assessment.go b/pkg/vetting/assessment.go index 6fc08ff9b..c27bd60a6 100644 --- a/pkg/vetting/assessment.go +++ b/pkg/vetting/assessment.go @@ -71,13 +71,12 @@ var ( type ( Config struct { - Client *llm.Client - Model string - MaxTokens int - ChromeAddr string - FirecrawlEndpoint string - FirecrawlAPIKey string - Logger *log.Logger + Client *llm.Client + Model string + MaxTokens int + ChromeAddr string + FirecrawlAPIKey string + Logger *log.Logger } Assessor struct { @@ -210,7 +209,6 @@ func (a *Assessor) Assess(ctx context.Context, websiteURL string, procedure stri a.cfg.Logger, thirdPartyBrowser, researchBrowser, - a.cfg.FirecrawlEndpoint, a.cfg.FirecrawlAPIKey, reporter, ) diff --git a/pkg/vetting/orchestrator.go b/pkg/vetting/orchestrator.go index 43fb46eec..3becd5e42 100644 --- a/pkg/vetting/orchestrator.go +++ b/pkg/vetting/orchestrator.go @@ -65,7 +65,6 @@ func newOrchestratorAgent( logger *log.Logger, thirdPartyBrowser *browser.Browser, researchBrowser *browser.Browser, - firecrawlEndpoint string, firecrawlAPIKey string, reporter agent.ProgressReporter, ) (*agent.Agent, error) { @@ -89,13 +88,13 @@ func newOrchestratorAgent( return opts } - hasFirecrawl := firecrawlEndpoint != "" && firecrawlAPIKey != "" + hasFirecrawl := firecrawlAPIKey != "" // Subprocessor agent benefits from web search when available so it can // find subprocessor pages hosted on third-party platforms. subprocessorTools := unrestrictedBrowserTools if hasFirecrawl { - subprocessorTools = append(subprocessorTools, search.FirecrawlSearchTool(firecrawlEndpoint, firecrawlAPIKey)) + subprocessorTools = append(subprocessorTools, search.FirecrawlSearchTool(firecrawlAPIKey)) } // Core sub-agents that always run. @@ -178,8 +177,8 @@ func newOrchestratorAgent( if hasFirecrawl { researchBrowserTools := browser.NewInteractiveToolset(researchBrowser).Tools() - searchTool := search.FirecrawlSearchTool(firecrawlEndpoint, firecrawlAPIKey) - govDBTool := search.CheckGovernmentDBTool(firecrawlEndpoint, firecrawlAPIKey) + searchTool := search.FirecrawlSearchTool(firecrawlAPIKey) + govDBTool := search.CheckGovernmentDBTool(firecrawlAPIKey) waybackTool := search.CheckWaybackTool() diffTool := search.DiffDocumentsTool()