Hardcode Firecrawl API endpoint, drop FIRECRAWL_ENDPOINT config

Firecrawl has a single public API at https://api.firecrawl.dev/v2.
The endpoint was configurable but never varied across environments,
so hardcode it as a package-level const and remove the Endpoint
field from FirecrawlConfig and all downstream wiring (bootstrap,
Helm chart, probod, vetting, cookiebanner).

Signed-off-by: Émile Ré <emile@probo.com>

contrib/helm/charts/probo/templates/deployment.yaml

@@ -233,10 +233,6 @@ spec:
               value: {{ .Values.probo.openai.maxTokens | quote }}
             {{- end }}
             # Firecrawl Integration
-            {{- if .Values.probo.firecrawl.endpoint }}
-            - name: FIRECRAWL_ENDPOINT
-              value: {{ .Values.probo.firecrawl.endpoint | quote }}
-            {{- end }}
             {{- if .Values.probo.firecrawl.apiKey }}
             - name: FIRECRAWL_API_KEY
               valueFrom:

contrib/helm/charts/probo/values-production.yaml.example

@@ -161,7 +161,6 @@ probo:
 
   # Firecrawl web search (optional, used by tracker mapping agent)
   # firecrawl:
-  #   endpoint: "https://api.firecrawl.dev/v2"
   #   apiKey: "CHANGE_ME_FIRECRAWL_API_KEY"
 
   # Tracker mapping agent (optional, auto-links tracker patterns to vendors)

contrib/helm/charts/probo/values.yaml

@@ -259,7 +259,6 @@ probo:
 
   # Firecrawl web search integration (optional, used by tracker mapping agent)
   firecrawl:
-    endpoint: ""
     apiKey: ""
 
   # Tracker mapping agent (optional, requires openai.apiKey or anthropic key)

pkg/agent/tools/search/firecrawl.go

@@ -26,6 +26,8 @@ import (
 	"go.probo.inc/probo/pkg/agent"
 )
 
+const firecrawlBaseURL = "https://api.firecrawl.dev/v2"
+
 type (
 	searchResult struct {
 		Title   string `json:"title"`
@@ -58,9 +60,8 @@ type (
 )
 
 // FirecrawlSearchTool creates a tool that searches the web using the Firecrawl
-// API. The endpoint should be the base URL of the Firecrawl instance (e.g.
-// "https://api.firecrawl.dev/v2"). The apiKey is used for Bearer authentication.
-func FirecrawlSearchTool(endpoint, apiKey string) agent.Tool {
+// API. The apiKey is used for Bearer authentication.
+func FirecrawlSearchTool(apiKey string) agent.Tool {
 	client := newHTTPClient()
 
 	return agent.FunctionTool(
@@ -75,7 +76,7 @@ func FirecrawlSearchTool(endpoint, apiKey string) agent.Tool {
 				maxResults = 10
 			}
 
-			results, err := firecrawlSearch(ctx, client, endpoint, apiKey, p.Query, maxResults)
+			results, err := firecrawlSearch(ctx, client, apiKey, p.Query, maxResults)
 			if err != nil {
 				return agent.ResultErrorf("search request failed: %s", err), nil
 			}
@@ -88,10 +89,10 @@ func FirecrawlSearchTool(endpoint, apiKey string) agent.Tool {
 func firecrawlSearch(
 	ctx context.Context,
 	client *http.Client,
-	endpoint, apiKey, query string,
+	apiKey, query string,
 	maxResults int,
 ) ([]searchResult, error) {
-	u, err := url.JoinPath(endpoint, "search")
+	u, err := url.JoinPath(firecrawlBaseURL, "search")
 	if err != nil {
 		return nil, fmt.Errorf("cannot build search URL: %w", err)
 	}

pkg/agent/tools/search/government_db.go

@@ -43,7 +43,7 @@ type (
 	}
 )
 
-func CheckGovernmentDBTool(endpoint, apiKey string) agent.Tool {
+func CheckGovernmentDBTool(apiKey string) agent.Tool {
 	client := newHTTPClient()
 
 	return agent.FunctionTool(
@@ -87,7 +87,7 @@ func CheckGovernmentDBTool(endpoint, apiKey string) agent.Tool {
 			}
 
 			for _, s := range searches {
-				entries, err := firecrawlSearch(ctx, client, endpoint, apiKey, s.query, 3)
+				entries, err := firecrawlSearch(ctx, client, apiKey, s.query, 3)
 				if err != nil {
 					continue
 				}

pkg/bootstrap/builder.go

@@ -177,8 +177,7 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
 				},
 			},
 			Firecrawl: probodconfig.FirecrawlConfig{
-				Endpoint: b.getEnv("FIRECRAWL_ENDPOINT"),
-				APIKey:   b.getEnv("FIRECRAWL_API_KEY"),
+				APIKey: b.getEnv("FIRECRAWL_API_KEY"),
 			},
 			Agents: probodconfig.AgentsConfig{
 				Providers: map[string]probodconfig.LLMProviderConfig{

pkg/bootstrap/builder_test.go

@@ -198,7 +198,6 @@ func TestBuilder_Build_Defaults(t *testing.T) {
 	assert.Equal(t, 86400, cfg.Probod.Notifications.Webhook.CacheTTL)
 
 	// Firecrawl — empty by default
-	assert.Empty(t, cfg.Probod.Firecrawl.Endpoint)
 	assert.Empty(t, cfg.Probod.Firecrawl.APIKey)
 
 	// Agents config — default
@@ -294,7 +293,6 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
 	env["WEBHOOK_CACHE_TTL"] = "3600"
 	env["CONNECTOR_SLACK_SIGNING_SECRET"] = "slack-signing-secret"
 	// Firecrawl
-	env["FIRECRAWL_ENDPOINT"] = "https://api.firecrawl.dev/v2"
 	env["FIRECRAWL_API_KEY"] = "fc-test-key"
 	// Agents — providers
 	env["OPENAI_API_KEY"] = "sk-test-key"
@@ -381,7 +379,6 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
 	assert.Equal(t, 10, cfg.Probod.Notifications.Webhook.SenderInterval)
 	assert.Equal(t, 3600, cfg.Probod.Notifications.Webhook.CacheTTL)
 	// Firecrawl
-	assert.Equal(t, "https://api.firecrawl.dev/v2", cfg.Probod.Firecrawl.Endpoint)
 	assert.Equal(t, "fc-test-key", cfg.Probod.Firecrawl.APIKey)
 	// Agents — providers
 	assert.Equal(t, "openai", cfg.Probod.Agents.Providers["openai"].Type)

pkg/cookiebanner/tracker_mapping_worker.go

@@ -51,10 +51,9 @@ type trackerMappingHandler struct {
 }
 
 type TrackerMappingConfig struct {
-	LLMClient         *llm.Client
-	Model             string
-	FirecrawlEndpoint string
-	FirecrawlAPIKey   string
+	LLMClient       *llm.Client
+	Model           string
+	FirecrawlAPIKey string
 }
 
 func NewTrackerMappingWorker(
@@ -90,8 +89,8 @@ func buildTrackerMappingAgent(
 		searchThirdPartiesTool(pgClient),
 	}
 
-	if cfg.FirecrawlEndpoint != "" && cfg.FirecrawlAPIKey != "" {
-		tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlEndpoint, cfg.FirecrawlAPIKey))
+	if cfg.FirecrawlAPIKey != "" {
+		tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlAPIKey))
 	}
 
 	outputType, err := agent.NewOutputType[TrackerIdentification]("tracker_identification")

pkg/probod/third_party_assessor.go

@@ -48,12 +48,11 @@ func (impl *Implm) buildThirdPartyAssessor(
 	}
 
 	return vetting.NewAssessor(vetting.Config{
-		Client:            llmClient,
-		Model:             agentCfg.ModelName,
-		MaxTokens:         maxTokens,
-		ChromeAddr:        impl.cfg.ChromeDPAddr,
-		FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint,
-		FirecrawlAPIKey:   impl.cfg.Firecrawl.APIKey,
-		Logger:            l.Named("third-party-assessor"),
+		Client:          llmClient,
+		Model:           agentCfg.ModelName,
+		MaxTokens:       maxTokens,
+		ChromeAddr:      impl.cfg.ChromeDPAddr,
+		FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey,
+		Logger:          l.Named("third-party-assessor"),
 	}), nil
 }

pkg/probod/tracker_mapping.go

@@ -47,9 +47,8 @@ func (impl *Implm) buildTrackerMappingConfig(
 	}
 
 	return cookiebanner.TrackerMappingConfig{
-		LLMClient:         llmClient,
-		Model:             agentCfg.ModelName,
-		FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint,
-		FirecrawlAPIKey:   impl.cfg.Firecrawl.APIKey,
+		LLMClient:       llmClient,
+		Model:           agentCfg.ModelName,
+		FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey,
 	}, nil
 }

pkg/probodconfig/config.go

@@ -49,8 +49,7 @@ type (
 
 	// FirecrawlConfig contains Firecrawl web scraping API configuration.
 	FirecrawlConfig struct {
-		Endpoint string `json:"endpoint"`
-		APIKey   string `json:"api-key"`
+		APIKey string `json:"api-key"`
 	}
 
 	// Config represents the probod application configuration.

pkg/vetting/assessment.go

@@ -71,13 +71,12 @@ var (
 
 type (
 	Config struct {
-		Client            *llm.Client
-		Model             string
-		MaxTokens         int
-		ChromeAddr        string
-		FirecrawlEndpoint string
-		FirecrawlAPIKey   string
-		Logger            *log.Logger
+		Client          *llm.Client
+		Model           string
+		MaxTokens       int
+		ChromeAddr      string
+		FirecrawlAPIKey string
+		Logger          *log.Logger
 	}
 
 	Assessor struct {
@@ -210,7 +209,6 @@ func (a *Assessor) Assess(ctx context.Context, websiteURL string, procedure stri
 		a.cfg.Logger,
 		thirdPartyBrowser,
 		researchBrowser,
-		a.cfg.FirecrawlEndpoint,
 		a.cfg.FirecrawlAPIKey,
 		reporter,
 	)

pkg/vetting/orchestrator.go

@@ -65,7 +65,6 @@ func newOrchestratorAgent(
 	logger *log.Logger,
 	thirdPartyBrowser *browser.Browser,
 	researchBrowser *browser.Browser,
-	firecrawlEndpoint string,
 	firecrawlAPIKey string,
 	reporter agent.ProgressReporter,
 ) (*agent.Agent, error) {
@@ -89,13 +88,13 @@ func newOrchestratorAgent(
 		return opts
 	}
 
-	hasFirecrawl := firecrawlEndpoint != "" && firecrawlAPIKey != ""
+	hasFirecrawl := firecrawlAPIKey != ""
 
 	// Subprocessor agent benefits from web search when available so it can
 	// find subprocessor pages hosted on third-party platforms.
 	subprocessorTools := unrestrictedBrowserTools
 	if hasFirecrawl {
-		subprocessorTools = append(subprocessorTools, search.FirecrawlSearchTool(firecrawlEndpoint, firecrawlAPIKey))
+		subprocessorTools = append(subprocessorTools, search.FirecrawlSearchTool(firecrawlAPIKey))
 	}
 
 	// Core sub-agents that always run.
@@ -178,8 +177,8 @@ func newOrchestratorAgent(
 	if hasFirecrawl {
 		researchBrowserTools := browser.NewInteractiveToolset(researchBrowser).Tools()
 
-		searchTool := search.FirecrawlSearchTool(firecrawlEndpoint, firecrawlAPIKey)
-		govDBTool := search.CheckGovernmentDBTool(firecrawlEndpoint, firecrawlAPIKey)
+		searchTool := search.FirecrawlSearchTool(firecrawlAPIKey)
+		govDBTool := search.CheckGovernmentDBTool(firecrawlAPIKey)
 		waybackTool := search.CheckWaybackTool()
 		diffTool := search.DiffDocumentsTool()