Remove SearXNG search backend, use Firecrawl exclusively

SearXNG was a fallback search backend that added complexity without
being used in practice. All search-dependent features (web search,
government DB checks, vetting orchestrator, tracker mapping) now use
Firecrawl exclusively. Removes the SEARCH_ENDPOINT config plumbing
from probodconfig, bootstrap, Helm charts, and all callers.

Signed-off-by: Émile Ré <emile@probo.com>

contrib/helm/charts/probo/templates/deployment.yaml

@@ -244,11 +244,6 @@ spec:
                   name: {{ include "probo.fullname" . }}
                   key: firecrawl-api-key
             {{- end }}
-            # Search Endpoint
-            {{- if .Values.probo.searchEndpoint }}
-            - name: SEARCH_ENDPOINT
-              value: {{ .Values.probo.searchEndpoint | quote }}
-            {{- end }}
             # Tracker Mapping Agent
             {{- if .Values.probo.trackerMapping.provider }}
             - name: AGENT_TRACKER_MAPPING_PROVIDER

contrib/helm/charts/probo/values-production.yaml.example

@@ -164,9 +164,6 @@ probo:
   #   endpoint: "https://api.firecrawl.dev/v2"
   #   apiKey: "CHANGE_ME_FIRECRAWL_API_KEY"
 
-  # SearXNG search endpoint (optional, fallback when Firecrawl is not configured)
-  # searchEndpoint: "https://search.example.com"
-
   # Tracker mapping agent (optional, auto-links tracker patterns to vendors)
   # trackerMapping:
   #   provider: "openai"

contrib/helm/charts/probo/values.yaml

@@ -262,9 +262,6 @@ probo:
     endpoint: ""
     apiKey: ""
 
-  # Search endpoint for SearXNG (optional, fallback when Firecrawl is not configured)
-  searchEndpoint: ""
-
   # Tracker mapping agent (optional, requires openai.apiKey or anthropic key)
   trackerMapping:
     provider: ""

pkg/agent/tools/search/firecrawl.go

@@ -22,12 +22,23 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"time"
 
 	"go.gearno.de/kit/httpclient"
 	"go.probo.inc/probo/pkg/agent"
 )
 
 type (
+	searchResult struct {
+		Title   string `json:"title"`
+		URL     string `json:"url"`
+		Snippet string `json:"snippet"`
+	}
+
+	userAgentTransport struct {
+		next http.RoundTripper
+	}
+
 	firecrawlParams struct {
 		Query      string `json:"query" jsonschema:"The search query to execute"`
 		MaxResults int    `json:"max_results" jsonschema:"Maximum number of results to return (default 5, max 10)"`
@@ -52,11 +63,19 @@ type (
 	}
 )
 
+func (t *userAgentTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	r2 := r.Clone(r.Context())
+	r2.Header.Set("User-Agent", "Probo-Agent/1.0")
+	return t.next.RoundTrip(r2)
+}
+
 // FirecrawlSearchTool creates a tool that searches the web using the Firecrawl
 // API. The endpoint should be the base URL of the Firecrawl instance (e.g.
 // "https://api.firecrawl.dev/v2"). The apiKey is used for Bearer authentication.
 func FirecrawlSearchTool(endpoint, apiKey string) agent.Tool {
 	client := httpclient.DefaultPooledClient()
+	client.Timeout = 15 * time.Second
+	client.Transport = &userAgentTransport{next: client.Transport}
 
 	return agent.FunctionTool(
 		"web_search",

pkg/agent/tools/search/government_db.go

@@ -16,11 +16,7 @@ package search
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
-	"io"
-	"net/http"
-	"net/url"
 
 	"go.gearno.de/kit/httpclient"
 	"go.probo.inc/probo/pkg/agent"
@@ -48,7 +44,7 @@ type (
 	}
 )
 
-func CheckGovernmentDBTool(searchEndpoint string) agent.Tool {
+func CheckGovernmentDBTool(endpoint, apiKey string) agent.Tool {
 	client := httpclient.DefaultPooledClient()
 	client.Transport = &userAgentTransport{next: client.Transport}
 
@@ -93,17 +89,20 @@ func CheckGovernmentDBTool(searchEndpoint string) agent.Tool {
 			}
 
 			for _, s := range searches {
-				entries, err := searxngSearch(ctx, client, searchEndpoint, s.query, 3)
+				entries, err := firecrawlSearch(ctx, client, endpoint, apiKey, s.query, 3)
 				if err != nil {
 					continue
 				}
 				for _, e := range entries {
-					*s.target = append(*s.target, govDBEntry{
+					*s.target = append(
+						*s.target,
+						govDBEntry{
 							Source:  s.source,
 							Title:   e.Title,
 							URL:     e.URL,
 							Snippet: e.Snippet,
-					})
+						},
+					)
 				}
 			}
 
@@ -111,60 +110,3 @@ func CheckGovernmentDBTool(searchEndpoint string) agent.Tool {
 		},
 	)
 }
-
-func searxngSearch(ctx context.Context, client *http.Client, endpoint, query string, maxResults int) ([]searchResult, error) {
-	u, err := url.JoinPath(endpoint, "search")
-	if err != nil {
-		return nil, fmt.Errorf("cannot build search URL: %w", err)
-	}
-
-	parsed, err := url.Parse(u)
-	if err != nil {
-		return nil, fmt.Errorf("cannot parse search URL: %w", err)
-	}
-
-	q := parsed.Query()
-	q.Set("q", query)
-	q.Set("format", "json")
-	q.Set("categories", "general")
-	parsed.RawQuery = q.Encode()
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
-	if err != nil {
-		return nil, fmt.Errorf("cannot create request: %w", err)
-	}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("cannot execute search request: %w", err)
-	}
-	defer func() { _ = resp.Body.Close() }()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("cannot read response body: %w", err)
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("search returned status %d", resp.StatusCode)
-	}
-
-	var searxResp searxngResponse
-	if err := json.Unmarshal(body, &searxResp); err != nil {
-		return nil, fmt.Errorf("cannot unmarshal response: %w", err)
-	}
-
-	results := make([]searchResult, 0, maxResults)
-	for i, r := range searxResp.Results {
-		if i >= maxResults {
-			break
-		}
-		results = append(results, searchResult{
-			Title:   r.Title,
-			URL:     r.URL,
-			Snippet: r.Content,
-		})
-	}
-
-	return results, nil
-}

pkg/agent/tools/search/search.go

@@ -1,71 +0,0 @@
-// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
-//
-// Permission to use, copy, modify, and/or distribute this software for any
-// purpose with or without fee is hereby granted, provided that the above
-// copyright notice and this permission notice appear in all copies.
-//
-// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
-// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
-// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
-// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-// PERFORMANCE OF THIS SOFTWARE.
-
-package search
-
-import (
-	"go.probo.inc/probo/pkg/agent"
-)
-
-// Toolset provides web search tools.
-type Toolset struct {
-	endpoint          string
-	firecrawlEndpoint string
-	firecrawlAPIKey   string
-}
-
-// Option configures a Toolset.
-type Option func(*Toolset)
-
-// WithFirecrawl configures the Firecrawl search backend. The endpoint
-// should be the base URL of the Firecrawl instance (e.g.
-// "https://api.firecrawl.dev/v2").
-func WithFirecrawl(endpoint, apiKey string) Option {
-	return func(t *Toolset) {
-		t.firecrawlEndpoint = endpoint
-		t.firecrawlAPIKey = apiKey
-	}
-}
-
-// NewToolset creates a search toolset with the given SearXNG endpoint.
-func NewToolset(endpoint string, opts ...Option) *Toolset {
-	t := &Toolset{endpoint: endpoint}
-	for _, opt := range opts {
-		opt(t)
-	}
-	return t
-}
-
-func (t *Toolset) Tools() []agent.Tool {
-	tools := []agent.Tool{
-		CheckWaybackTool(),
-		DiffDocumentsTool(),
-	}
-
-	if t.firecrawlEndpoint != "" && t.firecrawlAPIKey != "" {
-		tools = append(
-			tools,
-			FirecrawlSearchTool(t.firecrawlEndpoint, t.firecrawlAPIKey),
-			CheckGovernmentDBTool(t.endpoint),
-		)
-	} else if t.endpoint != "" {
-		tools = append(
-			tools,
-			WebSearchTool(t.endpoint),
-			CheckGovernmentDBTool(t.endpoint),
-		)
-	}
-
-	return tools
-}

pkg/agent/tools/search/web_search.go

@@ -1,85 +0,0 @@
-// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
-//
-// Permission to use, copy, modify, and/or distribute this software for any
-// purpose with or without fee is hereby granted, provided that the above
-// copyright notice and this permission notice appear in all copies.
-//
-// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
-// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
-// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
-// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-// PERFORMANCE OF THIS SOFTWARE.
-
-package search
-
-import (
-	"context"
-	"net/http"
-
-	"go.gearno.de/kit/httpclient"
-	"go.probo.inc/probo/pkg/agent"
-)
-
-type (
-	searchParams struct {
-		Query      string `json:"query" jsonschema:"The search query to execute"`
-		MaxResults int    `json:"max_results" jsonschema:"Maximum number of results to return (default 5, max 10)"`
-	}
-
-	searchResult struct {
-		Title   string `json:"title"`
-		URL     string `json:"url"`
-		Snippet string `json:"snippet"`
-	}
-
-	searxngResponse struct {
-		Results []searxngResult `json:"results"`
-	}
-
-	searxngResult struct {
-		Title   string `json:"title"`
-		URL     string `json:"url"`
-		Content string `json:"content"`
-	}
-
-	userAgentTransport struct {
-		next http.RoundTripper
-	}
-)
-
-func (t *userAgentTransport) RoundTrip(r *http.Request) (*http.Response, error) {
-	r2 := r.Clone(r.Context())
-	r2.Header.Set("User-Agent", "Probo-Agent/1.0")
-	return t.next.RoundTrip(r2)
-}
-
-// WebSearchTool creates a tool that searches the web using a SearXNG instance.
-// The endpoint should be the base URL of the SearXNG instance (e.g.
-// "http://localhost:8888").
-func WebSearchTool(endpoint string) agent.Tool {
-	client := httpclient.DefaultPooledClient()
-	client.Transport = &userAgentTransport{next: client.Transport}
-
-	return agent.FunctionTool(
-		"web_search",
-		"Search the web for information about a topic. Returns a list of results with title, URL, and snippet. Use this to find news, reviews, breach reports, regulatory actions, and other external information about a vendor.",
-		func(ctx context.Context, p searchParams) (agent.ToolResult, error) {
-			maxResults := p.MaxResults
-			if maxResults <= 0 {
-				maxResults = 5
-			}
-			if maxResults > 10 {
-				maxResults = 10
-			}
-
-			results, err := searxngSearch(ctx, client, endpoint, p.Query, maxResults)
-			if err != nil {
-				return agent.ResultErrorf("search request failed: %s", err), nil
-			}
-
-			return agent.ResultJSON(results), nil
-		},
-	)
-}

pkg/bootstrap/builder.go

@@ -176,7 +176,6 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
 					CacheTTL:       b.getEnvIntOrDefault("WEBHOOK_CACHE_TTL", 86400),
 				},
 			},
-			SearchEndpoint: b.getEnv("SEARCH_ENDPOINT"),
 			Firecrawl: probodconfig.FirecrawlConfig{
 				Endpoint: b.getEnv("FIRECRAWL_ENDPOINT"),
 				APIKey:   b.getEnv("FIRECRAWL_API_KEY"),

pkg/bootstrap/builder_test.go

@@ -197,8 +197,7 @@ func TestBuilder_Build_Defaults(t *testing.T) {
 	assert.Equal(t, 5, cfg.Probod.Notifications.Webhook.SenderInterval)
 	assert.Equal(t, 86400, cfg.Probod.Notifications.Webhook.CacheTTL)
 
-	// Search and Firecrawl — empty by default
+	// Firecrawl — empty by default
-	assert.Empty(t, cfg.Probod.SearchEndpoint)
 	assert.Empty(t, cfg.Probod.Firecrawl.Endpoint)
 	assert.Empty(t, cfg.Probod.Firecrawl.APIKey)
 
@@ -294,8 +293,7 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
 	env["WEBHOOK_SENDER_INTERVAL"] = "10"
 	env["WEBHOOK_CACHE_TTL"] = "3600"
 	env["CONNECTOR_SLACK_SIGNING_SECRET"] = "slack-signing-secret"
-	// Search and Firecrawl
+	// Firecrawl
-	env["SEARCH_ENDPOINT"] = "https://search.example.com"
 	env["FIRECRAWL_ENDPOINT"] = "https://api.firecrawl.dev/v2"
 	env["FIRECRAWL_API_KEY"] = "fc-test-key"
 	// Agents — providers
@@ -382,8 +380,7 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
 	assert.Equal(t, "slack-signing-secret", cfg.Probod.Notifications.Slack.SigningSecret)
 	assert.Equal(t, 10, cfg.Probod.Notifications.Webhook.SenderInterval)
 	assert.Equal(t, 3600, cfg.Probod.Notifications.Webhook.CacheTTL)
-	// Search and Firecrawl
+	// Firecrawl
-	assert.Equal(t, "https://search.example.com", cfg.Probod.SearchEndpoint)
 	assert.Equal(t, "https://api.firecrawl.dev/v2", cfg.Probod.Firecrawl.Endpoint)
 	assert.Equal(t, "fc-test-key", cfg.Probod.Firecrawl.APIKey)
 	// Agents — providers

pkg/cookiebanner/tracker_mapping_worker.go

@@ -53,7 +53,6 @@ type trackerMappingHandler struct {
 type TrackerMappingConfig struct {
 	LLMClient         *llm.Client
 	Model             string
-	SearchEndpoint    string
 	FirecrawlEndpoint string
 	FirecrawlAPIKey   string
 }
@@ -93,8 +92,6 @@ func buildTrackerMappingAgent(
 
 	if cfg.FirecrawlEndpoint != "" && cfg.FirecrawlAPIKey != "" {
 		tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlEndpoint, cfg.FirecrawlAPIKey))
-	} else if cfg.SearchEndpoint != "" {
-		tools = append(tools, search.WebSearchTool(cfg.SearchEndpoint))
 	}
 
 	outputType, err := agent.NewOutputType[TrackerIdentification]("tracker_identification")

pkg/probod/third_party_assessor.go

@@ -52,7 +52,8 @@ func (impl *Implm) buildThirdPartyAssessor(
 		Model:             agentCfg.ModelName,
 		MaxTokens:         maxTokens,
 		ChromeAddr:        impl.cfg.ChromeDPAddr,
-		SearchEndpoint: impl.cfg.SearchEndpoint,
+		FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint,
+		FirecrawlAPIKey:   impl.cfg.Firecrawl.APIKey,
 		Logger:            l.Named("third-party-assessor"),
 	}), nil
 }

pkg/probod/tracker_mapping.go

@@ -49,7 +49,6 @@ func (impl *Implm) buildTrackerMappingConfig(
 	return cookiebanner.TrackerMappingConfig{
 		LLMClient:         llmClient,
 		Model:             agentCfg.ModelName,
-		SearchEndpoint:    impl.cfg.SearchEndpoint,
 		FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint,
 		FirecrawlAPIKey:   impl.cfg.Firecrawl.APIKey,
 	}, nil

pkg/probodconfig/config.go

@@ -67,7 +67,6 @@ type (
 		Agents            AgentsConfig            `json:"llm"`
 		EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"`
 		ChromeDPAddr      string                  `json:"chrome-dp-addr"`
-		SearchEndpoint    string                  `json:"search-endpoint"`
 		Firecrawl         FirecrawlConfig         `json:"firecrawl"`
 		CustomDomains     CustomDomainsConfig     `json:"custom-domains"`
 		SCIMBridge        SCIMBridgeConfig        `json:"scim-bridge"`

pkg/vetting/assessment.go

@@ -75,7 +75,8 @@ type (
 		Model             string
 		MaxTokens         int
 		ChromeAddr        string
-		SearchEndpoint string
+		FirecrawlEndpoint string
+		FirecrawlAPIKey   string
 		Logger            *log.Logger
 	}
 
@@ -209,7 +210,8 @@ func (a *Assessor) Assess(ctx context.Context, websiteURL string, procedure stri
 		a.cfg.Logger,
 		thirdPartyBrowser,
 		researchBrowser,
-		a.cfg.SearchEndpoint,
+		a.cfg.FirecrawlEndpoint,
+		a.cfg.FirecrawlAPIKey,
 		reporter,
 	)
 	if err != nil {

pkg/vetting/orchestrator.go

@@ -65,7 +65,8 @@ func newOrchestratorAgent(
 	logger *log.Logger,
 	thirdPartyBrowser *browser.Browser,
 	researchBrowser *browser.Browser,
-	searchEndpoint string,
+	firecrawlEndpoint string,
+	firecrawlAPIKey string,
 	reporter agent.ProgressReporter,
 ) (*agent.Agent, error) {
 	readOnlyBrowserTools := browser.NewReadOnlyToolset(thirdPartyBrowser).Tools()
@@ -88,11 +89,13 @@ func newOrchestratorAgent(
 		return opts
 	}
 
+	hasFirecrawl := firecrawlEndpoint != "" && firecrawlAPIKey != ""
+
 	// Subprocessor agent benefits from web search when available so it can
 	// find subprocessor pages hosted on third-party platforms.
 	subprocessorTools := unrestrictedBrowserTools
-	if searchEndpoint != "" {
+	if hasFirecrawl {
-		subprocessorTools = append(subprocessorTools, search.WebSearchTool(searchEndpoint))
+		subprocessorTools = append(subprocessorTools, search.FirecrawlSearchTool(firecrawlEndpoint, firecrawlAPIKey))
 	}
 
 	// Core sub-agents that always run.
@@ -171,12 +174,12 @@ func newOrchestratorAgent(
 		},
 	}
 
-	// Optional sub-agents: only added when a search endpoint is configured.
+	// Optional sub-agents: only added when Firecrawl is configured.
-	if searchEndpoint != "" {
+	if hasFirecrawl {
 		researchBrowserTools := browser.NewInteractiveToolset(researchBrowser).Tools()
 
-		searchTool := search.WebSearchTool(searchEndpoint)
+		searchTool := search.FirecrawlSearchTool(firecrawlEndpoint, firecrawlAPIKey)
-		govDBTool := search.CheckGovernmentDBTool(searchEndpoint)
+		govDBTool := search.CheckGovernmentDBTool(firecrawlEndpoint, firecrawlAPIKey)
 		waybackTool := search.CheckWaybackTool()
 		diffTool := search.DiffDocumentsTool()