From a67ac462d75fc17b80906b90fd3e8058662071d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89mile=20R=C3=A9?= Date: Tue, 19 May 2026 10:01:01 +0400 Subject: [PATCH] Remove SearXNG search backend, use Firecrawl exclusively MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SearXNG was a fallback search backend that added complexity without being used in practice. All search-dependent features (web search, government DB checks, vetting orchestrator, tracker mapping) now use Firecrawl exclusively. Removes the SEARCH_ENDPOINT config plumbing from probodconfig, bootstrap, Helm charts, and all callers. Signed-off-by: Émile Ré --- .../charts/probo/templates/deployment.yaml | 5 -- .../probo/values-production.yaml.example | 3 - contrib/helm/charts/probo/values.yaml | 3 - pkg/agent/tools/search/firecrawl.go | 19 +++++ pkg/agent/tools/search/government_db.go | 80 +++-------------- pkg/agent/tools/search/search.go | 71 ---------------- pkg/agent/tools/search/web_search.go | 85 ------------------- pkg/bootstrap/builder.go | 1 - pkg/bootstrap/builder_test.go | 9 +- pkg/cookiebanner/tracker_mapping_worker.go | 3 - pkg/probod/third_party_assessor.go | 13 +-- pkg/probod/tracker_mapping.go | 1 - pkg/probodconfig/config.go | 1 - pkg/vetting/assessment.go | 16 ++-- pkg/vetting/orchestrator.go | 17 ++-- 15 files changed, 59 insertions(+), 268 deletions(-) delete mode 100644 pkg/agent/tools/search/search.go delete mode 100644 pkg/agent/tools/search/web_search.go diff --git a/contrib/helm/charts/probo/templates/deployment.yaml b/contrib/helm/charts/probo/templates/deployment.yaml index ee2ea8ca9..38128ef85 100644 --- a/contrib/helm/charts/probo/templates/deployment.yaml +++ b/contrib/helm/charts/probo/templates/deployment.yaml @@ -244,11 +244,6 @@ spec: name: {{ include "probo.fullname" . }} key: firecrawl-api-key {{- end }} - # Search Endpoint - {{- if .Values.probo.searchEndpoint }} - - name: SEARCH_ENDPOINT - value: {{ .Values.probo.searchEndpoint | quote }} - {{- end }} # Tracker Mapping Agent {{- if .Values.probo.trackerMapping.provider }} - name: AGENT_TRACKER_MAPPING_PROVIDER diff --git a/contrib/helm/charts/probo/values-production.yaml.example b/contrib/helm/charts/probo/values-production.yaml.example index 23aecc6a5..54eff370c 100644 --- a/contrib/helm/charts/probo/values-production.yaml.example +++ b/contrib/helm/charts/probo/values-production.yaml.example @@ -164,9 +164,6 @@ probo: # endpoint: "https://api.firecrawl.dev/v2" # apiKey: "CHANGE_ME_FIRECRAWL_API_KEY" - # SearXNG search endpoint (optional, fallback when Firecrawl is not configured) - # searchEndpoint: "https://search.example.com" - # Tracker mapping agent (optional, auto-links tracker patterns to vendors) # trackerMapping: # provider: "openai" diff --git a/contrib/helm/charts/probo/values.yaml b/contrib/helm/charts/probo/values.yaml index e9e073fde..bd5445976 100644 --- a/contrib/helm/charts/probo/values.yaml +++ b/contrib/helm/charts/probo/values.yaml @@ -262,9 +262,6 @@ probo: endpoint: "" apiKey: "" - # Search endpoint for SearXNG (optional, fallback when Firecrawl is not configured) - searchEndpoint: "" - # Tracker mapping agent (optional, requires openai.apiKey or anthropic key) trackerMapping: provider: "" diff --git a/pkg/agent/tools/search/firecrawl.go b/pkg/agent/tools/search/firecrawl.go index 86f940747..dcc473ca2 100644 --- a/pkg/agent/tools/search/firecrawl.go +++ b/pkg/agent/tools/search/firecrawl.go @@ -22,12 +22,23 @@ import ( "io" "net/http" "net/url" + "time" "go.gearno.de/kit/httpclient" "go.probo.inc/probo/pkg/agent" ) type ( + searchResult struct { + Title string `json:"title"` + URL string `json:"url"` + Snippet string `json:"snippet"` + } + + userAgentTransport struct { + next http.RoundTripper + } + firecrawlParams struct { Query string `json:"query" jsonschema:"The search query to execute"` MaxResults int `json:"max_results" jsonschema:"Maximum number of results to return (default 5, max 10)"` @@ -52,11 +63,19 @@ type ( } ) +func (t *userAgentTransport) RoundTrip(r *http.Request) (*http.Response, error) { + r2 := r.Clone(r.Context()) + r2.Header.Set("User-Agent", "Probo-Agent/1.0") + return t.next.RoundTrip(r2) +} + // FirecrawlSearchTool creates a tool that searches the web using the Firecrawl // API. The endpoint should be the base URL of the Firecrawl instance (e.g. // "https://api.firecrawl.dev/v2"). The apiKey is used for Bearer authentication. func FirecrawlSearchTool(endpoint, apiKey string) agent.Tool { client := httpclient.DefaultPooledClient() + client.Timeout = 15 * time.Second + client.Transport = &userAgentTransport{next: client.Transport} return agent.FunctionTool( "web_search", diff --git a/pkg/agent/tools/search/government_db.go b/pkg/agent/tools/search/government_db.go index 1b9f13e9d..0bdab10c8 100644 --- a/pkg/agent/tools/search/government_db.go +++ b/pkg/agent/tools/search/government_db.go @@ -16,11 +16,7 @@ package search import ( "context" - "encoding/json" "fmt" - "io" - "net/http" - "net/url" "go.gearno.de/kit/httpclient" "go.probo.inc/probo/pkg/agent" @@ -48,7 +44,7 @@ type ( } ) -func CheckGovernmentDBTool(searchEndpoint string) agent.Tool { +func CheckGovernmentDBTool(endpoint, apiKey string) agent.Tool { client := httpclient.DefaultPooledClient() client.Transport = &userAgentTransport{next: client.Transport} @@ -93,17 +89,20 @@ func CheckGovernmentDBTool(searchEndpoint string) agent.Tool { } for _, s := range searches { - entries, err := searxngSearch(ctx, client, searchEndpoint, s.query, 3) + entries, err := firecrawlSearch(ctx, client, endpoint, apiKey, s.query, 3) if err != nil { continue } for _, e := range entries { - *s.target = append(*s.target, govDBEntry{ - Source: s.source, - Title: e.Title, - URL: e.URL, - Snippet: e.Snippet, - }) + *s.target = append( + *s.target, + govDBEntry{ + Source: s.source, + Title: e.Title, + URL: e.URL, + Snippet: e.Snippet, + }, + ) } } @@ -111,60 +110,3 @@ func CheckGovernmentDBTool(searchEndpoint string) agent.Tool { }, ) } - -func searxngSearch(ctx context.Context, client *http.Client, endpoint, query string, maxResults int) ([]searchResult, error) { - u, err := url.JoinPath(endpoint, "search") - if err != nil { - return nil, fmt.Errorf("cannot build search URL: %w", err) - } - - parsed, err := url.Parse(u) - if err != nil { - return nil, fmt.Errorf("cannot parse search URL: %w", err) - } - - q := parsed.Query() - q.Set("q", query) - q.Set("format", "json") - q.Set("categories", "general") - parsed.RawQuery = q.Encode() - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil) - if err != nil { - return nil, fmt.Errorf("cannot create request: %w", err) - } - - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("cannot execute search request: %w", err) - } - defer func() { _ = resp.Body.Close() }() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("cannot read response body: %w", err) - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("search returned status %d", resp.StatusCode) - } - - var searxResp searxngResponse - if err := json.Unmarshal(body, &searxResp); err != nil { - return nil, fmt.Errorf("cannot unmarshal response: %w", err) - } - - results := make([]searchResult, 0, maxResults) - for i, r := range searxResp.Results { - if i >= maxResults { - break - } - results = append(results, searchResult{ - Title: r.Title, - URL: r.URL, - Snippet: r.Content, - }) - } - - return results, nil -} diff --git a/pkg/agent/tools/search/search.go b/pkg/agent/tools/search/search.go deleted file mode 100644 index fb582cb10..000000000 --- a/pkg/agent/tools/search/search.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2026 Probo Inc . -// -// Permission to use, copy, modify, and/or distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -// PERFORMANCE OF THIS SOFTWARE. - -package search - -import ( - "go.probo.inc/probo/pkg/agent" -) - -// Toolset provides web search tools. -type Toolset struct { - endpoint string - firecrawlEndpoint string - firecrawlAPIKey string -} - -// Option configures a Toolset. -type Option func(*Toolset) - -// WithFirecrawl configures the Firecrawl search backend. The endpoint -// should be the base URL of the Firecrawl instance (e.g. -// "https://api.firecrawl.dev/v2"). -func WithFirecrawl(endpoint, apiKey string) Option { - return func(t *Toolset) { - t.firecrawlEndpoint = endpoint - t.firecrawlAPIKey = apiKey - } -} - -// NewToolset creates a search toolset with the given SearXNG endpoint. -func NewToolset(endpoint string, opts ...Option) *Toolset { - t := &Toolset{endpoint: endpoint} - for _, opt := range opts { - opt(t) - } - return t -} - -func (t *Toolset) Tools() []agent.Tool { - tools := []agent.Tool{ - CheckWaybackTool(), - DiffDocumentsTool(), - } - - if t.firecrawlEndpoint != "" && t.firecrawlAPIKey != "" { - tools = append( - tools, - FirecrawlSearchTool(t.firecrawlEndpoint, t.firecrawlAPIKey), - CheckGovernmentDBTool(t.endpoint), - ) - } else if t.endpoint != "" { - tools = append( - tools, - WebSearchTool(t.endpoint), - CheckGovernmentDBTool(t.endpoint), - ) - } - - return tools -} diff --git a/pkg/agent/tools/search/web_search.go b/pkg/agent/tools/search/web_search.go deleted file mode 100644 index 3c31503f3..000000000 --- a/pkg/agent/tools/search/web_search.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) 2026 Probo Inc . -// -// Permission to use, copy, modify, and/or distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -// PERFORMANCE OF THIS SOFTWARE. - -package search - -import ( - "context" - "net/http" - - "go.gearno.de/kit/httpclient" - "go.probo.inc/probo/pkg/agent" -) - -type ( - searchParams struct { - Query string `json:"query" jsonschema:"The search query to execute"` - MaxResults int `json:"max_results" jsonschema:"Maximum number of results to return (default 5, max 10)"` - } - - searchResult struct { - Title string `json:"title"` - URL string `json:"url"` - Snippet string `json:"snippet"` - } - - searxngResponse struct { - Results []searxngResult `json:"results"` - } - - searxngResult struct { - Title string `json:"title"` - URL string `json:"url"` - Content string `json:"content"` - } - - userAgentTransport struct { - next http.RoundTripper - } -) - -func (t *userAgentTransport) RoundTrip(r *http.Request) (*http.Response, error) { - r2 := r.Clone(r.Context()) - r2.Header.Set("User-Agent", "Probo-Agent/1.0") - return t.next.RoundTrip(r2) -} - -// WebSearchTool creates a tool that searches the web using a SearXNG instance. -// The endpoint should be the base URL of the SearXNG instance (e.g. -// "http://localhost:8888"). -func WebSearchTool(endpoint string) agent.Tool { - client := httpclient.DefaultPooledClient() - client.Transport = &userAgentTransport{next: client.Transport} - - return agent.FunctionTool( - "web_search", - "Search the web for information about a topic. Returns a list of results with title, URL, and snippet. Use this to find news, reviews, breach reports, regulatory actions, and other external information about a vendor.", - func(ctx context.Context, p searchParams) (agent.ToolResult, error) { - maxResults := p.MaxResults - if maxResults <= 0 { - maxResults = 5 - } - if maxResults > 10 { - maxResults = 10 - } - - results, err := searxngSearch(ctx, client, endpoint, p.Query, maxResults) - if err != nil { - return agent.ResultErrorf("search request failed: %s", err), nil - } - - return agent.ResultJSON(results), nil - }, - ) -} diff --git a/pkg/bootstrap/builder.go b/pkg/bootstrap/builder.go index 89ca7bdae..cc9787c69 100644 --- a/pkg/bootstrap/builder.go +++ b/pkg/bootstrap/builder.go @@ -176,7 +176,6 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { CacheTTL: b.getEnvIntOrDefault("WEBHOOK_CACHE_TTL", 86400), }, }, - SearchEndpoint: b.getEnv("SEARCH_ENDPOINT"), Firecrawl: probodconfig.FirecrawlConfig{ Endpoint: b.getEnv("FIRECRAWL_ENDPOINT"), APIKey: b.getEnv("FIRECRAWL_API_KEY"), diff --git a/pkg/bootstrap/builder_test.go b/pkg/bootstrap/builder_test.go index df0cdb956..b93e43664 100644 --- a/pkg/bootstrap/builder_test.go +++ b/pkg/bootstrap/builder_test.go @@ -197,8 +197,7 @@ func TestBuilder_Build_Defaults(t *testing.T) { assert.Equal(t, 5, cfg.Probod.Notifications.Webhook.SenderInterval) assert.Equal(t, 86400, cfg.Probod.Notifications.Webhook.CacheTTL) - // Search and Firecrawl — empty by default - assert.Empty(t, cfg.Probod.SearchEndpoint) + // Firecrawl — empty by default assert.Empty(t, cfg.Probod.Firecrawl.Endpoint) assert.Empty(t, cfg.Probod.Firecrawl.APIKey) @@ -294,8 +293,7 @@ func TestBuilder_Build_CustomValues(t *testing.T) { env["WEBHOOK_SENDER_INTERVAL"] = "10" env["WEBHOOK_CACHE_TTL"] = "3600" env["CONNECTOR_SLACK_SIGNING_SECRET"] = "slack-signing-secret" - // Search and Firecrawl - env["SEARCH_ENDPOINT"] = "https://search.example.com" + // Firecrawl env["FIRECRAWL_ENDPOINT"] = "https://api.firecrawl.dev/v2" env["FIRECRAWL_API_KEY"] = "fc-test-key" // Agents — providers @@ -382,8 +380,7 @@ func TestBuilder_Build_CustomValues(t *testing.T) { assert.Equal(t, "slack-signing-secret", cfg.Probod.Notifications.Slack.SigningSecret) assert.Equal(t, 10, cfg.Probod.Notifications.Webhook.SenderInterval) assert.Equal(t, 3600, cfg.Probod.Notifications.Webhook.CacheTTL) - // Search and Firecrawl - assert.Equal(t, "https://search.example.com", cfg.Probod.SearchEndpoint) + // Firecrawl assert.Equal(t, "https://api.firecrawl.dev/v2", cfg.Probod.Firecrawl.Endpoint) assert.Equal(t, "fc-test-key", cfg.Probod.Firecrawl.APIKey) // Agents — providers diff --git a/pkg/cookiebanner/tracker_mapping_worker.go b/pkg/cookiebanner/tracker_mapping_worker.go index 15db3ea89..b8558a899 100644 --- a/pkg/cookiebanner/tracker_mapping_worker.go +++ b/pkg/cookiebanner/tracker_mapping_worker.go @@ -53,7 +53,6 @@ type trackerMappingHandler struct { type TrackerMappingConfig struct { LLMClient *llm.Client Model string - SearchEndpoint string FirecrawlEndpoint string FirecrawlAPIKey string } @@ -93,8 +92,6 @@ func buildTrackerMappingAgent( if cfg.FirecrawlEndpoint != "" && cfg.FirecrawlAPIKey != "" { tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlEndpoint, cfg.FirecrawlAPIKey)) - } else if cfg.SearchEndpoint != "" { - tools = append(tools, search.WebSearchTool(cfg.SearchEndpoint)) } outputType, err := agent.NewOutputType[TrackerIdentification]("tracker_identification") diff --git a/pkg/probod/third_party_assessor.go b/pkg/probod/third_party_assessor.go index fa814b1d7..0105ca326 100644 --- a/pkg/probod/third_party_assessor.go +++ b/pkg/probod/third_party_assessor.go @@ -48,11 +48,12 @@ func (impl *Implm) buildThirdPartyAssessor( } return vetting.NewAssessor(vetting.Config{ - Client: llmClient, - Model: agentCfg.ModelName, - MaxTokens: maxTokens, - ChromeAddr: impl.cfg.ChromeDPAddr, - SearchEndpoint: impl.cfg.SearchEndpoint, - Logger: l.Named("third-party-assessor"), + Client: llmClient, + Model: agentCfg.ModelName, + MaxTokens: maxTokens, + ChromeAddr: impl.cfg.ChromeDPAddr, + FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint, + FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey, + Logger: l.Named("third-party-assessor"), }), nil } diff --git a/pkg/probod/tracker_mapping.go b/pkg/probod/tracker_mapping.go index fb66dd16a..89a5e4d0a 100644 --- a/pkg/probod/tracker_mapping.go +++ b/pkg/probod/tracker_mapping.go @@ -49,7 +49,6 @@ func (impl *Implm) buildTrackerMappingConfig( return cookiebanner.TrackerMappingConfig{ LLMClient: llmClient, Model: agentCfg.ModelName, - SearchEndpoint: impl.cfg.SearchEndpoint, FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint, FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey, }, nil diff --git a/pkg/probodconfig/config.go b/pkg/probodconfig/config.go index ef9ad030b..50fd1e8f3 100644 --- a/pkg/probodconfig/config.go +++ b/pkg/probodconfig/config.go @@ -67,7 +67,6 @@ type ( Agents AgentsConfig `json:"llm"` EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"` ChromeDPAddr string `json:"chrome-dp-addr"` - SearchEndpoint string `json:"search-endpoint"` Firecrawl FirecrawlConfig `json:"firecrawl"` CustomDomains CustomDomainsConfig `json:"custom-domains"` SCIMBridge SCIMBridgeConfig `json:"scim-bridge"` diff --git a/pkg/vetting/assessment.go b/pkg/vetting/assessment.go index 26408665a..6fc08ff9b 100644 --- a/pkg/vetting/assessment.go +++ b/pkg/vetting/assessment.go @@ -71,12 +71,13 @@ var ( type ( Config struct { - Client *llm.Client - Model string - MaxTokens int - ChromeAddr string - SearchEndpoint string - Logger *log.Logger + Client *llm.Client + Model string + MaxTokens int + ChromeAddr string + FirecrawlEndpoint string + FirecrawlAPIKey string + Logger *log.Logger } Assessor struct { @@ -209,7 +210,8 @@ func (a *Assessor) Assess(ctx context.Context, websiteURL string, procedure stri a.cfg.Logger, thirdPartyBrowser, researchBrowser, - a.cfg.SearchEndpoint, + a.cfg.FirecrawlEndpoint, + a.cfg.FirecrawlAPIKey, reporter, ) if err != nil { diff --git a/pkg/vetting/orchestrator.go b/pkg/vetting/orchestrator.go index 9e059ad66..43fb46eec 100644 --- a/pkg/vetting/orchestrator.go +++ b/pkg/vetting/orchestrator.go @@ -65,7 +65,8 @@ func newOrchestratorAgent( logger *log.Logger, thirdPartyBrowser *browser.Browser, researchBrowser *browser.Browser, - searchEndpoint string, + firecrawlEndpoint string, + firecrawlAPIKey string, reporter agent.ProgressReporter, ) (*agent.Agent, error) { readOnlyBrowserTools := browser.NewReadOnlyToolset(thirdPartyBrowser).Tools() @@ -88,11 +89,13 @@ func newOrchestratorAgent( return opts } + hasFirecrawl := firecrawlEndpoint != "" && firecrawlAPIKey != "" + // Subprocessor agent benefits from web search when available so it can // find subprocessor pages hosted on third-party platforms. subprocessorTools := unrestrictedBrowserTools - if searchEndpoint != "" { - subprocessorTools = append(subprocessorTools, search.WebSearchTool(searchEndpoint)) + if hasFirecrawl { + subprocessorTools = append(subprocessorTools, search.FirecrawlSearchTool(firecrawlEndpoint, firecrawlAPIKey)) } // Core sub-agents that always run. @@ -171,12 +174,12 @@ func newOrchestratorAgent( }, } - // Optional sub-agents: only added when a search endpoint is configured. - if searchEndpoint != "" { + // Optional sub-agents: only added when Firecrawl is configured. + if hasFirecrawl { researchBrowserTools := browser.NewInteractiveToolset(researchBrowser).Tools() - searchTool := search.WebSearchTool(searchEndpoint) - govDBTool := search.CheckGovernmentDBTool(searchEndpoint) + searchTool := search.FirecrawlSearchTool(firecrawlEndpoint, firecrawlAPIKey) + govDBTool := search.CheckGovernmentDBTool(firecrawlEndpoint, firecrawlAPIKey) waybackTool := search.CheckWaybackTool() diffTool := search.DiffDocumentsTool()