rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove SearXNG search backend, use Firecrawl exclusively

Browse Source 
SearXNG was a fallback search backend that added complexity without
being used in practice. All search-dependent features (web search,
government DB checks, vetting orchestrator, tracker mapping) now use
Firecrawl exclusively. Removes the SEARCH_ENDPOINT config plumbing
from probodconfig, bootstrap, Helm charts, and all callers.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-05-19 10:01:01 +04:00
parent 091be2653a
commit a67ac462d7
15 changed files with 59 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
contrib/helm/charts/probo/templates/deployment.yaml
View File

@@ -244,11 +244,6 @@ spec:
name: {{ include "probo.fullname" . }}
key: firecrawl-api-key
{{- end }}
# Search Endpoint
{{- if .Values.probo.searchEndpoint }}
- name: SEARCH_ENDPOINT
value: {{ .Values.probo.searchEndpoint | quote }}
{{- end }}
# Tracker Mapping Agent
{{- if .Values.probo.trackerMapping.provider }}
- name: AGENT_TRACKER_MAPPING_PROVIDER

3
contrib/helm/charts/probo/values-production.yaml.example
View File

@@ -164,9 +164,6 @@ probo:
# endpoint: "https://api.firecrawl.dev/v2"
# apiKey: "CHANGE_ME_FIRECRAWL_API_KEY"
# SearXNG search endpoint (optional, fallback when Firecrawl is not configured)
# searchEndpoint: "https://search.example.com"
# Tracker mapping agent (optional, auto-links tracker patterns to vendors)
# trackerMapping:
# provider: "openai"

3
contrib/helm/charts/probo/values.yaml
View File

@@ -262,9 +262,6 @@ probo:
endpoint: ""
apiKey: ""
# Search endpoint for SearXNG (optional, fallback when Firecrawl is not configured)
searchEndpoint: ""
# Tracker mapping agent (optional, requires openai.apiKey or anthropic key)
trackerMapping:
provider: ""

19
pkg/agent/tools/search/firecrawl.go
View File

@@ -22,12 +22,23 @@ import (
"io"
"net/http"
"net/url"
"time"
"go.gearno.de/kit/httpclient"
"go.probo.inc/probo/pkg/agent"
)
type (
searchResult struct {
Title string `json:"title"`
URL string `json:"url"`
Snippet string `json:"snippet"`
}
userAgentTransport struct {
next http.RoundTripper
}
firecrawlParams struct {
Query string `json:"query" jsonschema:"The search query to execute"`
MaxResults int `json:"max_results" jsonschema:"Maximum number of results to return (default 5, max 10)"`
@@ -52,11 +63,19 @@ type (
}
)
func (t *userAgentTransport) RoundTrip(r *http.Request) (*http.Response, error) {
r2 := r.Clone(r.Context())
r2.Header.Set("User-Agent", "Probo-Agent/1.0")
return t.next.RoundTrip(r2)
}
// FirecrawlSearchTool creates a tool that searches the web using the Firecrawl
// API. The endpoint should be the base URL of the Firecrawl instance (e.g.
// "https://api.firecrawl.dev/v2"). The apiKey is used for Bearer authentication.
func FirecrawlSearchTool(endpoint, apiKey string) agent.Tool {
client := httpclient.DefaultPooledClient()
client.Timeout = 15 * time.Second
client.Transport = &userAgentTransport{next: client.Transport}
return agent.FunctionTool(
"web_search",

80
pkg/agent/tools/search/government_db.go
View File

@@ -16,11 +16,7 @@ package search
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"go.gearno.de/kit/httpclient"
"go.probo.inc/probo/pkg/agent"
@@ -48,7 +44,7 @@ type (
}
)
func CheckGovernmentDBTool(searchEndpoint string) agent.Tool {
func CheckGovernmentDBTool(endpoint, apiKey string) agent.Tool {
client := httpclient.DefaultPooledClient()
client.Transport = &userAgentTransport{next: client.Transport}
@@ -93,17 +89,20 @@ func CheckGovernmentDBTool(searchEndpoint string) agent.Tool {
}
for _, s := range searches {
entries, err := searxngSearch(ctx, client, searchEndpoint, s.query, 3)
entries, err := firecrawlSearch(ctx, client, endpoint, apiKey, s.query, 3)
if err != nil {
continue
}
for _, e := range entries {
*s.target = append(*s.target, govDBEntry{
Source: s.source,
Title: e.Title,
URL: e.URL,
Snippet: e.Snippet,
})
*s.target = append(
*s.target,
govDBEntry{
Source: s.source,
Title: e.Title,
URL: e.URL,
Snippet: e.Snippet,
},
)
}
}
@@ -111,60 +110,3 @@ func CheckGovernmentDBTool(searchEndpoint string) agent.Tool {
},
)
}
func searxngSearch(ctx context.Context, client *http.Client, endpoint, query string, maxResults int) ([]searchResult, error) {
u, err := url.JoinPath(endpoint, "search")
if err != nil {
return nil, fmt.Errorf("cannot build search URL: %w", err)
}
parsed, err := url.Parse(u)
if err != nil {
return nil, fmt.Errorf("cannot parse search URL: %w", err)
}
q := parsed.Query()
q.Set("q", query)
q.Set("format", "json")
q.Set("categories", "general")
parsed.RawQuery = q.Encode()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
return nil, fmt.Errorf("cannot create request: %w", err)
}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("cannot execute search request: %w", err)
}
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("cannot read response body: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("search returned status %d", resp.StatusCode)
}
var searxResp searxngResponse
if err := json.Unmarshal(body, &searxResp); err != nil {
return nil, fmt.Errorf("cannot unmarshal response: %w", err)
}
results := make([]searchResult, 0, maxResults)
for i, r := range searxResp.Results {
if i >= maxResults {
break
}
results = append(results, searchResult{
Title: r.Title,
URL: r.URL,
Snippet: r.Content,
})
}
return results, nil
}

71
pkg/agent/tools/search/search.go
View File

@@ -1,71 +0,0 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package search
import (
"go.probo.inc/probo/pkg/agent"
)
// Toolset provides web search tools.
type Toolset struct {
endpoint string
firecrawlEndpoint string
firecrawlAPIKey string
}
// Option configures a Toolset.
type Option func(*Toolset)
// WithFirecrawl configures the Firecrawl search backend. The endpoint
// should be the base URL of the Firecrawl instance (e.g.
// "https://api.firecrawl.dev/v2").
func WithFirecrawl(endpoint, apiKey string) Option {
return func(t *Toolset) {
t.firecrawlEndpoint = endpoint
t.firecrawlAPIKey = apiKey
}
}
// NewToolset creates a search toolset with the given SearXNG endpoint.
func NewToolset(endpoint string, opts ...Option) *Toolset {
t := &Toolset{endpoint: endpoint}
for _, opt := range opts {
opt(t)
}
return t
}
func (t *Toolset) Tools() []agent.Tool {
tools := []agent.Tool{
CheckWaybackTool(),
DiffDocumentsTool(),
}
if t.firecrawlEndpoint != "" && t.firecrawlAPIKey != "" {
tools = append(
tools,
FirecrawlSearchTool(t.firecrawlEndpoint, t.firecrawlAPIKey),
CheckGovernmentDBTool(t.endpoint),
)
} else if t.endpoint != "" {
tools = append(
tools,
WebSearchTool(t.endpoint),
CheckGovernmentDBTool(t.endpoint),
)
}
return tools
}

85
pkg/agent/tools/search/web_search.go
View File

@@ -1,85 +0,0 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package search
import (
"context"
"net/http"
"go.gearno.de/kit/httpclient"
"go.probo.inc/probo/pkg/agent"
)
type (
searchParams struct {
Query string `json:"query" jsonschema:"The search query to execute"`
MaxResults int `json:"max_results" jsonschema:"Maximum number of results to return (default 5, max 10)"`
}
searchResult struct {
Title string `json:"title"`
URL string `json:"url"`
Snippet string `json:"snippet"`
}
searxngResponse struct {
Results []searxngResult `json:"results"`
}
searxngResult struct {
Title string `json:"title"`
URL string `json:"url"`
Content string `json:"content"`
}
userAgentTransport struct {
next http.RoundTripper
}
)
func (t *userAgentTransport) RoundTrip(r *http.Request) (*http.Response, error) {
r2 := r.Clone(r.Context())
r2.Header.Set("User-Agent", "Probo-Agent/1.0")
return t.next.RoundTrip(r2)
}
// WebSearchTool creates a tool that searches the web using a SearXNG instance.
// The endpoint should be the base URL of the SearXNG instance (e.g.
// "http://localhost:8888").
func WebSearchTool(endpoint string) agent.Tool {
client := httpclient.DefaultPooledClient()
client.Transport = &userAgentTransport{next: client.Transport}
return agent.FunctionTool(
"web_search",
"Search the web for information about a topic. Returns a list of results with title, URL, and snippet. Use this to find news, reviews, breach reports, regulatory actions, and other external information about a vendor.",
func(ctx context.Context, p searchParams) (agent.ToolResult, error) {
maxResults := p.MaxResults
if maxResults <= 0 {
maxResults = 5
}
if maxResults > 10 {
maxResults = 10
}
results, err := searxngSearch(ctx, client, endpoint, p.Query, maxResults)
if err != nil {
return agent.ResultErrorf("search request failed: %s", err), nil
}
return agent.ResultJSON(results), nil
},
)
}

1
pkg/bootstrap/builder.go
View File

@@ -176,7 +176,6 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
CacheTTL: b.getEnvIntOrDefault("WEBHOOK_CACHE_TTL", 86400),
},
},
SearchEndpoint: b.getEnv("SEARCH_ENDPOINT"),
Firecrawl: probodconfig.FirecrawlConfig{
Endpoint: b.getEnv("FIRECRAWL_ENDPOINT"),
APIKey: b.getEnv("FIRECRAWL_API_KEY"),

9
pkg/bootstrap/builder_test.go
View File

@@ -197,8 +197,7 @@ func TestBuilder_Build_Defaults(t *testing.T) {
assert.Equal(t, 5, cfg.Probod.Notifications.Webhook.SenderInterval)
assert.Equal(t, 86400, cfg.Probod.Notifications.Webhook.CacheTTL)
// Search and Firecrawl — empty by default
assert.Empty(t, cfg.Probod.SearchEndpoint)
// Firecrawl — empty by default
assert.Empty(t, cfg.Probod.Firecrawl.Endpoint)
assert.Empty(t, cfg.Probod.Firecrawl.APIKey)
@@ -294,8 +293,7 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
env["WEBHOOK_SENDER_INTERVAL"] = "10"
env["WEBHOOK_CACHE_TTL"] = "3600"
env["CONNECTOR_SLACK_SIGNING_SECRET"] = "slack-signing-secret"
// Search and Firecrawl
env["SEARCH_ENDPOINT"] = "https://search.example.com"
// Firecrawl
env["FIRECRAWL_ENDPOINT"] = "https://api.firecrawl.dev/v2"
env["FIRECRAWL_API_KEY"] = "fc-test-key"
// Agents — providers
@@ -382,8 +380,7 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
assert.Equal(t, "slack-signing-secret", cfg.Probod.Notifications.Slack.SigningSecret)
assert.Equal(t, 10, cfg.Probod.Notifications.Webhook.SenderInterval)
assert.Equal(t, 3600, cfg.Probod.Notifications.Webhook.CacheTTL)
// Search and Firecrawl
assert.Equal(t, "https://search.example.com", cfg.Probod.SearchEndpoint)
// Firecrawl
assert.Equal(t, "https://api.firecrawl.dev/v2", cfg.Probod.Firecrawl.Endpoint)
assert.Equal(t, "fc-test-key", cfg.Probod.Firecrawl.APIKey)
// Agents — providers

3
pkg/cookiebanner/tracker_mapping_worker.go
View File

@@ -53,7 +53,6 @@ type trackerMappingHandler struct {
type TrackerMappingConfig struct {
LLMClient *llm.Client
Model string
SearchEndpoint string
FirecrawlEndpoint string
FirecrawlAPIKey string
}
@@ -93,8 +92,6 @@ func buildTrackerMappingAgent(
if cfg.FirecrawlEndpoint != "" && cfg.FirecrawlAPIKey != "" {
tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlEndpoint, cfg.FirecrawlAPIKey))
} else if cfg.SearchEndpoint != "" {
tools = append(tools, search.WebSearchTool(cfg.SearchEndpoint))
}
outputType, err := agent.NewOutputType[TrackerIdentification]("tracker_identification")

13
pkg/probod/third_party_assessor.go
View File

@@ -48,11 +48,12 @@ func (impl *Implm) buildThirdPartyAssessor(
}
return vetting.NewAssessor(vetting.Config{
Client: llmClient,
Model: agentCfg.ModelName,
MaxTokens: maxTokens,
ChromeAddr: impl.cfg.ChromeDPAddr,
SearchEndpoint: impl.cfg.SearchEndpoint,
Logger: l.Named("third-party-assessor"),
Client: llmClient,
Model: agentCfg.ModelName,
MaxTokens: maxTokens,
ChromeAddr: impl.cfg.ChromeDPAddr,
FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint,
FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey,
Logger: l.Named("third-party-assessor"),
}), nil
}

1
pkg/probod/tracker_mapping.go
View File

@@ -49,7 +49,6 @@ func (impl *Implm) buildTrackerMappingConfig(
return cookiebanner.TrackerMappingConfig{
LLMClient: llmClient,
Model: agentCfg.ModelName,
SearchEndpoint: impl.cfg.SearchEndpoint,
FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint,
FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey,
}, nil

1
pkg/probodconfig/config.go
View File

@@ -67,7 +67,6 @@ type (
Agents AgentsConfig `json:"llm"`
EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"`
ChromeDPAddr string `json:"chrome-dp-addr"`
SearchEndpoint string `json:"search-endpoint"`
Firecrawl FirecrawlConfig `json:"firecrawl"`
CustomDomains CustomDomainsConfig `json:"custom-domains"`
SCIMBridge SCIMBridgeConfig `json:"scim-bridge"`

16
pkg/vetting/assessment.go
View File

@@ -71,12 +71,13 @@ var (
type (
Config struct {
Client *llm.Client
Model string
MaxTokens int
ChromeAddr string
SearchEndpoint string
Logger *log.Logger
Client *llm.Client
Model string
MaxTokens int
ChromeAddr string
FirecrawlEndpoint string
FirecrawlAPIKey string
Logger *log.Logger
}
Assessor struct {
@@ -209,7 +210,8 @@ func (a *Assessor) Assess(ctx context.Context, websiteURL string, procedure stri
a.cfg.Logger,
thirdPartyBrowser,
researchBrowser,
a.cfg.SearchEndpoint,
a.cfg.FirecrawlEndpoint,
a.cfg.FirecrawlAPIKey,
reporter,
)
if err != nil {

17
pkg/vetting/orchestrator.go
View File

@@ -65,7 +65,8 @@ func newOrchestratorAgent(
logger *log.Logger,
thirdPartyBrowser *browser.Browser,
researchBrowser *browser.Browser,
searchEndpoint string,
firecrawlEndpoint string,
firecrawlAPIKey string,
reporter agent.ProgressReporter,
) (*agent.Agent, error) {
readOnlyBrowserTools := browser.NewReadOnlyToolset(thirdPartyBrowser).Tools()
@@ -88,11 +89,13 @@ func newOrchestratorAgent(
return opts
}
hasFirecrawl := firecrawlEndpoint != "" && firecrawlAPIKey != ""
// Subprocessor agent benefits from web search when available so it can
// find subprocessor pages hosted on third-party platforms.
subprocessorTools := unrestrictedBrowserTools
if searchEndpoint != "" {
subprocessorTools = append(subprocessorTools, search.WebSearchTool(searchEndpoint))
if hasFirecrawl {
subprocessorTools = append(subprocessorTools, search.FirecrawlSearchTool(firecrawlEndpoint, firecrawlAPIKey))
}
// Core sub-agents that always run.
@@ -171,12 +174,12 @@ func newOrchestratorAgent(
},
}
// Optional sub-agents: only added when a search endpoint is configured.
if searchEndpoint != "" {
// Optional sub-agents: only added when Firecrawl is configured.
if hasFirecrawl {
researchBrowserTools := browser.NewInteractiveToolset(researchBrowser).Tools()
searchTool := search.WebSearchTool(searchEndpoint)
govDBTool := search.CheckGovernmentDBTool(searchEndpoint)
searchTool := search.FirecrawlSearchTool(firecrawlEndpoint, firecrawlAPIKey)
govDBTool := search.CheckGovernmentDBTool(firecrawlEndpoint, firecrawlAPIKey)
waybackTool := search.CheckWaybackTool()
diffTool := search.DiffDocumentsTool()