rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove SearXNG search backend, use Firecrawl exclusively

Browse Source 
SearXNG was a fallback search backend that added complexity without
being used in practice. All search-dependent features (web search,
government DB checks, vetting orchestrator, tracker mapping) now use
Firecrawl exclusively. Removes the SEARCH_ENDPOINT config plumbing
from probodconfig, bootstrap, Helm charts, and all callers.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-05-19 10:01:01 +04:00
parent 091be2653a
commit a67ac462d7
15 changed files with 59 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
pkg/agent/tools/search/firecrawl.go
View File

@@ -22,12 +22,23 @@ import (
"io"
"net/http"
"net/url"
"time"
"go.gearno.de/kit/httpclient"
"go.probo.inc/probo/pkg/agent"
)
type (
searchResult struct {
Title string `json:"title"`
URL string `json:"url"`
Snippet string `json:"snippet"`
}
userAgentTransport struct {
next http.RoundTripper
}
firecrawlParams struct {
Query string `json:"query" jsonschema:"The search query to execute"`
MaxResults int `json:"max_results" jsonschema:"Maximum number of results to return (default 5, max 10)"`
@@ -52,11 +63,19 @@ type (
}
)
func (t *userAgentTransport) RoundTrip(r *http.Request) (*http.Response, error) {
r2 := r.Clone(r.Context())
r2.Header.Set("User-Agent", "Probo-Agent/1.0")
return t.next.RoundTrip(r2)
}
// FirecrawlSearchTool creates a tool that searches the web using the Firecrawl
// API. The endpoint should be the base URL of the Firecrawl instance (e.g.
// "https://api.firecrawl.dev/v2"). The apiKey is used for Bearer authentication.
func FirecrawlSearchTool(endpoint, apiKey string) agent.Tool {
client := httpclient.DefaultPooledClient()
client.Timeout = 15 * time.Second
client.Transport = &userAgentTransport{next: client.Transport}
return agent.FunctionTool(
"web_search",

80
pkg/agent/tools/search/government_db.go
View File

@@ -16,11 +16,7 @@ package search
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"go.gearno.de/kit/httpclient"
"go.probo.inc/probo/pkg/agent"
@@ -48,7 +44,7 @@ type (
}
)
func CheckGovernmentDBTool(searchEndpoint string) agent.Tool {
func CheckGovernmentDBTool(endpoint, apiKey string) agent.Tool {
client := httpclient.DefaultPooledClient()
client.Transport = &userAgentTransport{next: client.Transport}
@@ -93,17 +89,20 @@ func CheckGovernmentDBTool(searchEndpoint string) agent.Tool {
}
for _, s := range searches {
entries, err := searxngSearch(ctx, client, searchEndpoint, s.query, 3)
entries, err := firecrawlSearch(ctx, client, endpoint, apiKey, s.query, 3)
if err != nil {
continue
}
for _, e := range entries {
*s.target = append(*s.target, govDBEntry{
Source: s.source,
Title: e.Title,
URL: e.URL,
Snippet: e.Snippet,
})
*s.target = append(
*s.target,
govDBEntry{
Source: s.source,
Title: e.Title,
URL: e.URL,
Snippet: e.Snippet,
},
)
}
}
@@ -111,60 +110,3 @@ func CheckGovernmentDBTool(searchEndpoint string) agent.Tool {
},
)
}
func searxngSearch(ctx context.Context, client *http.Client, endpoint, query string, maxResults int) ([]searchResult, error) {
u, err := url.JoinPath(endpoint, "search")
if err != nil {
return nil, fmt.Errorf("cannot build search URL: %w", err)
}
parsed, err := url.Parse(u)
if err != nil {
return nil, fmt.Errorf("cannot parse search URL: %w", err)
}
q := parsed.Query()
q.Set("q", query)
q.Set("format", "json")
q.Set("categories", "general")
parsed.RawQuery = q.Encode()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
return nil, fmt.Errorf("cannot create request: %w", err)
}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("cannot execute search request: %w", err)
}
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("cannot read response body: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("search returned status %d", resp.StatusCode)
}
var searxResp searxngResponse
if err := json.Unmarshal(body, &searxResp); err != nil {
return nil, fmt.Errorf("cannot unmarshal response: %w", err)
}
results := make([]searchResult, 0, maxResults)
for i, r := range searxResp.Results {
if i >= maxResults {
break
}
results = append(results, searchResult{
Title: r.Title,
URL: r.URL,
Snippet: r.Content,
})
}
return results, nil
}

71
pkg/agent/tools/search/search.go
View File

@@ -1,71 +0,0 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package search
import (
"go.probo.inc/probo/pkg/agent"
)
// Toolset provides web search tools.
type Toolset struct {
endpoint string
firecrawlEndpoint string
firecrawlAPIKey string
}
// Option configures a Toolset.
type Option func(*Toolset)
// WithFirecrawl configures the Firecrawl search backend. The endpoint
// should be the base URL of the Firecrawl instance (e.g.
// "https://api.firecrawl.dev/v2").
func WithFirecrawl(endpoint, apiKey string) Option {
return func(t *Toolset) {
t.firecrawlEndpoint = endpoint
t.firecrawlAPIKey = apiKey
}
}
// NewToolset creates a search toolset with the given SearXNG endpoint.
func NewToolset(endpoint string, opts ...Option) *Toolset {
t := &Toolset{endpoint: endpoint}
for _, opt := range opts {
opt(t)
}
return t
}
func (t *Toolset) Tools() []agent.Tool {
tools := []agent.Tool{
CheckWaybackTool(),
DiffDocumentsTool(),
}
if t.firecrawlEndpoint != "" && t.firecrawlAPIKey != "" {
tools = append(
tools,
FirecrawlSearchTool(t.firecrawlEndpoint, t.firecrawlAPIKey),
CheckGovernmentDBTool(t.endpoint),
)
} else if t.endpoint != "" {
tools = append(
tools,
WebSearchTool(t.endpoint),
CheckGovernmentDBTool(t.endpoint),
)
}
return tools
}

85
pkg/agent/tools/search/web_search.go
View File

@@ -1,85 +0,0 @@
// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
package search
import (
"context"
"net/http"
"go.gearno.de/kit/httpclient"
"go.probo.inc/probo/pkg/agent"
)
type (
searchParams struct {
Query string `json:"query" jsonschema:"The search query to execute"`
MaxResults int `json:"max_results" jsonschema:"Maximum number of results to return (default 5, max 10)"`
}
searchResult struct {
Title string `json:"title"`
URL string `json:"url"`
Snippet string `json:"snippet"`
}
searxngResponse struct {
Results []searxngResult `json:"results"`
}
searxngResult struct {
Title string `json:"title"`
URL string `json:"url"`
Content string `json:"content"`
}
userAgentTransport struct {
next http.RoundTripper
}
)
func (t *userAgentTransport) RoundTrip(r *http.Request) (*http.Response, error) {
r2 := r.Clone(r.Context())
r2.Header.Set("User-Agent", "Probo-Agent/1.0")
return t.next.RoundTrip(r2)
}
// WebSearchTool creates a tool that searches the web using a SearXNG instance.
// The endpoint should be the base URL of the SearXNG instance (e.g.
// "http://localhost:8888").
func WebSearchTool(endpoint string) agent.Tool {
client := httpclient.DefaultPooledClient()
client.Transport = &userAgentTransport{next: client.Transport}
return agent.FunctionTool(
"web_search",
"Search the web for information about a topic. Returns a list of results with title, URL, and snippet. Use this to find news, reviews, breach reports, regulatory actions, and other external information about a vendor.",
func(ctx context.Context, p searchParams) (agent.ToolResult, error) {
maxResults := p.MaxResults
if maxResults <= 0 {
maxResults = 5
}
if maxResults > 10 {
maxResults = 10
}
results, err := searxngSearch(ctx, client, endpoint, p.Query, maxResults)
if err != nil {
return agent.ResultErrorf("search request failed: %s", err), nil
}
return agent.ResultJSON(results), nil
},
)
}