rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Hardcode Firecrawl API endpoint, drop FIRECRAWL_ENDPOINT config

Browse Source 
Firecrawl has a single public API at https://api.firecrawl.dev/v2.
The endpoint was configurable but never varied across environments,
so hardcode it as a package-level const and remove the Endpoint
field from FirecrawlConfig and all downstream wiring (bootstrap,
Helm chart, probod, vetting, cookiebanner).

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-05-19 10:28:05 +04:00
parent ef6aca2c93
commit 76d42ef2db
13 changed files with 35 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
contrib/helm/charts/probo/templates/deployment.yaml
View File

@@ -233,10 +233,6 @@ spec:
value: {{ .Values.probo.openai.maxTokens | quote }} value: {{ .Values.probo.openai.maxTokens | quote }}
{{- end }} {{- end }}
# Firecrawl Integration # Firecrawl Integration
{{- if .Values.probo.firecrawl.endpoint }}
- name: FIRECRAWL_ENDPOINT
value: {{ .Values.probo.firecrawl.endpoint | quote }}
{{- end }}
{{- if .Values.probo.firecrawl.apiKey }} {{- if .Values.probo.firecrawl.apiKey }}
- name: FIRECRAWL_API_KEY - name: FIRECRAWL_API_KEY
valueFrom: valueFrom:

1
contrib/helm/charts/probo/values-production.yaml.example
View File

@@ -161,7 +161,6 @@ probo:
# Firecrawl web search (optional, used by tracker mapping agent) # Firecrawl web search (optional, used by tracker mapping agent)
# firecrawl: # firecrawl:
# endpoint: "https://api.firecrawl.dev/v2"
# apiKey: "CHANGE_ME_FIRECRAWL_API_KEY" # apiKey: "CHANGE_ME_FIRECRAWL_API_KEY"
# Tracker mapping agent (optional, auto-links tracker patterns to vendors) # Tracker mapping agent (optional, auto-links tracker patterns to vendors)

1
contrib/helm/charts/probo/values.yaml
View File

@@ -259,7 +259,6 @@ probo:
# Firecrawl web search integration (optional, used by tracker mapping agent) # Firecrawl web search integration (optional, used by tracker mapping agent)
firecrawl: firecrawl:
endpoint: ""
apiKey: "" apiKey: ""
# Tracker mapping agent (optional, requires openai.apiKey or anthropic key) # Tracker mapping agent (optional, requires openai.apiKey or anthropic key)

13
pkg/agent/tools/search/firecrawl.go
View File

@@ -26,6 +26,8 @@ import (
"go.probo.inc/probo/pkg/agent" "go.probo.inc/probo/pkg/agent"
) )
const firecrawlBaseURL = "https://api.firecrawl.dev/v2"
type ( type (
searchResult struct { searchResult struct {
Title string `json:"title"` Title string `json:"title"`
@@ -58,9 +60,8 @@ type (
) )
// FirecrawlSearchTool creates a tool that searches the web using the Firecrawl // FirecrawlSearchTool creates a tool that searches the web using the Firecrawl
// API. The endpoint should be the base URL of the Firecrawl instance (e.g. // API. The apiKey is used for Bearer authentication.
// "https://api.firecrawl.dev/v2"). The apiKey is used for Bearer authentication. func FirecrawlSearchTool(apiKey string) agent.Tool {
func FirecrawlSearchTool(endpoint, apiKey string) agent.Tool {
client := newHTTPClient() client := newHTTPClient()
return agent.FunctionTool( return agent.FunctionTool(
@@ -75,7 +76,7 @@ func FirecrawlSearchTool(endpoint, apiKey string) agent.Tool {
maxResults = 10 maxResults = 10
} }
results, err := firecrawlSearch(ctx, client, endpoint, apiKey, p.Query, maxResults) results, err := firecrawlSearch(ctx, client, apiKey, p.Query, maxResults)
if err != nil { if err != nil {
return agent.ResultErrorf("search request failed: %s", err), nil return agent.ResultErrorf("search request failed: %s", err), nil
} }
@@ -88,10 +89,10 @@ func FirecrawlSearchTool(endpoint, apiKey string) agent.Tool {
func firecrawlSearch( func firecrawlSearch(
ctx context.Context, ctx context.Context,
client *http.Client, client *http.Client,
endpoint, apiKey, query string, apiKey, query string,
maxResults int, maxResults int,
) ([]searchResult, error) { ) ([]searchResult, error) {
u, err := url.JoinPath(endpoint, "search") u, err := url.JoinPath(firecrawlBaseURL, "search")
if err != nil { if err != nil {
return nil, fmt.Errorf("cannot build search URL: %w", err) return nil, fmt.Errorf("cannot build search URL: %w", err)
} }

4
pkg/agent/tools/search/government_db.go
View File

@@ -43,7 +43,7 @@ type (
} }
) )
func CheckGovernmentDBTool(endpoint, apiKey string) agent.Tool { func CheckGovernmentDBTool(apiKey string) agent.Tool {
client := newHTTPClient() client := newHTTPClient()
return agent.FunctionTool( return agent.FunctionTool(
@@ -87,7 +87,7 @@ func CheckGovernmentDBTool(endpoint, apiKey string) agent.Tool {
} }
for _, s := range searches { for _, s := range searches {
entries, err := firecrawlSearch(ctx, client, endpoint, apiKey, s.query, 3) entries, err := firecrawlSearch(ctx, client, apiKey, s.query, 3)
if err != nil { if err != nil {
continue continue
} }

3
pkg/bootstrap/builder.go
View File

@@ -177,8 +177,7 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
}, },
}, },
Firecrawl: probodconfig.FirecrawlConfig{ Firecrawl: probodconfig.FirecrawlConfig{
Endpoint: b.getEnv("FIRECRAWL_ENDPOINT"), APIKey: b.getEnv("FIRECRAWL_API_KEY"),
APIKey: b.getEnv("FIRECRAWL_API_KEY"),
}, },
Agents: probodconfig.AgentsConfig{ Agents: probodconfig.AgentsConfig{
Providers: map[string]probodconfig.LLMProviderConfig{ Providers: map[string]probodconfig.LLMProviderConfig{

3
pkg/bootstrap/builder_test.go
View File

@@ -198,7 +198,6 @@ func TestBuilder_Build_Defaults(t *testing.T) {
assert.Equal(t, 86400, cfg.Probod.Notifications.Webhook.CacheTTL) assert.Equal(t, 86400, cfg.Probod.Notifications.Webhook.CacheTTL)
// Firecrawl — empty by default // Firecrawl — empty by default
assert.Empty(t, cfg.Probod.Firecrawl.Endpoint)
assert.Empty(t, cfg.Probod.Firecrawl.APIKey) assert.Empty(t, cfg.Probod.Firecrawl.APIKey)
// Agents config — default // Agents config — default
@@ -294,7 +293,6 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
env["WEBHOOK_CACHE_TTL"] = "3600" env["WEBHOOK_CACHE_TTL"] = "3600"
env["CONNECTOR_SLACK_SIGNING_SECRET"] = "slack-signing-secret" env["CONNECTOR_SLACK_SIGNING_SECRET"] = "slack-signing-secret"
// Firecrawl // Firecrawl
env["FIRECRAWL_ENDPOINT"] = "https://api.firecrawl.dev/v2"
env["FIRECRAWL_API_KEY"] = "fc-test-key" env["FIRECRAWL_API_KEY"] = "fc-test-key"
// Agents — providers // Agents — providers
env["OPENAI_API_KEY"] = "sk-test-key" env["OPENAI_API_KEY"] = "sk-test-key"
@@ -381,7 +379,6 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
assert.Equal(t, 10, cfg.Probod.Notifications.Webhook.SenderInterval) assert.Equal(t, 10, cfg.Probod.Notifications.Webhook.SenderInterval)
assert.Equal(t, 3600, cfg.Probod.Notifications.Webhook.CacheTTL) assert.Equal(t, 3600, cfg.Probod.Notifications.Webhook.CacheTTL)
// Firecrawl // Firecrawl
assert.Equal(t, "https://api.firecrawl.dev/v2", cfg.Probod.Firecrawl.Endpoint)
assert.Equal(t, "fc-test-key", cfg.Probod.Firecrawl.APIKey) assert.Equal(t, "fc-test-key", cfg.Probod.Firecrawl.APIKey)
// Agents — providers // Agents — providers
assert.Equal(t, "openai", cfg.Probod.Agents.Providers["openai"].Type) assert.Equal(t, "openai", cfg.Probod.Agents.Providers["openai"].Type)

11
pkg/cookiebanner/tracker_mapping_worker.go
View File

@@ -51,10 +51,9 @@ type trackerMappingHandler struct {
} }
type TrackerMappingConfig struct { type TrackerMappingConfig struct {
LLMClient *llm.Client LLMClient *llm.Client
Model string Model string
FirecrawlEndpoint string FirecrawlAPIKey string
FirecrawlAPIKey string
} }
func NewTrackerMappingWorker( func NewTrackerMappingWorker(
@@ -90,8 +89,8 @@ func buildTrackerMappingAgent(
searchThirdPartiesTool(pgClient), searchThirdPartiesTool(pgClient),
} }
if cfg.FirecrawlEndpoint != "" && cfg.FirecrawlAPIKey != "" { if cfg.FirecrawlAPIKey != "" {
tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlEndpoint, cfg.FirecrawlAPIKey)) tools = append(tools, search.FirecrawlSearchTool(cfg.FirecrawlAPIKey))
} }
outputType, err := agent.NewOutputType[TrackerIdentification]("tracker_identification") outputType, err := agent.NewOutputType[TrackerIdentification]("tracker_identification")

13
pkg/probod/third_party_assessor.go
View File

@@ -48,12 +48,11 @@ func (impl *Implm) buildThirdPartyAssessor(
} }
return vetting.NewAssessor(vetting.Config{ return vetting.NewAssessor(vetting.Config{
Client: llmClient, Client: llmClient,
Model: agentCfg.ModelName, Model: agentCfg.ModelName,
MaxTokens: maxTokens, MaxTokens: maxTokens,
ChromeAddr: impl.cfg.ChromeDPAddr, ChromeAddr: impl.cfg.ChromeDPAddr,
FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint, FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey,
FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey, Logger: l.Named("third-party-assessor"),
Logger: l.Named("third-party-assessor"),
}), nil }), nil
} }

7
pkg/probod/tracker_mapping.go
View File

@@ -47,9 +47,8 @@ func (impl *Implm) buildTrackerMappingConfig(
} }
return cookiebanner.TrackerMappingConfig{ return cookiebanner.TrackerMappingConfig{
LLMClient: llmClient, LLMClient: llmClient,
Model: agentCfg.ModelName, Model: agentCfg.ModelName,
FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint, FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey,
FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey,
}, nil }, nil
} }

3
pkg/probodconfig/config.go
View File

@@ -49,8 +49,7 @@ type (
// FirecrawlConfig contains Firecrawl web scraping API configuration. // FirecrawlConfig contains Firecrawl web scraping API configuration.
FirecrawlConfig struct { FirecrawlConfig struct {
Endpoint string `json:"endpoint"` APIKey string `json:"api-key"`
APIKey string `json:"api-key"`
} }
// Config represents the probod application configuration. // Config represents the probod application configuration.

14
pkg/vetting/assessment.go
View File

@@ -71,13 +71,12 @@ var (
type ( type (
Config struct { Config struct {
Client *llm.Client Client *llm.Client
Model string Model string
MaxTokens int MaxTokens int
ChromeAddr string ChromeAddr string
FirecrawlEndpoint string FirecrawlAPIKey string
FirecrawlAPIKey string Logger *log.Logger
Logger *log.Logger
} }
Assessor struct { Assessor struct {
@@ -210,7 +209,6 @@ func (a *Assessor) Assess(ctx context.Context, websiteURL string, procedure stri
a.cfg.Logger, a.cfg.Logger,
thirdPartyBrowser, thirdPartyBrowser,
researchBrowser, researchBrowser,
a.cfg.FirecrawlEndpoint,
a.cfg.FirecrawlAPIKey, a.cfg.FirecrawlAPIKey,
reporter, reporter,
) )

9
pkg/vetting/orchestrator.go
View File

@@ -65,7 +65,6 @@ func newOrchestratorAgent(
logger *log.Logger, logger *log.Logger,
thirdPartyBrowser *browser.Browser, thirdPartyBrowser *browser.Browser,
researchBrowser *browser.Browser, researchBrowser *browser.Browser,
firecrawlEndpoint string,
firecrawlAPIKey string, firecrawlAPIKey string,
reporter agent.ProgressReporter, reporter agent.ProgressReporter,
) (*agent.Agent, error) { ) (*agent.Agent, error) {
@@ -89,13 +88,13 @@ func newOrchestratorAgent(
return opts return opts
} }
hasFirecrawl := firecrawlEndpoint != "" && firecrawlAPIKey != "" hasFirecrawl := firecrawlAPIKey != ""
// Subprocessor agent benefits from web search when available so it can // Subprocessor agent benefits from web search when available so it can
// find subprocessor pages hosted on third-party platforms. // find subprocessor pages hosted on third-party platforms.
subprocessorTools := unrestrictedBrowserTools subprocessorTools := unrestrictedBrowserTools
if hasFirecrawl { if hasFirecrawl {
subprocessorTools = append(subprocessorTools, search.FirecrawlSearchTool(firecrawlEndpoint, firecrawlAPIKey)) subprocessorTools = append(subprocessorTools, search.FirecrawlSearchTool(firecrawlAPIKey))
} }
// Core sub-agents that always run. // Core sub-agents that always run.
@@ -178,8 +177,8 @@ func newOrchestratorAgent(
if hasFirecrawl { if hasFirecrawl {
researchBrowserTools := browser.NewInteractiveToolset(researchBrowser).Tools() researchBrowserTools := browser.NewInteractiveToolset(researchBrowser).Tools()
searchTool := search.FirecrawlSearchTool(firecrawlEndpoint, firecrawlAPIKey) searchTool := search.FirecrawlSearchTool(firecrawlAPIKey)
govDBTool := search.CheckGovernmentDBTool(firecrawlEndpoint, firecrawlAPIKey) govDBTool := search.CheckGovernmentDBTool(firecrawlAPIKey)
waybackTool := search.CheckWaybackTool() waybackTool := search.CheckWaybackTool()
diffTool := search.DiffDocumentsTool() diffTool := search.DiffDocumentsTool()