Propagate firecrawl and tracker-mapping config to bootstrap and Helm

Group firecrawl-endpoint and firecrawl-api-key under a nested firecrawl
config key. Add env var mappings (FIRECRAWL_ENDPOINT, FIRECRAWL_API_KEY,
SEARCH_ENDPOINT, AGENT_TRACKER_MAPPING_*) to the bootstrap builder with
test coverage. Wire the new values through the Helm chart (values,
deployment, secret, production example).

Signed-off-by: Émile Ré <emile@probo.com>

contrib/helm/charts/probo/templates/deployment.yaml

@@ -232,6 +232,40 @@ spec:
             - name: OPENAI_MAX_TOKENS
               value: {{ .Values.probo.openai.maxTokens | quote }}
             {{- end }}
+            # Firecrawl Integration
+            {{- if .Values.probo.firecrawl.endpoint }}
+            - name: FIRECRAWL_ENDPOINT
+              value: {{ .Values.probo.firecrawl.endpoint | quote }}
+            {{- end }}
+            {{- if .Values.probo.firecrawl.apiKey }}
+            - name: FIRECRAWL_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "probo.fullname" . }}
+                  key: firecrawl-api-key
+            {{- end }}
+            # Search Endpoint
+            {{- if .Values.probo.searchEndpoint }}
+            - name: SEARCH_ENDPOINT
+              value: {{ .Values.probo.searchEndpoint | quote }}
+            {{- end }}
+            # Tracker Mapping Agent
+            {{- if .Values.probo.trackerMapping.provider }}
+            - name: AGENT_TRACKER_MAPPING_PROVIDER
+              value: {{ .Values.probo.trackerMapping.provider | quote }}
+            {{- end }}
+            {{- if .Values.probo.trackerMapping.modelName }}
+            - name: AGENT_TRACKER_MAPPING_MODEL_NAME
+              value: {{ .Values.probo.trackerMapping.modelName | quote }}
+            {{- end }}
+            {{- if .Values.probo.trackerMapping.temperature }}
+            - name: AGENT_TRACKER_MAPPING_TEMPERATURE
+              value: {{ .Values.probo.trackerMapping.temperature | quote }}
+            {{- end }}
+            {{- if .Values.probo.trackerMapping.maxTokens }}
+            - name: AGENT_TRACKER_MAPPING_MAX_TOKENS
+              value: {{ .Values.probo.trackerMapping.maxTokens | quote }}
+            {{- end }}
             # Custom Domains
             {{- if .Values.probo.customDomains.enabled }}
             - name: CUSTOM_DOMAINS_RENEWAL_INTERVAL

contrib/helm/charts/probo/templates/secret.yaml

@@ -35,6 +35,11 @@ stringData:
   openai-api-key: {{ .Values.probo.openai.apiKey | quote }}
   {{- end }}
 
+  # Optional Firecrawl API key
+  {{- if .Values.probo.firecrawl.apiKey }}
+  firecrawl-api-key: {{ .Values.probo.firecrawl.apiKey | quote }}
+  {{- end }}
+
   # Optional SAML secrets
   {{- if and .Values.probo.saml.enabled .Values.probo.saml.certificate }}
   saml-certificate: {{ .Values.probo.saml.certificate | quote }}

contrib/helm/charts/probo/values-production.yaml.example

@@ -159,6 +159,21 @@ probo:
     modelName: "gpt-4o"
     maxTokens: 4096
 
+  # Firecrawl web search (optional, used by tracker mapping agent)
+  # firecrawl:
+  #   endpoint: "https://api.firecrawl.dev/v2"
+  #   apiKey: "CHANGE_ME_FIRECRAWL_API_KEY"
+
+  # SearXNG search endpoint (optional, fallback when Firecrawl is not configured)
+  # searchEndpoint: "https://search.example.com"
+
+  # Tracker mapping agent (optional, auto-links tracker patterns to vendors)
+  # trackerMapping:
+  #   provider: "openai"
+  #   modelName: "gpt-4o-mini"
+  #   temperature: "0.1"
+  #   maxTokens: "1024"
+
   # OpenTelemetry tracing (optional)
   tracing:
     enabled: true

contrib/helm/charts/probo/values.yaml

@@ -257,6 +257,21 @@ probo:
     modelName: "gpt-4o"
     maxTokens: 4096
 
+  # Firecrawl web search integration (optional, used by tracker mapping agent)
+  firecrawl:
+    endpoint: ""
+    apiKey: ""
+
+  # Search endpoint for SearXNG (optional, fallback when Firecrawl is not configured)
+  searchEndpoint: ""
+
+  # Tracker mapping agent (optional, requires openai.apiKey or anthropic key)
+  trackerMapping:
+    provider: ""
+    modelName: ""
+    temperature: ""
+    maxTokens: ""
+
   # Custom domains configuration (optional)
   customDomains:
     enabled: false

pkg/bootstrap/builder.go

@@ -176,6 +176,11 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
 					CacheTTL:       b.getEnvIntOrDefault("WEBHOOK_CACHE_TTL", 86400),
 				},
 			},
+			SearchEndpoint: b.getEnv("SEARCH_ENDPOINT"),
+			Firecrawl: probodconfig.FirecrawlConfig{
+				Endpoint: b.getEnv("FIRECRAWL_ENDPOINT"),
+				APIKey:   b.getEnv("FIRECRAWL_API_KEY"),
+			},
 			Agents: probodconfig.AgentsConfig{
 				Providers: map[string]probodconfig.LLMProviderConfig{
 					"openai": {
@@ -205,6 +210,12 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
 					Temperature: b.getEnvFloatPtr("AGENT_EVIDENCE_DESCRIBER_TEMPERATURE"),
 					MaxTokens:   b.getEnvIntPtr("AGENT_EVIDENCE_DESCRIBER_MAX_TOKENS"),
 				},
+				TrackerMapping: probodconfig.LLMAgentConfig{
+					Provider:    b.getEnvOrDefault("AGENT_TRACKER_MAPPING_PROVIDER", ""),
+					ModelName:   b.getEnvOrDefault("AGENT_TRACKER_MAPPING_MODEL_NAME", ""),
+					Temperature: b.getEnvFloatPtr("AGENT_TRACKER_MAPPING_TEMPERATURE"),
+					MaxTokens:   b.getEnvIntPtr("AGENT_TRACKER_MAPPING_MAX_TOKENS"),
+				},
 			},
 			CustomDomains: probodconfig.CustomDomainsConfig{
 				RenewalInterval:   b.getEnvIntOrDefault("CUSTOM_DOMAINS_RENEWAL_INTERVAL", 3600),

pkg/bootstrap/builder_test.go

@@ -197,6 +197,11 @@ func TestBuilder_Build_Defaults(t *testing.T) {
 	assert.Equal(t, 5, cfg.Probod.Notifications.Webhook.SenderInterval)
 	assert.Equal(t, 86400, cfg.Probod.Notifications.Webhook.CacheTTL)
 
+	// Search and Firecrawl — empty by default
+	assert.Empty(t, cfg.Probod.SearchEndpoint)
+	assert.Empty(t, cfg.Probod.Firecrawl.Endpoint)
+	assert.Empty(t, cfg.Probod.Firecrawl.APIKey)
+
 	// Agents config — default
 	assert.Equal(t, "openai", cfg.Probod.Agents.Default.Provider)
 	assert.Equal(t, "gpt-4o", cfg.Probod.Agents.Default.ModelName)
@@ -211,6 +216,10 @@ func TestBuilder_Build_Defaults(t *testing.T) {
 	assert.Empty(t, cfg.Probod.Agents.EvidenceDescriber.ModelName)
 	assert.Nil(t, cfg.Probod.Agents.EvidenceDescriber.Temperature)
 	assert.Nil(t, cfg.Probod.Agents.EvidenceDescriber.MaxTokens)
+	assert.Empty(t, cfg.Probod.Agents.TrackerMapping.Provider)
+	assert.Empty(t, cfg.Probod.Agents.TrackerMapping.ModelName)
+	assert.Nil(t, cfg.Probod.Agents.TrackerMapping.Temperature)
+	assert.Nil(t, cfg.Probod.Agents.TrackerMapping.MaxTokens)
 
 	// Custom domains config
 	assert.Equal(t, 3600, cfg.Probod.CustomDomains.RenewalInterval)
@@ -285,6 +294,10 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
 	env["WEBHOOK_SENDER_INTERVAL"] = "10"
 	env["WEBHOOK_CACHE_TTL"] = "3600"
 	env["CONNECTOR_SLACK_SIGNING_SECRET"] = "slack-signing-secret"
+	// Search and Firecrawl
+	env["SEARCH_ENDPOINT"] = "https://search.example.com"
+	env["FIRECRAWL_ENDPOINT"] = "https://api.firecrawl.dev/v2"
+	env["FIRECRAWL_API_KEY"] = "fc-test-key"
 	// Agents — providers
 	env["OPENAI_API_KEY"] = "sk-test-key"
 	env["ANTHROPIC_API_KEY"] = "sk-ant-test-key"
@@ -298,6 +311,11 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
 	env["AGENT_EVIDENCE_DESCRIBER_MODEL_NAME"] = "claude-sonnet-4-20250514"
 	env["AGENT_EVIDENCE_DESCRIBER_TEMPERATURE"] = "0.2"
 	env["AGENT_EVIDENCE_DESCRIBER_MAX_TOKENS"] = "4096"
+	// Agents — tracker-mapping override
+	env["AGENT_TRACKER_MAPPING_PROVIDER"] = "openai"
+	env["AGENT_TRACKER_MAPPING_MODEL_NAME"] = "gpt-4o-mini"
+	env["AGENT_TRACKER_MAPPING_TEMPERATURE"] = "0.1"
+	env["AGENT_TRACKER_MAPPING_MAX_TOKENS"] = "1024"
 	// Custom domains
 	env["CUSTOM_DOMAINS_RESOLVER_ADDR"] = "1.1.1.1:53"
 	env["ACME_ACCOUNT_KEY"] = "-----BEGIN EC PRIVATE KEY-----\ntest\n-----END EC PRIVATE KEY-----"
@@ -364,6 +382,10 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
 	assert.Equal(t, "slack-signing-secret", cfg.Probod.Notifications.Slack.SigningSecret)
 	assert.Equal(t, 10, cfg.Probod.Notifications.Webhook.SenderInterval)
 	assert.Equal(t, 3600, cfg.Probod.Notifications.Webhook.CacheTTL)
+	// Search and Firecrawl
+	assert.Equal(t, "https://search.example.com", cfg.Probod.SearchEndpoint)
+	assert.Equal(t, "https://api.firecrawl.dev/v2", cfg.Probod.Firecrawl.Endpoint)
+	assert.Equal(t, "fc-test-key", cfg.Probod.Firecrawl.APIKey)
 	// Agents — providers
 	assert.Equal(t, "openai", cfg.Probod.Agents.Providers["openai"].Type)
 	assert.Equal(t, "sk-test-key", cfg.Probod.Agents.Providers["openai"].APIKey)
@@ -382,6 +404,11 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
 	assert.Equal(t, "claude-sonnet-4-20250514", cfg.Probod.Agents.EvidenceDescriber.ModelName)
 	assert.Equal(t, new(0.2), cfg.Probod.Agents.EvidenceDescriber.Temperature)
 	assert.Equal(t, new(4096), cfg.Probod.Agents.EvidenceDescriber.MaxTokens)
+	// Agents — tracker-mapping overrides
+	assert.Equal(t, "openai", cfg.Probod.Agents.TrackerMapping.Provider)
+	assert.Equal(t, "gpt-4o-mini", cfg.Probod.Agents.TrackerMapping.ModelName)
+	assert.Equal(t, new(0.1), cfg.Probod.Agents.TrackerMapping.Temperature)
+	assert.Equal(t, new(1024), cfg.Probod.Agents.TrackerMapping.MaxTokens)
 	// Custom domains
 	assert.Equal(t, "1.1.1.1:53", cfg.Probod.CustomDomains.ResolverAddr)
 	assert.Equal(t, "-----BEGIN EC PRIVATE KEY-----\ntest\n-----END EC PRIVATE KEY-----", cfg.Probod.CustomDomains.ACME.AccountKey)

pkg/probod/tracker_mapping.go

@@ -48,7 +48,7 @@ func (impl *Implm) buildTrackerMappingConfig(
 		LLMClient:         llmClient,
 		Model:             agentCfg.ModelName,
 		SearchEndpoint:    impl.cfg.SearchEndpoint,
-		FirecrawlEndpoint: impl.cfg.FirecrawlEndpoint,
+		FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint,
-		FirecrawlAPIKey:   impl.cfg.FirecrawlAPIKey,
+		FirecrawlAPIKey:   impl.cfg.Firecrawl.APIKey,
 	}, nil
 }

pkg/probodconfig/config.go

@@ -47,6 +47,12 @@ type (
 		TSAURL string `json:"tsa-url"`
 	}
 
+	// FirecrawlConfig contains Firecrawl web scraping API configuration.
+	FirecrawlConfig struct {
+		Endpoint string `json:"endpoint"`
+		APIKey   string `json:"api-key"`
+	}
+
 	// Config represents the probod application configuration.
 	Config struct {
 		BaseURL           string                  `json:"base-url"`
@@ -62,8 +68,7 @@ type (
 		EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"`
 		ChromeDPAddr      string                  `json:"chrome-dp-addr"`
 		SearchEndpoint    string                  `json:"search-endpoint"`
-		FirecrawlEndpoint string                  `json:"firecrawl-endpoint"`
+		Firecrawl         FirecrawlConfig         `json:"firecrawl"`
-		FirecrawlAPIKey   string                  `json:"firecrawl-api-key"`
 		CustomDomains     CustomDomainsConfig     `json:"custom-domains"`
 		SCIMBridge        SCIMBridgeConfig        `json:"scim-bridge"`
 		ESign             ESignConfig             `json:"esign"`