rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Propagate firecrawl and tracker-mapping config to bootstrap and Helm

Browse Source 
Group firecrawl-endpoint and firecrawl-api-key under a nested firecrawl
config key. Add env var mappings (FIRECRAWL_ENDPOINT, FIRECRAWL_API_KEY,
SEARCH_ENDPOINT, AGENT_TRACKER_MAPPING_*) to the bootstrap builder with
test coverage. Wire the new values through the Helm chart (values,
deployment, secret, production example).

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-05-18 15:31:03 +04:00
parent ecb726e724
commit e87b426801
8 changed files with 116 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
contrib/helm/charts/probo/templates/deployment.yaml
View File

@@ -232,6 +232,40 @@ spec:
- name: OPENAI_MAX_TOKENS
value: {{ .Values.probo.openai.maxTokens | quote }}
{{- end }}
# Firecrawl Integration
{{- if .Values.probo.firecrawl.endpoint }}
- name: FIRECRAWL_ENDPOINT
value: {{ .Values.probo.firecrawl.endpoint | quote }}
{{- end }}
{{- if .Values.probo.firecrawl.apiKey }}
- name: FIRECRAWL_API_KEY
valueFrom:
secretKeyRef:
name: {{ include "probo.fullname" . }}
key: firecrawl-api-key
{{- end }}
# Search Endpoint
{{- if .Values.probo.searchEndpoint }}
- name: SEARCH_ENDPOINT
value: {{ .Values.probo.searchEndpoint | quote }}
{{- end }}
# Tracker Mapping Agent
{{- if .Values.probo.trackerMapping.provider }}
- name: AGENT_TRACKER_MAPPING_PROVIDER
value: {{ .Values.probo.trackerMapping.provider | quote }}
{{- end }}
{{- if .Values.probo.trackerMapping.modelName }}
- name: AGENT_TRACKER_MAPPING_MODEL_NAME
value: {{ .Values.probo.trackerMapping.modelName | quote }}
{{- end }}
{{- if .Values.probo.trackerMapping.temperature }}
- name: AGENT_TRACKER_MAPPING_TEMPERATURE
value: {{ .Values.probo.trackerMapping.temperature | quote }}
{{- end }}
{{- if .Values.probo.trackerMapping.maxTokens }}
- name: AGENT_TRACKER_MAPPING_MAX_TOKENS
value: {{ .Values.probo.trackerMapping.maxTokens | quote }}
{{- end }}
# Custom Domains
{{- if .Values.probo.customDomains.enabled }}
- name: CUSTOM_DOMAINS_RENEWAL_INTERVAL

5
contrib/helm/charts/probo/templates/secret.yaml
View File

@@ -35,6 +35,11 @@ stringData:
openai-api-key: {{ .Values.probo.openai.apiKey | quote }}
{{- end }}
# Optional Firecrawl API key
{{- if .Values.probo.firecrawl.apiKey }}
firecrawl-api-key: {{ .Values.probo.firecrawl.apiKey | quote }}
{{- end }}
# Optional SAML secrets
{{- if and .Values.probo.saml.enabled .Values.probo.saml.certificate }}
saml-certificate: {{ .Values.probo.saml.certificate | quote }}

15
contrib/helm/charts/probo/values-production.yaml.example
View File

@@ -159,6 +159,21 @@ probo:
modelName: "gpt-4o"
maxTokens: 4096
# Firecrawl web search (optional, used by tracker mapping agent)
# firecrawl:
# endpoint: "https://api.firecrawl.dev/v2"
# apiKey: "CHANGE_ME_FIRECRAWL_API_KEY"
# SearXNG search endpoint (optional, fallback when Firecrawl is not configured)
# searchEndpoint: "https://search.example.com"
# Tracker mapping agent (optional, auto-links tracker patterns to vendors)
# trackerMapping:
# provider: "openai"
# modelName: "gpt-4o-mini"
# temperature: "0.1"
# maxTokens: "1024"
# OpenTelemetry tracing (optional)
tracing:
enabled: true

15
contrib/helm/charts/probo/values.yaml
View File

@@ -257,6 +257,21 @@ probo:
modelName: "gpt-4o"
maxTokens: 4096
# Firecrawl web search integration (optional, used by tracker mapping agent)
firecrawl:
endpoint: ""
apiKey: ""
# Search endpoint for SearXNG (optional, fallback when Firecrawl is not configured)
searchEndpoint: ""
# Tracker mapping agent (optional, requires openai.apiKey or anthropic key)
trackerMapping:
provider: ""
modelName: ""
temperature: ""
maxTokens: ""
# Custom domains configuration (optional)
customDomains:
enabled: false

11
pkg/bootstrap/builder.go
View File

@@ -176,6 +176,11 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
CacheTTL: b.getEnvIntOrDefault("WEBHOOK_CACHE_TTL", 86400),
},
},
SearchEndpoint: b.getEnv("SEARCH_ENDPOINT"),
Firecrawl: probodconfig.FirecrawlConfig{
Endpoint: b.getEnv("FIRECRAWL_ENDPOINT"),
APIKey: b.getEnv("FIRECRAWL_API_KEY"),
},
Agents: probodconfig.AgentsConfig{
Providers: map[string]probodconfig.LLMProviderConfig{
"openai": {
@@ -205,6 +210,12 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
Temperature: b.getEnvFloatPtr("AGENT_EVIDENCE_DESCRIBER_TEMPERATURE"),
MaxTokens: b.getEnvIntPtr("AGENT_EVIDENCE_DESCRIBER_MAX_TOKENS"),
},
TrackerMapping: probodconfig.LLMAgentConfig{
Provider: b.getEnvOrDefault("AGENT_TRACKER_MAPPING_PROVIDER", ""),
ModelName: b.getEnvOrDefault("AGENT_TRACKER_MAPPING_MODEL_NAME", ""),
Temperature: b.getEnvFloatPtr("AGENT_TRACKER_MAPPING_TEMPERATURE"),
MaxTokens: b.getEnvIntPtr("AGENT_TRACKER_MAPPING_MAX_TOKENS"),
},
},
CustomDomains: probodconfig.CustomDomainsConfig{
RenewalInterval: b.getEnvIntOrDefault("CUSTOM_DOMAINS_RENEWAL_INTERVAL", 3600),

27
pkg/bootstrap/builder_test.go
View File

@@ -197,6 +197,11 @@ func TestBuilder_Build_Defaults(t *testing.T) {
assert.Equal(t, 5, cfg.Probod.Notifications.Webhook.SenderInterval)
assert.Equal(t, 86400, cfg.Probod.Notifications.Webhook.CacheTTL)
// Search and Firecrawl — empty by default
assert.Empty(t, cfg.Probod.SearchEndpoint)
assert.Empty(t, cfg.Probod.Firecrawl.Endpoint)
assert.Empty(t, cfg.Probod.Firecrawl.APIKey)
// Agents config — default
assert.Equal(t, "openai", cfg.Probod.Agents.Default.Provider)
assert.Equal(t, "gpt-4o", cfg.Probod.Agents.Default.ModelName)
@@ -211,6 +216,10 @@ func TestBuilder_Build_Defaults(t *testing.T) {
assert.Empty(t, cfg.Probod.Agents.EvidenceDescriber.ModelName)
assert.Nil(t, cfg.Probod.Agents.EvidenceDescriber.Temperature)
assert.Nil(t, cfg.Probod.Agents.EvidenceDescriber.MaxTokens)
assert.Empty(t, cfg.Probod.Agents.TrackerMapping.Provider)
assert.Empty(t, cfg.Probod.Agents.TrackerMapping.ModelName)
assert.Nil(t, cfg.Probod.Agents.TrackerMapping.Temperature)
assert.Nil(t, cfg.Probod.Agents.TrackerMapping.MaxTokens)
// Custom domains config
assert.Equal(t, 3600, cfg.Probod.CustomDomains.RenewalInterval)
@@ -285,6 +294,10 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
env["WEBHOOK_SENDER_INTERVAL"] = "10"
env["WEBHOOK_CACHE_TTL"] = "3600"
env["CONNECTOR_SLACK_SIGNING_SECRET"] = "slack-signing-secret"
// Search and Firecrawl
env["SEARCH_ENDPOINT"] = "https://search.example.com"
env["FIRECRAWL_ENDPOINT"] = "https://api.firecrawl.dev/v2"
env["FIRECRAWL_API_KEY"] = "fc-test-key"
// Agents — providers
env["OPENAI_API_KEY"] = "sk-test-key"
env["ANTHROPIC_API_KEY"] = "sk-ant-test-key"
@@ -298,6 +311,11 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
env["AGENT_EVIDENCE_DESCRIBER_MODEL_NAME"] = "claude-sonnet-4-20250514"
env["AGENT_EVIDENCE_DESCRIBER_TEMPERATURE"] = "0.2"
env["AGENT_EVIDENCE_DESCRIBER_MAX_TOKENS"] = "4096"
// Agents — tracker-mapping override
env["AGENT_TRACKER_MAPPING_PROVIDER"] = "openai"
env["AGENT_TRACKER_MAPPING_MODEL_NAME"] = "gpt-4o-mini"
env["AGENT_TRACKER_MAPPING_TEMPERATURE"] = "0.1"
env["AGENT_TRACKER_MAPPING_MAX_TOKENS"] = "1024"
// Custom domains
env["CUSTOM_DOMAINS_RESOLVER_ADDR"] = "1.1.1.1:53"
env["ACME_ACCOUNT_KEY"] = "-----BEGIN EC PRIVATE KEY-----\ntest\n-----END EC PRIVATE KEY-----"
@@ -364,6 +382,10 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
assert.Equal(t, "slack-signing-secret", cfg.Probod.Notifications.Slack.SigningSecret)
assert.Equal(t, 10, cfg.Probod.Notifications.Webhook.SenderInterval)
assert.Equal(t, 3600, cfg.Probod.Notifications.Webhook.CacheTTL)
// Search and Firecrawl
assert.Equal(t, "https://search.example.com", cfg.Probod.SearchEndpoint)
assert.Equal(t, "https://api.firecrawl.dev/v2", cfg.Probod.Firecrawl.Endpoint)
assert.Equal(t, "fc-test-key", cfg.Probod.Firecrawl.APIKey)
// Agents — providers
assert.Equal(t, "openai", cfg.Probod.Agents.Providers["openai"].Type)
assert.Equal(t, "sk-test-key", cfg.Probod.Agents.Providers["openai"].APIKey)
@@ -382,6 +404,11 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
assert.Equal(t, "claude-sonnet-4-20250514", cfg.Probod.Agents.EvidenceDescriber.ModelName)
assert.Equal(t, new(0.2), cfg.Probod.Agents.EvidenceDescriber.Temperature)
assert.Equal(t, new(4096), cfg.Probod.Agents.EvidenceDescriber.MaxTokens)
// Agents — tracker-mapping overrides
assert.Equal(t, "openai", cfg.Probod.Agents.TrackerMapping.Provider)
assert.Equal(t, "gpt-4o-mini", cfg.Probod.Agents.TrackerMapping.ModelName)
assert.Equal(t, new(0.1), cfg.Probod.Agents.TrackerMapping.Temperature)
assert.Equal(t, new(1024), cfg.Probod.Agents.TrackerMapping.MaxTokens)
// Custom domains
assert.Equal(t, "1.1.1.1:53", cfg.Probod.CustomDomains.ResolverAddr)
assert.Equal(t, "-----BEGIN EC PRIVATE KEY-----\ntest\n-----END EC PRIVATE KEY-----", cfg.Probod.CustomDomains.ACME.AccountKey)

4
pkg/probod/tracker_mapping.go
View File

@@ -48,7 +48,7 @@ func (impl *Implm) buildTrackerMappingConfig(
LLMClient: llmClient,
Model: agentCfg.ModelName,
SearchEndpoint: impl.cfg.SearchEndpoint,
FirecrawlEndpoint: impl.cfg.FirecrawlEndpoint,
FirecrawlAPIKey: impl.cfg.FirecrawlAPIKey,
FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint,
FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey,
}, nil
}

9
pkg/probodconfig/config.go
View File

@@ -47,6 +47,12 @@ type (
TSAURL string `json:"tsa-url"`
}
// FirecrawlConfig contains Firecrawl web scraping API configuration.
FirecrawlConfig struct {
Endpoint string `json:"endpoint"`
APIKey string `json:"api-key"`
}
// Config represents the probod application configuration.
Config struct {
BaseURL string `json:"base-url"`
@@ -62,8 +68,7 @@ type (
EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"`
ChromeDPAddr string `json:"chrome-dp-addr"`
SearchEndpoint string `json:"search-endpoint"`
FirecrawlEndpoint string `json:"firecrawl-endpoint"`
FirecrawlAPIKey string `json:"firecrawl-api-key"`
Firecrawl FirecrawlConfig `json:"firecrawl"`
CustomDomains CustomDomainsConfig `json:"custom-domains"`
SCIMBridge SCIMBridgeConfig `json:"scim-bridge"`
ESign ESignConfig `json:"esign"`