diff --git a/contrib/helm/charts/probo/templates/deployment.yaml b/contrib/helm/charts/probo/templates/deployment.yaml index 4cf8b19f6..ee2ea8ca9 100644 --- a/contrib/helm/charts/probo/templates/deployment.yaml +++ b/contrib/helm/charts/probo/templates/deployment.yaml @@ -232,6 +232,40 @@ spec: - name: OPENAI_MAX_TOKENS value: {{ .Values.probo.openai.maxTokens | quote }} {{- end }} + # Firecrawl Integration + {{- if .Values.probo.firecrawl.endpoint }} + - name: FIRECRAWL_ENDPOINT + value: {{ .Values.probo.firecrawl.endpoint | quote }} + {{- end }} + {{- if .Values.probo.firecrawl.apiKey }} + - name: FIRECRAWL_API_KEY + valueFrom: + secretKeyRef: + name: {{ include "probo.fullname" . }} + key: firecrawl-api-key + {{- end }} + # Search Endpoint + {{- if .Values.probo.searchEndpoint }} + - name: SEARCH_ENDPOINT + value: {{ .Values.probo.searchEndpoint | quote }} + {{- end }} + # Tracker Mapping Agent + {{- if .Values.probo.trackerMapping.provider }} + - name: AGENT_TRACKER_MAPPING_PROVIDER + value: {{ .Values.probo.trackerMapping.provider | quote }} + {{- end }} + {{- if .Values.probo.trackerMapping.modelName }} + - name: AGENT_TRACKER_MAPPING_MODEL_NAME + value: {{ .Values.probo.trackerMapping.modelName | quote }} + {{- end }} + {{- if .Values.probo.trackerMapping.temperature }} + - name: AGENT_TRACKER_MAPPING_TEMPERATURE + value: {{ .Values.probo.trackerMapping.temperature | quote }} + {{- end }} + {{- if .Values.probo.trackerMapping.maxTokens }} + - name: AGENT_TRACKER_MAPPING_MAX_TOKENS + value: {{ .Values.probo.trackerMapping.maxTokens | quote }} + {{- end }} # Custom Domains {{- if .Values.probo.customDomains.enabled }} - name: CUSTOM_DOMAINS_RENEWAL_INTERVAL diff --git a/contrib/helm/charts/probo/templates/secret.yaml b/contrib/helm/charts/probo/templates/secret.yaml index 0cbcb10f7..4638af44e 100644 --- a/contrib/helm/charts/probo/templates/secret.yaml +++ b/contrib/helm/charts/probo/templates/secret.yaml @@ -35,6 +35,11 @@ stringData: openai-api-key: {{ .Values.probo.openai.apiKey | quote }} {{- end }} + # Optional Firecrawl API key + {{- if .Values.probo.firecrawl.apiKey }} + firecrawl-api-key: {{ .Values.probo.firecrawl.apiKey | quote }} + {{- end }} + # Optional SAML secrets {{- if and .Values.probo.saml.enabled .Values.probo.saml.certificate }} saml-certificate: {{ .Values.probo.saml.certificate | quote }} diff --git a/contrib/helm/charts/probo/values-production.yaml.example b/contrib/helm/charts/probo/values-production.yaml.example index 6e1b60660..23aecc6a5 100644 --- a/contrib/helm/charts/probo/values-production.yaml.example +++ b/contrib/helm/charts/probo/values-production.yaml.example @@ -159,6 +159,21 @@ probo: modelName: "gpt-4o" maxTokens: 4096 + # Firecrawl web search (optional, used by tracker mapping agent) + # firecrawl: + # endpoint: "https://api.firecrawl.dev/v2" + # apiKey: "CHANGE_ME_FIRECRAWL_API_KEY" + + # SearXNG search endpoint (optional, fallback when Firecrawl is not configured) + # searchEndpoint: "https://search.example.com" + + # Tracker mapping agent (optional, auto-links tracker patterns to vendors) + # trackerMapping: + # provider: "openai" + # modelName: "gpt-4o-mini" + # temperature: "0.1" + # maxTokens: "1024" + # OpenTelemetry tracing (optional) tracing: enabled: true diff --git a/contrib/helm/charts/probo/values.yaml b/contrib/helm/charts/probo/values.yaml index ad5a5982a..e9e073fde 100644 --- a/contrib/helm/charts/probo/values.yaml +++ b/contrib/helm/charts/probo/values.yaml @@ -257,6 +257,21 @@ probo: modelName: "gpt-4o" maxTokens: 4096 + # Firecrawl web search integration (optional, used by tracker mapping agent) + firecrawl: + endpoint: "" + apiKey: "" + + # Search endpoint for SearXNG (optional, fallback when Firecrawl is not configured) + searchEndpoint: "" + + # Tracker mapping agent (optional, requires openai.apiKey or anthropic key) + trackerMapping: + provider: "" + modelName: "" + temperature: "" + maxTokens: "" + # Custom domains configuration (optional) customDomains: enabled: false diff --git a/pkg/bootstrap/builder.go b/pkg/bootstrap/builder.go index 2ded54299..89ca7bdae 100644 --- a/pkg/bootstrap/builder.go +++ b/pkg/bootstrap/builder.go @@ -176,6 +176,11 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { CacheTTL: b.getEnvIntOrDefault("WEBHOOK_CACHE_TTL", 86400), }, }, + SearchEndpoint: b.getEnv("SEARCH_ENDPOINT"), + Firecrawl: probodconfig.FirecrawlConfig{ + Endpoint: b.getEnv("FIRECRAWL_ENDPOINT"), + APIKey: b.getEnv("FIRECRAWL_API_KEY"), + }, Agents: probodconfig.AgentsConfig{ Providers: map[string]probodconfig.LLMProviderConfig{ "openai": { @@ -205,6 +210,12 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { Temperature: b.getEnvFloatPtr("AGENT_EVIDENCE_DESCRIBER_TEMPERATURE"), MaxTokens: b.getEnvIntPtr("AGENT_EVIDENCE_DESCRIBER_MAX_TOKENS"), }, + TrackerMapping: probodconfig.LLMAgentConfig{ + Provider: b.getEnvOrDefault("AGENT_TRACKER_MAPPING_PROVIDER", ""), + ModelName: b.getEnvOrDefault("AGENT_TRACKER_MAPPING_MODEL_NAME", ""), + Temperature: b.getEnvFloatPtr("AGENT_TRACKER_MAPPING_TEMPERATURE"), + MaxTokens: b.getEnvIntPtr("AGENT_TRACKER_MAPPING_MAX_TOKENS"), + }, }, CustomDomains: probodconfig.CustomDomainsConfig{ RenewalInterval: b.getEnvIntOrDefault("CUSTOM_DOMAINS_RENEWAL_INTERVAL", 3600), diff --git a/pkg/bootstrap/builder_test.go b/pkg/bootstrap/builder_test.go index f4f5192ac..df0cdb956 100644 --- a/pkg/bootstrap/builder_test.go +++ b/pkg/bootstrap/builder_test.go @@ -197,6 +197,11 @@ func TestBuilder_Build_Defaults(t *testing.T) { assert.Equal(t, 5, cfg.Probod.Notifications.Webhook.SenderInterval) assert.Equal(t, 86400, cfg.Probod.Notifications.Webhook.CacheTTL) + // Search and Firecrawl — empty by default + assert.Empty(t, cfg.Probod.SearchEndpoint) + assert.Empty(t, cfg.Probod.Firecrawl.Endpoint) + assert.Empty(t, cfg.Probod.Firecrawl.APIKey) + // Agents config — default assert.Equal(t, "openai", cfg.Probod.Agents.Default.Provider) assert.Equal(t, "gpt-4o", cfg.Probod.Agents.Default.ModelName) @@ -211,6 +216,10 @@ func TestBuilder_Build_Defaults(t *testing.T) { assert.Empty(t, cfg.Probod.Agents.EvidenceDescriber.ModelName) assert.Nil(t, cfg.Probod.Agents.EvidenceDescriber.Temperature) assert.Nil(t, cfg.Probod.Agents.EvidenceDescriber.MaxTokens) + assert.Empty(t, cfg.Probod.Agents.TrackerMapping.Provider) + assert.Empty(t, cfg.Probod.Agents.TrackerMapping.ModelName) + assert.Nil(t, cfg.Probod.Agents.TrackerMapping.Temperature) + assert.Nil(t, cfg.Probod.Agents.TrackerMapping.MaxTokens) // Custom domains config assert.Equal(t, 3600, cfg.Probod.CustomDomains.RenewalInterval) @@ -285,6 +294,10 @@ func TestBuilder_Build_CustomValues(t *testing.T) { env["WEBHOOK_SENDER_INTERVAL"] = "10" env["WEBHOOK_CACHE_TTL"] = "3600" env["CONNECTOR_SLACK_SIGNING_SECRET"] = "slack-signing-secret" + // Search and Firecrawl + env["SEARCH_ENDPOINT"] = "https://search.example.com" + env["FIRECRAWL_ENDPOINT"] = "https://api.firecrawl.dev/v2" + env["FIRECRAWL_API_KEY"] = "fc-test-key" // Agents — providers env["OPENAI_API_KEY"] = "sk-test-key" env["ANTHROPIC_API_KEY"] = "sk-ant-test-key" @@ -298,6 +311,11 @@ func TestBuilder_Build_CustomValues(t *testing.T) { env["AGENT_EVIDENCE_DESCRIBER_MODEL_NAME"] = "claude-sonnet-4-20250514" env["AGENT_EVIDENCE_DESCRIBER_TEMPERATURE"] = "0.2" env["AGENT_EVIDENCE_DESCRIBER_MAX_TOKENS"] = "4096" + // Agents — tracker-mapping override + env["AGENT_TRACKER_MAPPING_PROVIDER"] = "openai" + env["AGENT_TRACKER_MAPPING_MODEL_NAME"] = "gpt-4o-mini" + env["AGENT_TRACKER_MAPPING_TEMPERATURE"] = "0.1" + env["AGENT_TRACKER_MAPPING_MAX_TOKENS"] = "1024" // Custom domains env["CUSTOM_DOMAINS_RESOLVER_ADDR"] = "1.1.1.1:53" env["ACME_ACCOUNT_KEY"] = "-----BEGIN EC PRIVATE KEY-----\ntest\n-----END EC PRIVATE KEY-----" @@ -364,6 +382,10 @@ func TestBuilder_Build_CustomValues(t *testing.T) { assert.Equal(t, "slack-signing-secret", cfg.Probod.Notifications.Slack.SigningSecret) assert.Equal(t, 10, cfg.Probod.Notifications.Webhook.SenderInterval) assert.Equal(t, 3600, cfg.Probod.Notifications.Webhook.CacheTTL) + // Search and Firecrawl + assert.Equal(t, "https://search.example.com", cfg.Probod.SearchEndpoint) + assert.Equal(t, "https://api.firecrawl.dev/v2", cfg.Probod.Firecrawl.Endpoint) + assert.Equal(t, "fc-test-key", cfg.Probod.Firecrawl.APIKey) // Agents — providers assert.Equal(t, "openai", cfg.Probod.Agents.Providers["openai"].Type) assert.Equal(t, "sk-test-key", cfg.Probod.Agents.Providers["openai"].APIKey) @@ -382,6 +404,11 @@ func TestBuilder_Build_CustomValues(t *testing.T) { assert.Equal(t, "claude-sonnet-4-20250514", cfg.Probod.Agents.EvidenceDescriber.ModelName) assert.Equal(t, new(0.2), cfg.Probod.Agents.EvidenceDescriber.Temperature) assert.Equal(t, new(4096), cfg.Probod.Agents.EvidenceDescriber.MaxTokens) + // Agents — tracker-mapping overrides + assert.Equal(t, "openai", cfg.Probod.Agents.TrackerMapping.Provider) + assert.Equal(t, "gpt-4o-mini", cfg.Probod.Agents.TrackerMapping.ModelName) + assert.Equal(t, new(0.1), cfg.Probod.Agents.TrackerMapping.Temperature) + assert.Equal(t, new(1024), cfg.Probod.Agents.TrackerMapping.MaxTokens) // Custom domains assert.Equal(t, "1.1.1.1:53", cfg.Probod.CustomDomains.ResolverAddr) assert.Equal(t, "-----BEGIN EC PRIVATE KEY-----\ntest\n-----END EC PRIVATE KEY-----", cfg.Probod.CustomDomains.ACME.AccountKey) diff --git a/pkg/probod/tracker_mapping.go b/pkg/probod/tracker_mapping.go index f9c2a3756..86dc4c3f4 100644 --- a/pkg/probod/tracker_mapping.go +++ b/pkg/probod/tracker_mapping.go @@ -48,7 +48,7 @@ func (impl *Implm) buildTrackerMappingConfig( LLMClient: llmClient, Model: agentCfg.ModelName, SearchEndpoint: impl.cfg.SearchEndpoint, - FirecrawlEndpoint: impl.cfg.FirecrawlEndpoint, - FirecrawlAPIKey: impl.cfg.FirecrawlAPIKey, + FirecrawlEndpoint: impl.cfg.Firecrawl.Endpoint, + FirecrawlAPIKey: impl.cfg.Firecrawl.APIKey, }, nil } diff --git a/pkg/probodconfig/config.go b/pkg/probodconfig/config.go index 5ed9ad9e0..ef9ad030b 100644 --- a/pkg/probodconfig/config.go +++ b/pkg/probodconfig/config.go @@ -47,6 +47,12 @@ type ( TSAURL string `json:"tsa-url"` } + // FirecrawlConfig contains Firecrawl web scraping API configuration. + FirecrawlConfig struct { + Endpoint string `json:"endpoint"` + APIKey string `json:"api-key"` + } + // Config represents the probod application configuration. Config struct { BaseURL string `json:"base-url"` @@ -62,8 +68,7 @@ type ( EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"` ChromeDPAddr string `json:"chrome-dp-addr"` SearchEndpoint string `json:"search-endpoint"` - FirecrawlEndpoint string `json:"firecrawl-endpoint"` - FirecrawlAPIKey string `json:"firecrawl-api-key"` + Firecrawl FirecrawlConfig `json:"firecrawl"` CustomDomains CustomDomainsConfig `json:"custom-domains"` SCIMBridge SCIMBridgeConfig `json:"scim-bridge"` ESign ESignConfig `json:"esign"`