diff --git a/.env.example b/.env.example index 6356cc90c..d9cb9aabe 100644 --- a/.env.example +++ b/.env.example @@ -19,8 +19,8 @@ # PROBOD_BASE_URL=http://localhost:8080 # PROBOD_API_ADDR=:8080 # PROBOD_API_CORS_ALLOWED_ORIGINS=http://localhost:8080,http://localhost:5173,http://localhost:5174 -# PROBOD_TRUST_CENTER_HTTP_ADDR=:80 -# PROBOD_TRUST_CENTER_HTTPS_ADDR=:443 +# PROBOD_TRUST_CENTER_HTTP_ADDR=:10080 +# PROBOD_TRUST_CENTER_HTTPS_ADDR=:10443 # ── Observability ───────────────────────────────────────────────────── # PROBOD_METRICS_ADDR=localhost:8081 diff --git a/GNUmakefile b/GNUmakefile index 8120c8fe8..1cc940285 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -201,18 +201,37 @@ $(CFG_DEV_OAUTH2_KEY): cfg/dev.yaml: bin/probod-bootstrap $(CFG_DEV_OAUTH2_KEY) compose/pebble/certs/rootCA.pem $(wildcard $(DEV_ENV)) @$(MKDIR) $(@D) set -a; \ + PROBOD_BASE_URL=http://localhost:8080; \ + PROBOD_API_ADDR=:8080; \ PROBOD_ENCRYPTION_KEY="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; \ + PROBOD_AUTH_COOKIE_NAME=SSID; \ + PROBOD_AUTH_COOKIE_DOMAIN=localhost; \ PROBOD_AUTH_COOKIE_SECRET="this-is-a-secure-secret-for-cookie-signing-at-least-32-bytes"; \ PROBOD_AUTH_PASSWORD_PEPPER="this-is-a-secure-pepper-for-password-hashing-at-least-32-bytes"; \ PROBOD_AUTH_COOKIE_SECURE=false; \ PROBOD_OAUTH2_SERVER_SIGNING_KEY="$$($(CAT) $(CFG_DEV_OAUTH2_KEY))"; \ PROBOD_API_CORS_ALLOWED_ORIGINS="http://localhost:8080,http://localhost:5173,http://localhost:5174"; \ + PROBOD_PG_ADDR=localhost:5432; \ + PROBOD_PG_USERNAME=postgres; \ + PROBOD_PG_PASSWORD=postgres; \ + PROBOD_PG_DATABASE=probod; \ + PROBOD_TRUST_CENTER_HTTP_ADDR=:10080; \ + PROBOD_TRUST_CENTER_HTTPS_ADDR=:10443; \ + PROBOD_AWS_REGION=us-east-1; \ + PROBOD_AWS_BUCKET=probod; \ PROBOD_AWS_ACCESS_KEY_ID=probod; \ PROBOD_AWS_SECRET_ACCESS_KEY=thisisnotasecret; \ PROBOD_AWS_ENDPOINT=http://127.0.0.1:8333; \ + PROBOD_SMTP_ADDR=localhost:1025; \ + PROBOD_MAILER_SENDER_EMAIL=no-reply@notification.getprobo.com; \ + PROBOD_MAILER_SENDER_NAME=Probo; \ PROBOD_OPENAI_API_KEY=thisisnotasecret; \ PROBOD_AGENT_THIRD_PARTY_VETTER_PROVIDER=openai; \ + PROBOD_AGENT_THIRD_PARTY_VETTER_MODEL_NAME=gpt-4o; \ + PROBOD_CHROME_DP_ADDR=localhost:9222; \ PROBOD_ACME_DIRECTORY=https://localhost:14000/dir; \ + PROBOD_ACME_EMAIL=admin@probo.com; \ + PROBOD_ACME_ROOT_CA="$$($(CAT) compose/pebble/certs/rootCA.pem)"; \ if [ -f $(DEV_ENV) ]; then . $(DEV_ENV); fi; \ set +a; \ ./bin/probod-bootstrap -output $@ diff --git a/pkg/bootstrap/builder.go b/pkg/bootstrap/builder.go index 344c3eced..821835920 100644 --- a/pkg/bootstrap/builder.go +++ b/pkg/bootstrap/builder.go @@ -78,7 +78,7 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { Cors: probodconfig.CorsConfig{ AllowedOrigins: b.parseOriginsList(b.resolver.getEnv("PROBOD_API_CORS_ALLOWED_ORIGINS")), }, - ExtraHeaderFields: make(map[string]string), + ExtraHeaderFields: nil, GraphQL: probodconfig.GraphQLConfig{ ParserTokenLimit: b.resolver.getEnvIntOrDefault("PROBOD_API_GRAPHQL_PARSER_TOKEN_LIMIT", 15000), ComplexityLimit: b.resolver.getEnvIntOrDefault("PROBOD_API_GRAPHQL_COMPLEXITY_LIMIT", 2000), @@ -191,83 +191,78 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { ReminderInterval: b.resolver.getEnvIntOrDefault("PROBOD_DOCUMENT_NOTIFICATION_REMINDER_INTERVAL", 86400), }, }, - Agents: probodconfig.AgentsConfig{ - Providers: map[string]probodconfig.LLMProviderConfig{ - "openai": { - Type: "openai", - APIKey: b.resolver.getEnv("PROBOD_OPENAI_API_KEY"), + Agents: func() probodconfig.AgentsConfig { + defaultProvider := b.resolver.getEnvOrDefault("PROBOD_AGENT_DEFAULT_PROVIDER", "openai") + + return probodconfig.AgentsConfig{ + Providers: b.buildLLMProviders(), + Default: probodconfig.LLMAgentConfig{ + Provider: defaultProvider, + ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_DEFAULT_MODEL_NAME", "gpt-4o"), + Temperature: new(b.resolver.getEnvFloatOrDefault("PROBOD_AGENT_DEFAULT_TEMPERATURE", 0.1)), + MaxTokens: new(b.resolver.getEnvIntOrDefault("PROBOD_AGENT_DEFAULT_MAX_TOKENS", 4096)), }, - "anthropic": { - Type: "anthropic", - APIKey: b.resolver.getEnv("PROBOD_ANTHROPIC_API_KEY"), + Probo: probodconfig.LLMAgentConfig{ + Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_PROBO_PROVIDER", ""), + ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_PROBO_MODEL_NAME", ""), + Temperature: b.resolver.getEnvFloatPtr("PROBOD_AGENT_PROBO_TEMPERATURE"), + MaxTokens: b.resolver.getEnvIntPtr("PROBOD_AGENT_PROBO_MAX_TOKENS"), }, - }, - Default: probodconfig.LLMAgentConfig{ - Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_DEFAULT_PROVIDER", "openai"), - ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_DEFAULT_MODEL_NAME", "gpt-4o"), - Temperature: new(b.resolver.getEnvFloatOrDefault("PROBOD_AGENT_DEFAULT_TEMPERATURE", 0.1)), - MaxTokens: new(b.resolver.getEnvIntOrDefault("PROBOD_AGENT_DEFAULT_MAX_TOKENS", 4096)), - }, - Probo: probodconfig.LLMAgentConfig{ - Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_PROBO_PROVIDER", ""), - ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_PROBO_MODEL_NAME", ""), - Temperature: b.resolver.getEnvFloatPtr("PROBOD_AGENT_PROBO_TEMPERATURE"), - MaxTokens: b.resolver.getEnvIntPtr("PROBOD_AGENT_PROBO_MAX_TOKENS"), - }, - EvidenceDescriber: probodconfig.LLMAgentConfig{ - Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_EVIDENCE_DESCRIBER_PROVIDER", ""), - ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_EVIDENCE_DESCRIBER_MODEL_NAME", ""), - Temperature: b.resolver.getEnvFloatPtr("PROBOD_AGENT_EVIDENCE_DESCRIBER_TEMPERATURE"), - MaxTokens: b.resolver.getEnvIntPtr("PROBOD_AGENT_EVIDENCE_DESCRIBER_MAX_TOKENS"), - }, - ThirdPartyVetter: probodconfig.LLMAgentConfig{ - Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_THIRD_PARTY_VETTER_PROVIDER", ""), - ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_THIRD_PARTY_VETTER_MODEL_NAME", ""), - Temperature: b.resolver.getEnvFloatPtr("PROBOD_AGENT_THIRD_PARTY_VETTER_TEMPERATURE"), - MaxTokens: b.resolver.getEnvIntPtr("PROBOD_AGENT_THIRD_PARTY_VETTER_MAX_TOKENS"), - }, - ThirdPartyDisambiguation: probodconfig.LLMAgentConfig{ - Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_THIRD_PARTY_DISAMBIGUATION_PROVIDER", ""), - ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_THIRD_PARTY_DISAMBIGUATION_MODEL_NAME", ""), - // The disambiguation agent emits a single id plus a - // short rationale, but the budget must leave headroom - // for reasoning models whose reasoning tokens count - // against max_tokens; too small a budget truncates the - // JSON. - Temperature: b.resolver.getEnvFloatPtr("PROBOD_AGENT_THIRD_PARTY_DISAMBIGUATION_TEMPERATURE"), - MaxTokens: new(b.resolver.getEnvIntOrDefault("PROBOD_AGENT_THIRD_PARTY_DISAMBIGUATION_MAX_TOKENS", 4096)), - }, - TrackerMapping: probodconfig.LLMAgentConfig{ - Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_TRACKER_MAPPING_PROVIDER", ""), - ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_TRACKER_MAPPING_MODEL_NAME", ""), - // The tracker agents emit tiny structured JSON, but - // the budget must leave headroom for reasoning - // models whose reasoning tokens count against - // max_tokens; too small a budget truncates the JSON. - Temperature: b.resolver.getEnvFloatPtr("PROBOD_AGENT_TRACKER_MAPPING_TEMPERATURE"), - MaxTokens: new(b.resolver.getEnvIntOrDefault("PROBOD_AGENT_TRACKER_MAPPING_MAX_TOKENS", 4096)), - }, - TrackerEnrichment: probodconfig.LLMAgentConfig{ - Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_TRACKER_ENRICHMENT_PROVIDER", ""), - ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_TRACKER_ENRICHMENT_MODEL_NAME", ""), - // See the tracker-mapping note: keep ample headroom so - // reasoning models do not truncate the structured JSON. - Temperature: b.resolver.getEnvFloatPtr("PROBOD_AGENT_TRACKER_ENRICHMENT_TEMPERATURE"), - MaxTokens: new(b.resolver.getEnvIntOrDefault("PROBOD_AGENT_TRACKER_ENRICHMENT_MAX_TOKENS", 4096)), - }, - CommonThirdPartyEnrichment: probodconfig.LLMAgentConfig{ - Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_COMMON_THIRD_PARTY_ENRICHMENT_PROVIDER", ""), - ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MODEL_NAME", ""), - // Agent B browses pages and emits a moderate structured - // output; the budget must leave headroom for reasoning - // models whose reasoning tokens count against max_tokens. - Temperature: b.resolver.getEnvFloatPtr("PROBOD_AGENT_COMMON_THIRD_PARTY_ENRICHMENT_TEMPERATURE"), - MaxTokens: new(b.resolver.getEnvIntOrDefault("PROBOD_AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MAX_TOKENS", 8192)), - }, - Tools: probodconfig.AgentToolsConfig{ - FirecrawlAPIKey: b.resolver.getEnv("PROBOD_FIRECRAWL_API_KEY"), - }, - }, + EvidenceDescriber: probodconfig.LLMAgentConfig{ + Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_EVIDENCE_DESCRIBER_PROVIDER", ""), + ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_EVIDENCE_DESCRIBER_MODEL_NAME", ""), + Temperature: b.resolver.getEnvFloatPtr("PROBOD_AGENT_EVIDENCE_DESCRIBER_TEMPERATURE"), + MaxTokens: b.resolver.getEnvIntPtr("PROBOD_AGENT_EVIDENCE_DESCRIBER_MAX_TOKENS"), + }, + ThirdPartyVetter: probodconfig.LLMAgentConfig{ + Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_THIRD_PARTY_VETTER_PROVIDER", ""), + ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_THIRD_PARTY_VETTER_MODEL_NAME", ""), + Temperature: b.resolver.getEnvFloatPtr("PROBOD_AGENT_THIRD_PARTY_VETTER_TEMPERATURE"), + MaxTokens: b.resolver.getEnvIntPtr("PROBOD_AGENT_THIRD_PARTY_VETTER_MAX_TOKENS"), + }, + ThirdPartyDisambiguation: probodconfig.LLMAgentConfig{ + Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_THIRD_PARTY_DISAMBIGUATION_PROVIDER", ""), + ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_THIRD_PARTY_DISAMBIGUATION_MODEL_NAME", ""), + // The disambiguation agent emits a single id plus a + // short rationale, but the budget must leave headroom + // for reasoning models whose reasoning tokens count + // against max_tokens; too small a budget truncates the + // JSON. + Temperature: b.resolver.getEnvFloatPtr("PROBOD_AGENT_THIRD_PARTY_DISAMBIGUATION_TEMPERATURE"), + MaxTokens: new(b.resolver.getEnvIntOrDefault("PROBOD_AGENT_THIRD_PARTY_DISAMBIGUATION_MAX_TOKENS", 4096)), + }, + TrackerMapping: probodconfig.LLMAgentConfig{ + Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_TRACKER_MAPPING_PROVIDER", ""), + ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_TRACKER_MAPPING_MODEL_NAME", ""), + // The tracker agents emit tiny structured JSON, but + // the budget must leave headroom for reasoning + // models whose reasoning tokens count against + // max_tokens; too small a budget truncates the JSON. + Temperature: b.resolver.getEnvFloatPtr("PROBOD_AGENT_TRACKER_MAPPING_TEMPERATURE"), + MaxTokens: new(b.resolver.getEnvIntOrDefault("PROBOD_AGENT_TRACKER_MAPPING_MAX_TOKENS", 4096)), + }, + TrackerEnrichment: probodconfig.LLMAgentConfig{ + Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_TRACKER_ENRICHMENT_PROVIDER", ""), + ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_TRACKER_ENRICHMENT_MODEL_NAME", ""), + // See the tracker-mapping note: keep ample headroom so + // reasoning models do not truncate the structured JSON. + Temperature: b.resolver.getEnvFloatPtr("PROBOD_AGENT_TRACKER_ENRICHMENT_TEMPERATURE"), + MaxTokens: new(b.resolver.getEnvIntOrDefault("PROBOD_AGENT_TRACKER_ENRICHMENT_MAX_TOKENS", 4096)), + }, + CommonThirdPartyEnrichment: probodconfig.LLMAgentConfig{ + Provider: b.resolver.getEnvOrDefault("PROBOD_AGENT_COMMON_THIRD_PARTY_ENRICHMENT_PROVIDER", ""), + ModelName: b.resolver.getEnvOrDefault("PROBOD_AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MODEL_NAME", ""), + // Agent B browses pages and emits a moderate structured + // output; the budget must leave headroom for reasoning + // models whose reasoning tokens count against max_tokens. + Temperature: b.resolver.getEnvFloatPtr("PROBOD_AGENT_COMMON_THIRD_PARTY_ENRICHMENT_TEMPERATURE"), + MaxTokens: new(b.resolver.getEnvIntOrDefault("PROBOD_AGENT_COMMON_THIRD_PARTY_ENRICHMENT_MAX_TOKENS", 8192)), + }, + Tools: probodconfig.AgentToolsConfig{ + FirecrawlAPIKey: b.resolver.getEnv("PROBOD_FIRECRAWL_API_KEY"), + }, + } + }(), CustomDomains: probodconfig.CustomDomainsConfig{ RenewalInterval: b.resolver.getEnvIntOrDefault("PROBOD_CUSTOM_DOMAINS_RENEWAL_INTERVAL", 3600), ProvisionInterval: b.resolver.getEnvIntOrDefault("PROBOD_CUSTOM_DOMAINS_PROVISION_INTERVAL", 30), @@ -647,6 +642,30 @@ func (b *Builder) getPgCACertBundle() string { return b.resolver.getEnv("PROBOD_PG_CA_BUNDLE") } +func (b *Builder) buildLLMProviders() map[string]probodconfig.LLMProviderConfig { + providers := map[string]probodconfig.LLMProviderConfig{} + + if apiKey := b.resolver.getEnv("PROBOD_OPENAI_API_KEY"); apiKey != "" { + providers["openai"] = probodconfig.LLMProviderConfig{ + Type: "openai", + APIKey: apiKey, + } + } + + if apiKey := b.resolver.getEnv("PROBOD_ANTHROPIC_API_KEY"); apiKey != "" { + providers["anthropic"] = probodconfig.LLMProviderConfig{ + Type: "anthropic", + APIKey: apiKey, + } + } + + if len(providers) == 0 { + return nil + } + + return providers +} + func (b *Builder) parseOriginsList(s string) []string { s = strings.TrimSpace(s) if s == "" { diff --git a/pkg/bootstrap/builder_test.go b/pkg/bootstrap/builder_test.go index 38bda637b..0c093caa3 100644 --- a/pkg/bootstrap/builder_test.go +++ b/pkg/bootstrap/builder_test.go @@ -214,6 +214,7 @@ func TestBuilder_Build_Defaults(t *testing.T) { assert.Equal(t, "gpt-4o", cfg.Probod.Agents.Default.ModelName) assert.Equal(t, new(0.1), cfg.Probod.Agents.Default.Temperature) assert.Equal(t, new(4096), cfg.Probod.Agents.Default.MaxTokens) + assert.Nil(t, cfg.Probod.Agents.Providers) // Agents config — per-agent overrides are empty (inherit from default) assert.Empty(t, cfg.Probod.Agents.Probo.Provider) assert.Empty(t, cfg.Probod.Agents.Probo.ModelName) diff --git a/pkg/bootstrap/write.go b/pkg/bootstrap/write.go index f92436652..0b3535c28 100644 --- a/pkg/bootstrap/write.go +++ b/pkg/bootstrap/write.go @@ -37,28 +37,21 @@ func WriteConfig(cfg *probodconfig.FullConfig, path string, format Format) error return fmt.Errorf("create directory %s: %w", dir, err) } - data, err := yaml.Marshal(cfg) - if err != nil { - return fmt.Errorf("marshal config: %w", err) - } - - var tree any - if err := yaml.Unmarshal(data, &tree); err != nil { - return fmt.Errorf("unmarshal config: %w", err) - } - - pruned := pruneEmptyStrings(tree) + var ( + data []byte + err error + ) switch format { case FormatJSON: - data, err = json.MarshalIndent(pruned, "", " ") + data, err = json.MarshalIndent(cfg, "", " ") if err != nil { - return fmt.Errorf("marshal pruned config as json: %w", err) + return fmt.Errorf("marshal config as json: %w", err) } case FormatYAML: - data, err = yaml.Marshal(pruned) + data, err = yaml.Marshal(cfg) if err != nil { - return fmt.Errorf("marshal pruned config as yaml: %w", err) + return fmt.Errorf("marshal config as yaml: %w", err) } default: return fmt.Errorf("unsupported config format: %q", format) @@ -70,28 +63,3 @@ func WriteConfig(cfg *probodconfig.FullConfig, path string, format Format) error return nil } - -func pruneEmptyStrings(value any) any { - switch v := value.(type) { - case map[string]any: - for key, child := range v { - if s, ok := child.(string); ok && s == "" { - delete(v, key) - - continue - } - - v[key] = pruneEmptyStrings(child) - } - - return v - case []any: - for i, child := range v { - v[i] = pruneEmptyStrings(child) - } - - return v - default: - return v - } -} diff --git a/pkg/bootstrap/write_test.go b/pkg/bootstrap/write_test.go index 3b9476963..f2b570736 100644 --- a/pkg/bootstrap/write_test.go +++ b/pkg/bootstrap/write_test.go @@ -86,7 +86,7 @@ func TestWriteConfig_FilePermissions(t *testing.T) { assert.Equal(t, os.FileMode(0600), info.Mode().Perm()) } -func TestWriteConfig_DropsEmptyStrings(t *testing.T) { +func TestWriteConfig_OmitsOptionalFields(t *testing.T) { tmpDir := t.TempDir() configPath := filepath.Join(tmpDir, "probod.yml") @@ -96,9 +96,8 @@ func TestWriteConfig_DropsEmptyStrings(t *testing.T) { Tracing: probodconfig.TracingConfig{Addr: ""}, }, Probod: probodconfig.Config{ - BaseURL: "http://localhost:8080", - EncryptionKey: "", - ChromeDPAddr: "", + BaseURL: "http://localhost:8080", + ChromeDPAddr: "", Pg: probodconfig.PgConfig{ Addr: "localhost:5432", Username: "postgres", @@ -124,8 +123,8 @@ func TestWriteConfig_DropsEmptyStrings(t *testing.T) { require.True(t, ok) assert.Equal(t, "http://localhost:8080", probod["base-url"]) - assert.NotContains(t, probod, "encryption-key") assert.NotContains(t, probod, "chrome-dp-addr") + assert.NotContains(t, probod, "esign") pg, ok := probod["pg"].(map[string]any) require.True(t, ok) @@ -146,12 +145,168 @@ func TestWriteConfig_DropsEmptyStrings(t *testing.T) { tracing, ok := unit["tracing"].(map[string]any) require.True(t, ok) assert.NotContains(t, tracing, "addr") +} - loaded := probodconfig.FullConfig{} - err = yaml.Unmarshal(data, &loaded) +func TestWriteConfig_OmitsEmptyOptionalBlocks(t *testing.T) { + tmpDir := t.TempDir() + configPath := filepath.Join(tmpDir, "probod.yml") + + cfg := &probodconfig.FullConfig{ + Probod: probodconfig.Config{ + BaseURL: "http://localhost:8080", + EncryptionKey: "test-key", + Api: probodconfig.APIConfig{ + Addr: ":8080", + }, + Auth: probodconfig.AuthConfig{ + Cookie: probodconfig.CookieConfig{ + Name: "SSID", + Secret: "secret", + }, + Password: probodconfig.PasswordConfig{ + Pepper: "pepper", + }, + }, + TrustCenter: probodconfig.TrustCenterConfig{ + HTTPAddr: ":80", + }, + CustomDomains: probodconfig.CustomDomainsConfig{ + RenewalInterval: 3600, + }, + Agents: probodconfig.AgentsConfig{ + Default: probodconfig.LLMAgentConfig{ + Provider: "openai", + ModelName: "gpt-4o", + }, + ThirdPartyDisambiguation: probodconfig.LLMAgentConfig{ + MaxTokens: new(4096), + }, + }, + }, + } + + err := WriteConfig(cfg, configPath, FormatYAML) require.NoError(t, err) - assert.Equal(t, cfg.Probod.BaseURL, loaded.Probod.BaseURL) - assert.Empty(t, loaded.Probod.EncryptionKey) + + data, err := os.ReadFile(configPath) + require.NoError(t, err) + + var tree map[string]any + + err = yaml.Unmarshal(data, &tree) + require.NoError(t, err) + + probod, ok := tree["probod"].(map[string]any) + require.True(t, ok) + + assert.NotContains(t, probod, "esign") + + api, ok := probod["api"].(map[string]any) + require.True(t, ok) + assert.NotContains(t, api, "cors") + assert.NotContains(t, api, "proxy-protocol") + assert.NotContains(t, api, "extra-header-fields") + + auth, ok := probod["auth"].(map[string]any) + require.True(t, ok) + assert.NotContains(t, auth, "google") + assert.NotContains(t, auth, "microsoft") + + customDomains, ok := probod["custom-domains"].(map[string]any) + require.True(t, ok) + assert.NotContains(t, customDomains, "acme") + + trustCenter, ok := probod["trust-center"].(map[string]any) + require.True(t, ok) + assert.NotContains(t, trustCenter, "proxy-protocol") + + llm, ok := probod["llm"].(map[string]any) + require.True(t, ok) + assert.NotContains(t, llm, "probo") + assert.NotContains(t, llm, "third-party-disambiguation") + assert.NotContains(t, llm, "tools") +} + +func TestWriteConfig_OmitsEmptyProxyProtocolAndCorsSlices(t *testing.T) { + tmpDir := t.TempDir() + configPath := filepath.Join(tmpDir, "probod.yml") + + cfg := &probodconfig.FullConfig{ + Probod: probodconfig.Config{ + BaseURL: "http://localhost:8080", + Api: probodconfig.APIConfig{ + Addr: ":8080", + ProxyProtocol: probodconfig.ProxyProtocolConfig{ + TrustedProxies: []string{}, + }, + Cors: probodconfig.CorsConfig{ + AllowedOrigins: []string{}, + }, + }, + TrustCenter: probodconfig.TrustCenterConfig{ + HTTPAddr: ":10080", + ProxyProtocol: probodconfig.ProxyProtocolConfig{ + TrustedProxies: make([]string, 0), + }, + }, + }, + } + + err := WriteConfig(cfg, configPath, FormatYAML) + require.NoError(t, err) + + data, err := os.ReadFile(configPath) + require.NoError(t, err) + + var tree map[string]any + + err = yaml.Unmarshal(data, &tree) + require.NoError(t, err) + + probod, ok := tree["probod"].(map[string]any) + require.True(t, ok) + + api, ok := probod["api"].(map[string]any) + require.True(t, ok) + assert.NotContains(t, api, "proxy-protocol") + assert.NotContains(t, api, "cors") + + trustCenter, ok := probod["trust-center"].(map[string]any) + require.True(t, ok) + assert.NotContains(t, trustCenter, "proxy-protocol") +} + +func TestWriteConfig_OmitsEmptyExtraHeaderFieldsMap(t *testing.T) { + tmpDir := t.TempDir() + configPath := filepath.Join(tmpDir, "probod.yml") + + cfg := &probodconfig.FullConfig{ + Probod: probodconfig.Config{ + BaseURL: "http://localhost:8080", + Api: probodconfig.APIConfig{ + Addr: ":8080", + ExtraHeaderFields: map[string]string{}, + }, + }, + } + + err := WriteConfig(cfg, configPath, FormatYAML) + require.NoError(t, err) + + data, err := os.ReadFile(configPath) + require.NoError(t, err) + + var tree map[string]any + + err = yaml.Unmarshal(data, &tree) + require.NoError(t, err) + + probod, ok := tree["probod"].(map[string]any) + require.True(t, ok) + + api, ok := probod["api"].(map[string]any) + require.True(t, ok) + assert.NotContains(t, api, "extra-header-fields") } func TestWriteConfig_CompleteConfig(t *testing.T) { @@ -178,7 +333,6 @@ func TestWriteConfig_CompleteConfig(t *testing.T) { Cors: probodconfig.CorsConfig{ AllowedOrigins: []string{"http://localhost:8080"}, }, - ExtraHeaderFields: map[string]string{}, }, Pg: probodconfig.PgConfig{ Addr: "localhost:5432", @@ -256,7 +410,7 @@ func TestWriteConfig_JSON(t *testing.T) { probod, ok := tree["probod"].(map[string]any) require.True(t, ok) assert.Equal(t, "http://localhost:8080", probod["base-url"]) - assert.NotContains(t, probod, "encryption-key") + assert.Equal(t, "", probod["encryption-key"]) var loaded probodconfig.FullConfig diff --git a/pkg/probodconfig/api_config.go b/pkg/probodconfig/api_config.go index eceeb9bd6..4a912a6b0 100644 --- a/pkg/probodconfig/api_config.go +++ b/pkg/probodconfig/api_config.go @@ -15,11 +15,19 @@ package probodconfig type CorsConfig struct { - AllowedOrigins []string `json:"allowed-origins"` + AllowedOrigins []string `json:"allowed-origins,omitzero,omitempty"` +} + +func (c CorsConfig) IsZero() bool { + return len(c.AllowedOrigins) == 0 } type ProxyProtocolConfig struct { - TrustedProxies []string `json:"trusted-proxies"` + TrustedProxies []string `json:"trusted-proxies,omitzero,omitempty"` +} + +func (c ProxyProtocolConfig) IsZero() bool { + return len(c.TrustedProxies) == 0 } type GraphQLConfig struct { @@ -30,9 +38,9 @@ type GraphQLConfig struct { } type APIConfig struct { - Addr string `json:"addr"` - ProxyProtocol ProxyProtocolConfig `json:"proxy-protocol"` - Cors CorsConfig `json:"cors"` - ExtraHeaderFields map[string]string `json:"extra-header-fields"` - GraphQL GraphQLConfig `json:"graphql"` + Addr string `json:"addr,omitempty"` + ProxyProtocol ProxyProtocolConfig `json:"proxy-protocol,omitzero"` + Cors CorsConfig `json:"cors,omitzero"` + ExtraHeaderFields map[string]string `json:"extra-header-fields,omitzero,omitempty"` + GraphQL GraphQLConfig `json:"graphql,omitzero"` } diff --git a/pkg/probodconfig/auth_config.go b/pkg/probodconfig/auth_config.go index a936d578c..a1c04370d 100644 --- a/pkg/probodconfig/auth_config.go +++ b/pkg/probodconfig/auth_config.go @@ -27,8 +27,8 @@ type AuthConfig struct { PasswordResetTokenValidity int `json:"password-reset-token-validity"` MagicLinkTokenValidity int `json:"magic-link-token-validity"` SAML SAMLConfig `json:"saml"` - Google OIDCProviderConfig `json:"google"` - Microsoft OIDCProviderConfig `json:"microsoft"` + Google OIDCProviderConfig `json:"google,omitzero"` + Microsoft OIDCProviderConfig `json:"microsoft,omitzero"` OAuth2Server OAuth2ServerConfig `json:"oauth2-server"` } @@ -38,7 +38,7 @@ type OAuth2ServerConfig struct { RefreshTokenDuration int `json:"refresh-token-duration"` AuthorizationCodeDuration int `json:"authorization-code-duration"` DeviceCodeDuration int `json:"device-code-duration"` - CIMDAllowedClientIDs []string `json:"cimd-allowed-client-ids"` + CIMDAllowedClientIDs []string `json:"cimd-allowed-client-ids,omitempty"` } type OAuth2SigningKeyConfig struct { @@ -48,10 +48,10 @@ type OAuth2SigningKeyConfig struct { } type CookieConfig struct { - Domain string `json:"domain"` + Domain string `json:"domain,omitempty"` Secret string `json:"secret"` Duration int `json:"duration"` - Name string `json:"name"` + Name string `json:"name,omitempty"` Secure bool `json:"secure"` } diff --git a/pkg/probodconfig/aws_config.go b/pkg/probodconfig/aws_config.go index c0138b5dc..950e7d886 100644 --- a/pkg/probodconfig/aws_config.go +++ b/pkg/probodconfig/aws_config.go @@ -15,10 +15,10 @@ package probodconfig type AWSConfig struct { - Region string `json:"region"` - Bucket string `json:"bucket"` - AccessKeyID string `json:"access-key-id"` - SecretAccessKey string `json:"secret-access-key"` - Endpoint string `json:"endpoint"` + Region string `json:"region,omitempty"` + Bucket string `json:"bucket,omitempty"` + AccessKeyID string `json:"access-key-id,omitempty"` + SecretAccessKey string `json:"secret-access-key,omitempty"` + Endpoint string `json:"endpoint,omitempty"` UsePathStyle bool `json:"use-path-style"` } diff --git a/pkg/probodconfig/config.go b/pkg/probodconfig/config.go index 2c15a982e..d0e3182e5 100644 --- a/pkg/probodconfig/config.go +++ b/pkg/probodconfig/config.go @@ -35,7 +35,7 @@ type ( // TracingConfig contains tracing configuration. TracingConfig struct { - Addr string `json:"addr"` + Addr string `json:"addr,omitempty"` MaxBatchSize int `json:"max-batch-size"` BatchTimeout int `json:"batch-timeout"` ExportTimeout int `json:"export-timeout"` @@ -44,12 +44,12 @@ type ( // ESignConfig contains electronic signature configuration. ESignConfig struct { - TSAURL string `json:"tsa-url"` + TSAURL string `json:"tsa-url,omitempty"` } // Config represents the probod application configuration. Config struct { - BaseURL string `json:"base-url"` + BaseURL string `json:"base-url,omitempty"` EncryptionKey string `json:"encryption-key"` Pg PgConfig `json:"pg"` Api APIConfig `json:"api"` @@ -57,7 +57,7 @@ type ( TrustCenter TrustCenterConfig `json:"trust-center"` AWS AWSConfig `json:"aws"` Notifications NotificationsConfig `json:"notifications"` - Connectors []ConnectorConfig `json:"connectors"` + Connectors []ConnectorConfig `json:"connectors,omitempty"` Agents AgentsConfig `json:"llm"` EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"` ThirdPartyVetting ThirdPartyVettingWorkerConfig `json:"third-party-vetting-worker"` @@ -66,17 +66,17 @@ type ( CommonPatternEnrichmentWorker CommonPatternEnrichmentWorkerConfig `json:"common-pattern-enrichment-worker"` CommonThirdPartyEnrichmentWorker CommonThirdPartyEnrichmentWorkerConfig `json:"common-third-party-enrichment-worker"` - ChromeDPAddr string `json:"chrome-dp-addr"` + ChromeDPAddr string `json:"chrome-dp-addr,omitempty"` CustomDomains CustomDomainsConfig `json:"custom-domains"` SCIMBridge SCIMBridgeConfig `json:"scim-bridge"` - ESign ESignConfig `json:"esign"` + ESign ESignConfig `json:"esign,omitzero"` Branding bool `json:"branding"` } // TrustCenterConfig contains trust center server configuration. TrustCenterConfig struct { - HTTPAddr string `json:"http-addr"` - HTTPSAddr string `json:"https-addr"` - ProxyProtocol ProxyProtocolConfig `json:"proxy-protocol"` + HTTPAddr string `json:"http-addr,omitempty"` + HTTPSAddr string `json:"https-addr,omitempty"` + ProxyProtocol ProxyProtocolConfig `json:"proxy-protocol,omitzero"` } ) diff --git a/pkg/probodconfig/custom_domains_config.go b/pkg/probodconfig/custom_domains_config.go index 50046bbd1..32adf6931 100644 --- a/pkg/probodconfig/custom_domains_config.go +++ b/pkg/probodconfig/custom_domains_config.go @@ -17,16 +17,16 @@ package probodconfig type CustomDomainsConfig struct { RenewalInterval int `json:"renewal-interval"` ProvisionInterval int `json:"provision-interval"` - ResolverAddr string `json:"resolver-addr"` + ResolverAddr string `json:"resolver-addr,omitempty"` CnameTarget string `json:"cname-target"` CAAIssuerDomain string `json:"caa-issuer-domain"` - ACME ACMEConfig `json:"acme"` + ACME ACMEConfig `json:"acme,omitzero"` } type ACMEConfig struct { - Directory string `json:"directory"` - Email string `json:"email"` - KeyType string `json:"key-type"` - AccountKey string `json:"account-key"` - RootCA string `json:"root-ca"` + Directory string `json:"directory,omitempty"` + Email string `json:"email,omitempty"` + KeyType string `json:"key-type,omitempty"` + AccountKey string `json:"account-key,omitempty"` + RootCA string `json:"root-ca,omitempty"` } diff --git a/pkg/probodconfig/llm_config.go b/pkg/probodconfig/llm_config.go index f81d43bab..ac121288e 100644 --- a/pkg/probodconfig/llm_config.go +++ b/pkg/probodconfig/llm_config.go @@ -18,17 +18,17 @@ type ( // LLMProviderConfig holds authentication and connection settings for an // LLM provider (e.g. OpenAI, Anthropic). LLMProviderConfig struct { - Type string `json:"type"` // "openai", "anthropic", "bedrock" - APIKey string `json:"api-key"` // for OpenAI and Anthropic + Type string `json:"type"` // "openai", "anthropic", "bedrock" + APIKey string `json:"api-key,omitempty"` // for OpenAI and Anthropic } // LLMAgentConfig holds model parameters for a single agent. Provider // references one of the keys in AgentsConfig.Providers. LLMAgentConfig struct { - Provider string `json:"provider"` // key into AgentsConfig.Providers - ModelName string `json:"model-name"` - Temperature *float64 `json:"temperature"` - MaxTokens *int `json:"max-tokens"` + Provider string `json:"provider,omitempty"` // key into AgentsConfig.Providers + ModelName string `json:"model-name,omitempty"` + Temperature *float64 `json:"temperature,omitempty"` + MaxTokens *int `json:"max-tokens,omitempty"` } // EvidenceDescriberConfig holds worker-side tuning for the evidence @@ -95,26 +95,38 @@ type ( // AgentToolsConfig holds API keys and settings for external tools // that agents can use (web search, scraping, etc.). AgentToolsConfig struct { - FirecrawlAPIKey string `json:"firecrawl-api-key"` + FirecrawlAPIKey string `json:"firecrawl-api-key,omitempty"` } // AgentsConfig groups LLM provider credentials and per-agent model // settings. Default is used as a fallback when an agent-specific field // is zero-valued. AgentsConfig struct { - Providers map[string]LLMProviderConfig `json:"providers"` + Providers map[string]LLMProviderConfig `json:"providers,omitempty"` Default LLMAgentConfig `json:"defaults"` - Probo LLMAgentConfig `json:"probo"` - EvidenceDescriber LLMAgentConfig `json:"evidence-describer"` - ThirdPartyVetter LLMAgentConfig `json:"third-party-vetter"` - ThirdPartyDisambiguation LLMAgentConfig `json:"third-party-disambiguation"` - TrackerMapping LLMAgentConfig `json:"tracker-mapping"` - TrackerEnrichment LLMAgentConfig `json:"tracker-enrichment"` - CommonThirdPartyEnrichment LLMAgentConfig `json:"common-third-party-enrichment"` - Tools AgentToolsConfig `json:"tools"` + Probo LLMAgentConfig `json:"probo,omitzero"` + EvidenceDescriber LLMAgentConfig `json:"evidence-describer,omitzero"` + ThirdPartyVetter LLMAgentConfig `json:"third-party-vetter,omitzero"` + ThirdPartyDisambiguation LLMAgentConfig `json:"third-party-disambiguation,omitzero"` + TrackerMapping LLMAgentConfig `json:"tracker-mapping,omitzero"` + TrackerEnrichment LLMAgentConfig `json:"tracker-enrichment,omitzero"` + CommonThirdPartyEnrichment LLMAgentConfig `json:"common-third-party-enrichment,omitzero"` + Tools AgentToolsConfig `json:"tools,omitzero"` } ) +func (c LLMProviderConfig) IsZero() bool { + return c.APIKey == "" +} + +func (c LLMAgentConfig) IsZero() bool { + return c.Provider == "" && c.ModelName == "" +} + +func (c AgentToolsConfig) IsZero() bool { + return c.FirecrawlAPIKey == "" +} + // ResolveAgent returns a fully populated LLMAgentConfig by filling in // zero-valued fields from the default config. func (c *AgentsConfig) ResolveAgent(agent LLMAgentConfig) LLMAgentConfig { diff --git a/pkg/probodconfig/mailer_config.go b/pkg/probodconfig/mailer_config.go index d9d91b0dd..a4a5fdbaf 100644 --- a/pkg/probodconfig/mailer_config.go +++ b/pkg/probodconfig/mailer_config.go @@ -16,15 +16,15 @@ package probodconfig type MailerConfig struct { MailerInterval int `json:"mailer-interval"` - SenderName string `json:"sender-name"` - SenderEmail string `json:"sender-email"` - SMTP SMTPConfig `json:"smtp"` + SenderName string `json:"sender-name,omitempty"` + SenderEmail string `json:"sender-email,omitempty"` + SMTP SMTPConfig `json:"smtp,omitzero"` } type SMTPConfig struct { - Addr string `json:"addr"` - User string `json:"user"` - Password string `json:"password"` + Addr string `json:"addr,omitempty"` + User string `json:"user,omitempty"` + Password string `json:"password,omitempty"` TLSRequired bool `json:"tls-required"` - HelloName string `json:"hello-name"` + HelloName string `json:"hello-name,omitempty"` } diff --git a/pkg/probodconfig/oidc_config.go b/pkg/probodconfig/oidc_config.go index ad1dd69e1..59fe34531 100644 --- a/pkg/probodconfig/oidc_config.go +++ b/pkg/probodconfig/oidc_config.go @@ -15,7 +15,11 @@ package probodconfig type OIDCProviderConfig struct { - ClientID string `json:"client-id"` - ClientSecret string `json:"client-secret"` - Enabled bool `json:"enabled"` + ClientID string `json:"client-id,omitempty"` + ClientSecret string `json:"client-secret,omitempty"` + Enabled bool `json:"enabled,omitempty"` +} + +func (c OIDCProviderConfig) IsZero() bool { + return c.ClientID == "" && c.ClientSecret == "" } diff --git a/pkg/probodconfig/pg_config.go b/pkg/probodconfig/pg_config.go index f75dc127b..de70a977c 100644 --- a/pkg/probodconfig/pg_config.go +++ b/pkg/probodconfig/pg_config.go @@ -23,17 +23,17 @@ import ( ) type PgConfig struct { - Addr string `json:"addr"` - Username string `json:"username"` - Password string `json:"password"` - Database string `json:"database"` + Addr string `json:"addr,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + Database string `json:"database,omitempty"` PoolSize int32 `json:"pool-size"` MinPoolSize int32 `json:"min-pool-size"` MaxConnIdleTimeSeconds int `json:"max-conn-idle-time-seconds"` MaxConnLifetimeSeconds int `json:"max-conn-lifetime-seconds"` MaxConnLifetimeJitterSeconds int `json:"max-conn-lifetime-jitter-seconds"` HealthCheckPeriodSeconds int `json:"health-check-period-seconds"` - CACertBundle string `json:"ca-cert-bundle"` + CACertBundle string `json:"ca-cert-bundle,omitempty"` Debug bool `json:"debug"` } diff --git a/pkg/probodconfig/saml_config.go b/pkg/probodconfig/saml_config.go index 7caf19688..8cb098177 100644 --- a/pkg/probodconfig/saml_config.go +++ b/pkg/probodconfig/saml_config.go @@ -24,7 +24,7 @@ type SAMLConfig struct { Certificate string `json:"certificate"` PrivateKey string `json:"private-key"` DomainVerificationIntervalSeconds int `json:"domain-verification-interval-seconds"` - DomainVerificationResolverAddr string `json:"domain-verification-resolver-addr"` + DomainVerificationResolverAddr string `json:"domain-verification-resolver-addr,omitempty"` } func (c SAMLConfig) SessionDurationTime() time.Duration { diff --git a/pkg/probodconfig/slack_config.go b/pkg/probodconfig/slack_config.go index 452ffc321..602757d7e 100644 --- a/pkg/probodconfig/slack_config.go +++ b/pkg/probodconfig/slack_config.go @@ -16,5 +16,5 @@ package probodconfig type SlackConfig struct { SenderInterval int `json:"sender-interval"` - SigningSecret string `json:"signing-secret"` + SigningSecret string `json:"signing-secret,omitempty"` }