rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tune tracker workers and bound agent budgets

Browse Source 
The tracker-mapping and common-pattern-enrichment workers ran with the
kit/worker defaults (interval 10s, max-concurrency 5 each) and dropped
the resolved per-agent max-tokens/temperature, so up to ten LLM
pipelines could run unbounded on one OpenAI client. The mapping worker
also held a FOR UPDATE transaction across the LLM and Firecrawl calls
while its DB search tools acquired a second pooled connection, risking
pool exhaustion under concurrency.

Plumb max-tokens, temperature, agent timeout, and per-worker max-turns
through TrackerAgentsConfig and DisambiguationConfig into all three
agent builders, replacing the hard-coded constants with config-fed
fields and package fallbacks. Expose worker interval, concurrency,
stale-after, agent timeout, and max-turns as config (env, Helm values,
deployment template) mirroring the evidence-describer pattern, and
apply them at registration.

Refactor Process into deterministic-read, agent (no transaction), and
persist phases so neither the mapping agent nor disambiguation runs
inside an open transaction, removing the row locks held across network
latency and the nested-connection pressure.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-05-29 16:37:32 +02:00
parent 50c5454681
commit 55302d18f0
17 changed files with 818 additions and 339 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
pkg/bootstrap/builder.go
View File

@@ -207,10 +207,13 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
MaxTokens: b.getEnvIntPtr("AGENT_EVIDENCE_DESCRIBER_MAX_TOKENS"),
},
TrackerMapping: probodconfig.LLMAgentConfig{
Provider: b.getEnvOrDefault("AGENT_TRACKER_MAPPING_PROVIDER", ""),
ModelName: b.getEnvOrDefault("AGENT_TRACKER_MAPPING_MODEL_NAME", ""),
Provider: b.getEnvOrDefault("AGENT_TRACKER_MAPPING_PROVIDER", ""),
ModelName: b.getEnvOrDefault("AGENT_TRACKER_MAPPING_MODEL_NAME", ""),
// The tracker agents emit tiny structured JSON, so
// they default to a smaller token budget than the
// shared default rather than inheriting it.
Temperature: b.getEnvFloatPtr("AGENT_TRACKER_MAPPING_TEMPERATURE"),
MaxTokens: b.getEnvIntPtr("AGENT_TRACKER_MAPPING_MAX_TOKENS"),
MaxTokens: new(b.getEnvIntOrDefault("AGENT_TRACKER_MAPPING_MAX_TOKENS", 1024)),
},
Tools: probodconfig.AgentToolsConfig{
FirecrawlAPIKey: b.getEnv("FIRECRAWL_API_KEY"),
@@ -242,6 +245,19 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
StaleAfter: b.getEnvIntOrDefault("EVIDENCE_DESCRIBER_STALE_AFTER", 300),
MaxConcurrency: b.getEnvIntOrDefault("EVIDENCE_DESCRIBER_MAX_CONCURRENCY", 10),
},
TrackerMappingWorker: probodconfig.TrackerMappingWorkerConfig{
Interval: b.getEnvIntOrDefault("TRACKER_MAPPING_INTERVAL", 10),
MaxConcurrency: b.getEnvIntOrDefault("TRACKER_MAPPING_MAX_CONCURRENCY", 3),
AgentTimeout: b.getEnvIntOrDefault("TRACKER_MAPPING_AGENT_TIMEOUT", 45),
AgentMaxTurns: b.getEnvIntOrDefault("TRACKER_MAPPING_AGENT_MAX_TURNS", 4),
},
CommonPatternEnrichmentWorker: probodconfig.CommonPatternEnrichmentWorkerConfig{
Interval: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_INTERVAL", 10),
MaxConcurrency: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_MAX_CONCURRENCY", 2),
StaleAfter: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_STALE_AFTER", 600),
AgentTimeout: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_AGENT_TIMEOUT", 45),
AgentMaxTurns: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS", 3),
},
Branding: b.getEnvBoolOrDefault("BRANDING", true),
},
}

33
pkg/bootstrap/builder_test.go
View File

@@ -218,7 +218,18 @@ func TestBuilder_Build_Defaults(t *testing.T) {
assert.Empty(t, cfg.Probod.Agents.TrackerMapping.Provider)
assert.Empty(t, cfg.Probod.Agents.TrackerMapping.ModelName)
assert.Nil(t, cfg.Probod.Agents.TrackerMapping.Temperature)
assert.Nil(t, cfg.Probod.Agents.TrackerMapping.MaxTokens)
assert.Equal(t, new(1024), cfg.Probod.Agents.TrackerMapping.MaxTokens)
// Tracker worker tuning — defaults
assert.Equal(t, 10, cfg.Probod.TrackerMappingWorker.Interval)
assert.Equal(t, 3, cfg.Probod.TrackerMappingWorker.MaxConcurrency)
assert.Equal(t, 45, cfg.Probod.TrackerMappingWorker.AgentTimeout)
assert.Equal(t, 4, cfg.Probod.TrackerMappingWorker.AgentMaxTurns)
assert.Equal(t, 10, cfg.Probod.CommonPatternEnrichmentWorker.Interval)
assert.Equal(t, 2, cfg.Probod.CommonPatternEnrichmentWorker.MaxConcurrency)
assert.Equal(t, 600, cfg.Probod.CommonPatternEnrichmentWorker.StaleAfter)
assert.Equal(t, 45, cfg.Probod.CommonPatternEnrichmentWorker.AgentTimeout)
assert.Equal(t, 3, cfg.Probod.CommonPatternEnrichmentWorker.AgentMaxTurns)
// Custom domains config
assert.Equal(t, 3600, cfg.Probod.CustomDomains.RenewalInterval)
@@ -313,6 +324,16 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
env["AGENT_TRACKER_MAPPING_MODEL_NAME"] = "gpt-4o-mini"
env["AGENT_TRACKER_MAPPING_TEMPERATURE"] = "0.1"
env["AGENT_TRACKER_MAPPING_MAX_TOKENS"] = "1024"
// Tracker worker tuning override
env["TRACKER_MAPPING_INTERVAL"] = "20"
env["TRACKER_MAPPING_MAX_CONCURRENCY"] = "5"
env["TRACKER_MAPPING_AGENT_TIMEOUT"] = "30"
env["TRACKER_MAPPING_AGENT_MAX_TURNS"] = "6"
env["COMMON_PATTERN_ENRICHMENT_INTERVAL"] = "15"
env["COMMON_PATTERN_ENRICHMENT_MAX_CONCURRENCY"] = "4"
env["COMMON_PATTERN_ENRICHMENT_STALE_AFTER"] = "900"
env["COMMON_PATTERN_ENRICHMENT_AGENT_TIMEOUT"] = "50"
env["COMMON_PATTERN_ENRICHMENT_AGENT_MAX_TURNS"] = "5"
// Custom domains
env["CUSTOM_DOMAINS_RESOLVER_ADDR"] = "1.1.1.1:53"
env["ACME_ACCOUNT_KEY"] = "-----BEGIN EC PRIVATE KEY-----\ntest\n-----END EC PRIVATE KEY-----"
@@ -404,6 +425,16 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
assert.Equal(t, "gpt-4o-mini", cfg.Probod.Agents.TrackerMapping.ModelName)
assert.Equal(t, new(0.1), cfg.Probod.Agents.TrackerMapping.Temperature)
assert.Equal(t, new(1024), cfg.Probod.Agents.TrackerMapping.MaxTokens)
// Tracker worker tuning — overrides
assert.Equal(t, 20, cfg.Probod.TrackerMappingWorker.Interval)
assert.Equal(t, 5, cfg.Probod.TrackerMappingWorker.MaxConcurrency)
assert.Equal(t, 30, cfg.Probod.TrackerMappingWorker.AgentTimeout)
assert.Equal(t, 6, cfg.Probod.TrackerMappingWorker.AgentMaxTurns)
assert.Equal(t, 15, cfg.Probod.CommonPatternEnrichmentWorker.Interval)
assert.Equal(t, 4, cfg.Probod.CommonPatternEnrichmentWorker.MaxConcurrency)
assert.Equal(t, 900, cfg.Probod.CommonPatternEnrichmentWorker.StaleAfter)
assert.Equal(t, 50, cfg.Probod.CommonPatternEnrichmentWorker.AgentTimeout)
assert.Equal(t, 5, cfg.Probod.CommonPatternEnrichmentWorker.AgentMaxTurns)
// Custom domains
assert.Equal(t, "1.1.1.1:53", cfg.Probod.CustomDomains.ResolverAddr)
assert.Equal(t, "-----BEGIN EC PRIVATE KEY-----\ntest\n-----END EC PRIVATE KEY-----", cfg.Probod.CustomDomains.ACME.AccountKey)