From 27ff38df7d4392afa0bc803ac863ea4656875a12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89mile=20R=C3=A9?= Date: Wed, 10 Jun 2026 14:49:24 +0200 Subject: [PATCH] Remove third-party disambiguation agent and matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tracker-mapping worker no longer auto-creates or auto-links org third parties, so the heuristic ranker, the disambiguation agent, and the catalog-to-org seeding helpers are dead code. Delete pkg/thirdparty/match.go (RankCandidates, ScoredCandidate, LinkToCommon, CreateFromCommon, suffix stripping, score thresholds) and disambiguation_agent.go, along with their tests. Drop the ThirdPartyDisambiguation agent slot and the worker's DisambiguationAgentTimeout from probodconfig, the builder env wiring, and the builder tests. Remove the matching helm surface too: the thirdPartyDisambiguation agent values, the disambiguationAgentTimeout worker tuning, and the AGENT_THIRD_PARTY_DISAMBIGUATION_* / TRACKER_MAPPING_DISAMBIGUATION_AGENT _TIMEOUT environment mappings, with a chart changelog note. The probod config is built from env lookups with defaults, so a lingering value in an older deployment is simply ignored. Signed-off-by: Émile Ré --- contrib/helm/charts/probo/CHANGELOG.md | 4 + .../charts/probo/templates/deployment.yaml | 21 -- .../probo/values-production.yaml.example | 12 +- contrib/helm/charts/probo/values.yaml | 16 +- pkg/bootstrap/builder.go | 22 +- pkg/bootstrap/builder_test.go | 17 -- pkg/probodconfig/llm_config.go | 31 +- pkg/thirdparty/disambiguation_agent.go | 241 --------------- pkg/thirdparty/match.go | 276 ------------------ pkg/thirdparty/match_test.go | 178 ----------- 10 files changed, 26 insertions(+), 792 deletions(-) delete mode 100644 pkg/thirdparty/disambiguation_agent.go delete mode 100644 pkg/thirdparty/match.go delete mode 100644 pkg/thirdparty/match_test.go diff --git a/contrib/helm/charts/probo/CHANGELOG.md b/contrib/helm/charts/probo/CHANGELOG.md index 315e6e925..607c60acc 100644 --- a/contrib/helm/charts/probo/CHANGELOG.md +++ b/contrib/helm/charts/probo/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to the Probo Helm chart will be documented in this file. ## Unreleased +### Removed + +- `thirdPartyDisambiguation` agent config slot and `trackerMappingWorker.disambiguationAgentTimeout`; the tracker-mapping worker no longer runs a third-party disambiguation agent + ## [0.5.0] - 2026-06-09 ### Added diff --git a/contrib/helm/charts/probo/templates/deployment.yaml b/contrib/helm/charts/probo/templates/deployment.yaml index 795fc1bbd..db8f724fe 100644 --- a/contrib/helm/charts/probo/templates/deployment.yaml +++ b/contrib/helm/charts/probo/templates/deployment.yaml @@ -281,23 +281,6 @@ spec: - name: THIRD_PARTY_VETTING_STALE_AFTER value: {{ .Values.probo.thirdPartyVettingWorker.staleAfter | quote }} {{- end }} - # Third-party Disambiguation Agent - {{- if .Values.probo.thirdPartyDisambiguation.provider }} - - name: AGENT_THIRD_PARTY_DISAMBIGUATION_PROVIDER - value: {{ .Values.probo.thirdPartyDisambiguation.provider | quote }} - {{- end }} - {{- if .Values.probo.thirdPartyDisambiguation.modelName }} - - name: AGENT_THIRD_PARTY_DISAMBIGUATION_MODEL_NAME - value: {{ .Values.probo.thirdPartyDisambiguation.modelName | quote }} - {{- end }} - {{- if .Values.probo.thirdPartyDisambiguation.temperature }} - - name: AGENT_THIRD_PARTY_DISAMBIGUATION_TEMPERATURE - value: {{ .Values.probo.thirdPartyDisambiguation.temperature | quote }} - {{- end }} - {{- if .Values.probo.thirdPartyDisambiguation.maxTokens }} - - name: AGENT_THIRD_PARTY_DISAMBIGUATION_MAX_TOKENS - value: {{ .Values.probo.thirdPartyDisambiguation.maxTokens | quote }} - {{- end }} # Tracker Mapping Agent {{- if .Values.probo.trackerMapping.provider }} - name: AGENT_TRACKER_MAPPING_PROVIDER @@ -353,10 +336,6 @@ spec: - name: TRACKER_MAPPING_AGENT_MAX_TURNS value: {{ .Values.probo.trackerMappingWorker.agentMaxTurns | quote }} {{- end }} - {{- if .Values.probo.trackerMappingWorker.disambiguationAgentTimeout }} - - name: TRACKER_MAPPING_DISAMBIGUATION_AGENT_TIMEOUT - value: {{ .Values.probo.trackerMappingWorker.disambiguationAgentTimeout | quote }} - {{- end }} # Common Pattern Enrichment Worker {{- if .Values.probo.commonPatternEnrichmentWorker.interval }} - name: COMMON_PATTERN_ENRICHMENT_INTERVAL diff --git a/contrib/helm/charts/probo/values-production.yaml.example b/contrib/helm/charts/probo/values-production.yaml.example index 4de046539..f302ad9e5 100644 --- a/contrib/helm/charts/probo/values-production.yaml.example +++ b/contrib/helm/charts/probo/values-production.yaml.example @@ -185,15 +185,6 @@ probo: # maxConcurrency: 1 # staleAfter: 1500 - # Third-party disambiguation agent (optional; runs inside the tracker - # mapping worker to pick the best matching org third party). Falls back - # to trackerMapping when its provider is unset. - # thirdPartyDisambiguation: - # provider: "openai" - # modelName: "gpt-4o" - # temperature: "0.4" - # maxTokens: "4096" - # Tracker mapping agent (optional, auto-links tracker patterns to vendors) # trackerMapping: # provider: "openai" @@ -210,7 +201,7 @@ probo: # maxTokens: "4096" # Tracker mapping worker tuning (optional; seconds for - # interval/staleAfter/agentTimeout/disambiguationAgentTimeout). + # interval/staleAfter/agentTimeout). # Keep concurrency modest to stay under OpenAI/Firecrawl limits and the DB pool. # trackerMappingWorker: # interval: 10 @@ -218,7 +209,6 @@ probo: # staleAfter: 600 # agentTimeout: 45 # agentMaxTurns: 10 - # disambiguationAgentTimeout: 45 # Common-pattern enrichment worker tuning (optional; seconds for # interval/staleAfter/agentTimeout). diff --git a/contrib/helm/charts/probo/values.yaml b/contrib/helm/charts/probo/values.yaml index 27e2d8656..9724e2381 100644 --- a/contrib/helm/charts/probo/values.yaml +++ b/contrib/helm/charts/probo/values.yaml @@ -286,15 +286,6 @@ probo: maxConcurrency: 1 staleAfter: 1500 - # Third-party disambiguation agent (optional; runs inside the tracker - # mapping worker to pick the best matching org third party). Falls back - # to trackerMapping when its provider is unset. - thirdPartyDisambiguation: - provider: "" - modelName: "" - temperature: "" - maxTokens: "" - # Tracker mapping agent (optional, requires openai.apiKey or anthropic key) trackerMapping: provider: "" @@ -311,16 +302,15 @@ probo: maxTokens: "" # Tracker mapping background worker tuning (optional). interval, - # staleAfter, agentTimeout, and disambiguationAgentTimeout are in - # seconds. Keep concurrency modest to stay under OpenAI/Firecrawl rate - # limits and the database connection pool. + # staleAfter, and agentTimeout are in seconds. Keep concurrency modest + # to stay under OpenAI/Firecrawl rate limits and the database + # connection pool. trackerMappingWorker: interval: 10 maxConcurrency: 3 staleAfter: 600 agentTimeout: 45 agentMaxTurns: 10 - disambiguationAgentTimeout: 45 # Common-pattern enrichment background worker tuning (optional). # interval, staleAfter, and agentTimeout are in seconds. diff --git a/pkg/bootstrap/builder.go b/pkg/bootstrap/builder.go index 2eac4449d..b065293cd 100644 --- a/pkg/bootstrap/builder.go +++ b/pkg/bootstrap/builder.go @@ -213,17 +213,6 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { Temperature: b.getEnvFloatPtr("AGENT_THIRD_PARTY_VETTER_TEMPERATURE"), MaxTokens: b.getEnvIntPtr("AGENT_THIRD_PARTY_VETTER_MAX_TOKENS"), }, - ThirdPartyDisambiguation: probodconfig.LLMAgentConfig{ - Provider: b.getEnvOrDefault("AGENT_THIRD_PARTY_DISAMBIGUATION_PROVIDER", ""), - ModelName: b.getEnvOrDefault("AGENT_THIRD_PARTY_DISAMBIGUATION_MODEL_NAME", ""), - // The disambiguation agent emits a single id plus a - // short rationale, but the budget must leave headroom - // for reasoning models whose reasoning tokens count - // against max_tokens; too small a budget truncates the - // JSON. - Temperature: b.getEnvFloatPtr("AGENT_THIRD_PARTY_DISAMBIGUATION_TEMPERATURE"), - MaxTokens: new(b.getEnvIntOrDefault("AGENT_THIRD_PARTY_DISAMBIGUATION_MAX_TOKENS", 4096)), - }, TrackerMapping: probodconfig.LLMAgentConfig{ Provider: b.getEnvOrDefault("AGENT_TRACKER_MAPPING_PROVIDER", ""), ModelName: b.getEnvOrDefault("AGENT_TRACKER_MAPPING_MODEL_NAME", ""), @@ -278,12 +267,11 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { MaxConcurrency: b.getEnvIntOrDefault("THIRD_PARTY_VETTING_MAX_CONCURRENCY", 1), }, TrackerMappingWorker: probodconfig.TrackerMappingWorkerConfig{ - Interval: b.getEnvIntOrDefault("TRACKER_MAPPING_INTERVAL", 10), - MaxConcurrency: b.getEnvIntOrDefault("TRACKER_MAPPING_MAX_CONCURRENCY", 3), - StaleAfter: b.getEnvIntOrDefault("TRACKER_MAPPING_STALE_AFTER", 600), - AgentTimeout: b.getEnvIntOrDefault("TRACKER_MAPPING_AGENT_TIMEOUT", 45), - AgentMaxTurns: b.getEnvIntOrDefault("TRACKER_MAPPING_AGENT_MAX_TURNS", 10), - DisambiguationAgentTimeout: b.getEnvIntOrDefault("TRACKER_MAPPING_DISAMBIGUATION_AGENT_TIMEOUT", 45), + Interval: b.getEnvIntOrDefault("TRACKER_MAPPING_INTERVAL", 10), + MaxConcurrency: b.getEnvIntOrDefault("TRACKER_MAPPING_MAX_CONCURRENCY", 3), + StaleAfter: b.getEnvIntOrDefault("TRACKER_MAPPING_STALE_AFTER", 600), + AgentTimeout: b.getEnvIntOrDefault("TRACKER_MAPPING_AGENT_TIMEOUT", 45), + AgentMaxTurns: b.getEnvIntOrDefault("TRACKER_MAPPING_AGENT_MAX_TURNS", 10), }, CommonPatternEnrichmentWorker: probodconfig.CommonPatternEnrichmentWorkerConfig{ Interval: b.getEnvIntOrDefault("COMMON_PATTERN_ENRICHMENT_INTERVAL", 10), diff --git a/pkg/bootstrap/builder_test.go b/pkg/bootstrap/builder_test.go index b8889ee6b..bbd3a296e 100644 --- a/pkg/bootstrap/builder_test.go +++ b/pkg/bootstrap/builder_test.go @@ -220,10 +220,6 @@ func TestBuilder_Build_Defaults(t *testing.T) { assert.Empty(t, cfg.Probod.Agents.ThirdPartyVetter.ModelName) assert.Nil(t, cfg.Probod.Agents.ThirdPartyVetter.Temperature) assert.Nil(t, cfg.Probod.Agents.ThirdPartyVetter.MaxTokens) - assert.Empty(t, cfg.Probod.Agents.ThirdPartyDisambiguation.Provider) - assert.Empty(t, cfg.Probod.Agents.ThirdPartyDisambiguation.ModelName) - assert.Nil(t, cfg.Probod.Agents.ThirdPartyDisambiguation.Temperature) - assert.Equal(t, new(4096), cfg.Probod.Agents.ThirdPartyDisambiguation.MaxTokens) assert.Empty(t, cfg.Probod.Agents.TrackerMapping.Provider) assert.Empty(t, cfg.Probod.Agents.TrackerMapping.ModelName) assert.Nil(t, cfg.Probod.Agents.TrackerMapping.Temperature) @@ -239,7 +235,6 @@ func TestBuilder_Build_Defaults(t *testing.T) { assert.Equal(t, 600, cfg.Probod.TrackerMappingWorker.StaleAfter) assert.Equal(t, 45, cfg.Probod.TrackerMappingWorker.AgentTimeout) assert.Equal(t, 10, cfg.Probod.TrackerMappingWorker.AgentMaxTurns) - assert.Equal(t, 45, cfg.Probod.TrackerMappingWorker.DisambiguationAgentTimeout) assert.Equal(t, 10, cfg.Probod.CommonPatternEnrichmentWorker.Interval) assert.Equal(t, 2, cfg.Probod.CommonPatternEnrichmentWorker.MaxConcurrency) assert.Equal(t, 600, cfg.Probod.CommonPatternEnrichmentWorker.StaleAfter) @@ -342,11 +337,6 @@ func TestBuilder_Build_CustomValues(t *testing.T) { env["AGENT_THIRD_PARTY_VETTER_MODEL_NAME"] = "gpt-4o" env["AGENT_THIRD_PARTY_VETTER_TEMPERATURE"] = "0.3" env["AGENT_THIRD_PARTY_VETTER_MAX_TOKENS"] = "8192" - // Agents — third-party-disambiguation override - env["AGENT_THIRD_PARTY_DISAMBIGUATION_PROVIDER"] = "anthropic" - env["AGENT_THIRD_PARTY_DISAMBIGUATION_MODEL_NAME"] = "claude-sonnet-4-20250514" - env["AGENT_THIRD_PARTY_DISAMBIGUATION_TEMPERATURE"] = "0.4" - env["AGENT_THIRD_PARTY_DISAMBIGUATION_MAX_TOKENS"] = "2048" // Agents — tracker-mapping override env["AGENT_TRACKER_MAPPING_PROVIDER"] = "openai" env["AGENT_TRACKER_MAPPING_MODEL_NAME"] = "gpt-4o-mini" @@ -363,7 +353,6 @@ func TestBuilder_Build_CustomValues(t *testing.T) { env["TRACKER_MAPPING_STALE_AFTER"] = "1200" env["TRACKER_MAPPING_AGENT_TIMEOUT"] = "30" env["TRACKER_MAPPING_AGENT_MAX_TURNS"] = "6" - env["TRACKER_MAPPING_DISAMBIGUATION_AGENT_TIMEOUT"] = "35" env["COMMON_PATTERN_ENRICHMENT_INTERVAL"] = "15" env["COMMON_PATTERN_ENRICHMENT_MAX_CONCURRENCY"] = "4" env["COMMON_PATTERN_ENRICHMENT_STALE_AFTER"] = "900" @@ -463,11 +452,6 @@ func TestBuilder_Build_CustomValues(t *testing.T) { assert.Equal(t, "gpt-4o", cfg.Probod.Agents.ThirdPartyVetter.ModelName) assert.Equal(t, new(0.3), cfg.Probod.Agents.ThirdPartyVetter.Temperature) assert.Equal(t, new(8192), cfg.Probod.Agents.ThirdPartyVetter.MaxTokens) - // Agents — third-party-disambiguation overrides - assert.Equal(t, "anthropic", cfg.Probod.Agents.ThirdPartyDisambiguation.Provider) - assert.Equal(t, "claude-sonnet-4-20250514", cfg.Probod.Agents.ThirdPartyDisambiguation.ModelName) - assert.Equal(t, new(0.4), cfg.Probod.Agents.ThirdPartyDisambiguation.Temperature) - assert.Equal(t, new(2048), cfg.Probod.Agents.ThirdPartyDisambiguation.MaxTokens) // Agents — tracker-mapping overrides assert.Equal(t, "openai", cfg.Probod.Agents.TrackerMapping.Provider) assert.Equal(t, "gpt-4o-mini", cfg.Probod.Agents.TrackerMapping.ModelName) @@ -484,7 +468,6 @@ func TestBuilder_Build_CustomValues(t *testing.T) { assert.Equal(t, 1200, cfg.Probod.TrackerMappingWorker.StaleAfter) assert.Equal(t, 30, cfg.Probod.TrackerMappingWorker.AgentTimeout) assert.Equal(t, 6, cfg.Probod.TrackerMappingWorker.AgentMaxTurns) - assert.Equal(t, 35, cfg.Probod.TrackerMappingWorker.DisambiguationAgentTimeout) assert.Equal(t, 15, cfg.Probod.CommonPatternEnrichmentWorker.Interval) assert.Equal(t, 4, cfg.Probod.CommonPatternEnrichmentWorker.MaxConcurrency) assert.Equal(t, 900, cfg.Probod.CommonPatternEnrichmentWorker.StaleAfter) diff --git a/pkg/probodconfig/llm_config.go b/pkg/probodconfig/llm_config.go index 589c042a6..63a447a74 100644 --- a/pkg/probodconfig/llm_config.go +++ b/pkg/probodconfig/llm_config.go @@ -53,16 +53,12 @@ type ( // tracker-mapping background worker. LLM parameters for the mapping // agent it runs live under AgentsConfig.TrackerMapping. AgentTimeout // and AgentMaxTurns bound a single mapping agent run. - // DisambiguationAgentTimeout caps a single third-party - // disambiguation agent run; that agent runs inside this worker but - // uses its own LLM parameters from AgentsConfig.ThirdPartyDisambiguation. TrackerMappingWorkerConfig struct { - Interval int `json:"interval"` // seconds between polls - MaxConcurrency int `json:"max-concurrency"` - StaleAfter int `json:"stale-after"` // seconds before a claim is recycled - AgentTimeout int `json:"agent-timeout"` // seconds, single agent run - AgentMaxTurns int `json:"agent-max-turns"` - DisambiguationAgentTimeout int `json:"disambiguation-agent-timeout"` // seconds, single disambiguation run + Interval int `json:"interval"` // seconds between polls + MaxConcurrency int `json:"max-concurrency"` + StaleAfter int `json:"stale-after"` // seconds before a claim is recycled + AgentTimeout int `json:"agent-timeout"` // seconds, single agent run + AgentMaxTurns int `json:"agent-max-turns"` } // CommonPatternEnrichmentWorkerConfig holds worker-side tuning for @@ -86,15 +82,14 @@ type ( // settings. Default is used as a fallback when an agent-specific field // is zero-valued. AgentsConfig struct { - Providers map[string]LLMProviderConfig `json:"providers"` - Default LLMAgentConfig `json:"defaults"` - Probo LLMAgentConfig `json:"probo"` - EvidenceDescriber LLMAgentConfig `json:"evidence-describer"` - ThirdPartyVetter LLMAgentConfig `json:"third-party-vetter"` - ThirdPartyDisambiguation LLMAgentConfig `json:"third-party-disambiguation"` - TrackerMapping LLMAgentConfig `json:"tracker-mapping"` - TrackerEnrichment LLMAgentConfig `json:"tracker-enrichment"` - Tools AgentToolsConfig `json:"tools"` + Providers map[string]LLMProviderConfig `json:"providers"` + Default LLMAgentConfig `json:"defaults"` + Probo LLMAgentConfig `json:"probo"` + EvidenceDescriber LLMAgentConfig `json:"evidence-describer"` + ThirdPartyVetter LLMAgentConfig `json:"third-party-vetter"` + TrackerMapping LLMAgentConfig `json:"tracker-mapping"` + TrackerEnrichment LLMAgentConfig `json:"tracker-enrichment"` + Tools AgentToolsConfig `json:"tools"` } ) diff --git a/pkg/thirdparty/disambiguation_agent.go b/pkg/thirdparty/disambiguation_agent.go deleted file mode 100644 index e7487c8f7..000000000 --- a/pkg/thirdparty/disambiguation_agent.go +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright (c) 2026 Probo Inc . -// -// Permission to use, copy, modify, and/or distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -// PERFORMANCE OF THIS SOFTWARE. - -package thirdparty - -import ( - "context" - _ "embed" - "fmt" - "strings" - "time" - - "go.gearno.de/kit/log" - "go.probo.inc/probo/pkg/agent" - "go.probo.inc/probo/pkg/coredata" - "go.probo.inc/probo/pkg/gid" - "go.probo.inc/probo/pkg/llm" -) - -//go:embed prompts/disambiguation.txt.tmpl -var disambiguationPrompt string - -const ( - // disambiguationConfidenceThreshold is the floor below which we - // treat the agent's pick as "no confident match" even when it - // returned a non-nil matched_id. Mirrors the conservative bias - // described in the prompt. - disambiguationConfidenceThreshold = 0.6 - - // defaultDisambiguationTimeout caps a single disambiguation run - // when the config supplies none. The agent has no tools and a - // single turn, so this is mostly a guard against a hung LLM - // provider, not a real budget. - defaultDisambiguationTimeout = 45 * time.Second - - // defaultDisambiguationMaxTokens caps the agent's output when the - // config carries no max-tokens budget. The final output is tiny (a - // single id plus a one-sentence rationale), but the budget must - // leave ample headroom for reasoning models (e.g. the GPT-5 - // family): their reasoning tokens count against max_tokens, so too - // small a budget gets consumed by reasoning and truncates the JSON, - // surfacing as "unexpected end of JSON input". - defaultDisambiguationMaxTokens = 4096 -) - -// DisambiguationAgentConfig configures the third-party disambiguation -// agent. The agent has no DB tools and no web-search tools: the -// candidate list is supplied entirely in the prompt and the agent -// only picks among it. -// -// MaxTokens and Temperature bound and steer the single LLM call, and -// Timeout caps a single run. Zero-valued fields fall back to package -// defaults. -type DisambiguationAgentConfig struct { - LLMClient *llm.Client - Model string - MaxTokens *int - Temperature *float64 - Timeout time.Duration -} - -// DisambiguationResult is the structured output the disambiguation -// agent returns when picking the best existing org ThirdParty for a -// catalog entry. -type DisambiguationResult struct { - MatchedID *string `json:"matched_id" jsonschema:"GID of the org third party that best matches, or null if none of the candidates is a confident match."` - Confidence float64 `json:"confidence" jsonschema:"Confidence level from 0.0 to 1.0. Below 0.6 means 'no confident match' and matched_id MUST be null."` - Reasoning string `json:"reasoning" jsonschema:"One short sentence describing the rationale."` -} - -// BuildDisambiguationAgent wires the agent that picks the best -// existing org ThirdParty for a catalog entry. It deliberately has -// no tools: the candidate list is supplied in the prompt and the -// agent must only choose among it. -func BuildDisambiguationAgent( - cfg DisambiguationAgentConfig, - logger *log.Logger, -) *agent.Agent { - outputType, err := agent.NewOutputType[DisambiguationResult]("third_party_disambiguation") - if err != nil { - panic(fmt.Sprintf("thirdparty: cannot build disambiguation output type: %s", err)) - } - - maxTokens := defaultDisambiguationMaxTokens - if cfg.MaxTokens != nil && *cfg.MaxTokens > 0 { - maxTokens = *cfg.MaxTokens - } - - opts := []agent.Option{ - agent.WithInstructions(disambiguationPrompt), - agent.WithModel(cfg.Model), - agent.WithOutputType(outputType), - agent.WithMaxTurns(1), - agent.WithMaxTokens(maxTokens), - agent.WithLogger(logger), - } - - if cfg.Temperature != nil { - opts = append(opts, agent.WithTemperature(*cfg.Temperature)) - } - - return agent.New("third-party-disambiguation", cfg.LLMClient, opts...) -} - -// Disambiguate runs the agent against the given catalog third party -// and candidate list, and returns the matched candidate's ID — or -// nil when the agent picks "none", returns a confidence below the -// threshold, or fails. Errors from the agent itself are returned; -// "no confident match" is not an error. -// -// The matched candidate is identified by string equality against the -// IDs supplied in `candidates`; we never invent IDs from the agent's -// output, so a model that hallucinates an ID is treated as "none". -func Disambiguate( - ctx context.Context, - a *agent.Agent, - logger *log.Logger, - commonParty coredata.CommonThirdParty, - commonDomains coredata.CommonThirdPartyDomains, - candidates []ScoredCandidate, - timeout time.Duration, -) (*gid.GID, error) { - if a == nil || len(candidates) == 0 { - return nil, nil - } - - if timeout <= 0 { - timeout = defaultDisambiguationTimeout - } - - prompt := buildDisambiguationPrompt(commonParty, commonDomains, candidates) - - agentCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - result, err := agent.RunTyped[DisambiguationResult]( - agentCtx, - a, - []llm.Message{ - { - Role: llm.RoleUser, - Parts: []llm.Part{llm.TextPart{Text: prompt}}, - }, - }, - ) - if err != nil { - return nil, fmt.Errorf("cannot run disambiguation agent: %w", err) - } - - out := result.Output - - if out.MatchedID == nil || *out.MatchedID == "" { - return nil, nil - } - - if out.Confidence < disambiguationConfidenceThreshold { - logger.InfoCtx( - ctx, - "disambiguation agent below confidence threshold", - log.String("matched_id", *out.MatchedID), - log.Float64("confidence", out.Confidence), - ) - - return nil, nil - } - - for _, c := range candidates { - if c.ThirdParty.ID.String() == *out.MatchedID { - id := c.ThirdParty.ID - - return &id, nil - } - } - - logger.WarnCtx( - ctx, - "disambiguation agent returned id not in candidate list", - log.String("matched_id", *out.MatchedID), - ) - - return nil, nil -} - -// buildDisambiguationPrompt formats the catalog third party and the -// heuristic-ranked candidate list into the user message for the -// disambiguation agent. The prompt is intentionally compact: the -// agent only needs ids, names, websites, and the heuristic score to -// decide. -func buildDisambiguationPrompt( - commonParty coredata.CommonThirdParty, - commonDomains coredata.CommonThirdPartyDomains, - candidates []ScoredCandidate, -) string { - var b strings.Builder - - b.WriteString("Catalog third party:\n") - fmt.Fprintf(&b, " name: %s\n", commonParty.Name) - - if commonParty.WebsiteURL != nil && *commonParty.WebsiteURL != "" { - fmt.Fprintf(&b, " website: %s\n", *commonParty.WebsiteURL) - } - - if len(commonDomains) > 0 { - domains := make([]string, len(commonDomains)) - for i, d := range commonDomains { - domains[i] = d.Domain - } - - fmt.Fprintf(&b, " domains: %s\n", strings.Join(domains, ", ")) - } - - b.WriteString("\nCandidate organisation third parties (heuristic-ranked):\n") - - for i, c := range candidates { - fmt.Fprintf(&b, "- id: %s\n", c.ThirdParty.ID.String()) - fmt.Fprintf(&b, " name: %s\n", c.ThirdParty.Name) - - if c.ThirdParty.WebsiteURL != nil && *c.ThirdParty.WebsiteURL != "" { - fmt.Fprintf(&b, " website: %s\n", *c.ThirdParty.WebsiteURL) - } - - fmt.Fprintf(&b, " heuristic_score: %.2f\n", c.Score) - - if i < len(candidates)-1 { - b.WriteString("\n") - } - } - - return b.String() -} diff --git a/pkg/thirdparty/match.go b/pkg/thirdparty/match.go deleted file mode 100644 index 4bfcd756c..000000000 --- a/pkg/thirdparty/match.go +++ /dev/null @@ -1,276 +0,0 @@ -// Copyright (c) 2026 Probo Inc . -// -// Permission to use, copy, modify, and/or distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -// PERFORMANCE OF THIS SOFTWARE. - -package thirdparty - -import ( - "context" - "fmt" - "sort" - "strings" - "time" - - "go.gearno.de/kit/pg" - "go.probo.inc/probo/pkg/coredata" - "go.probo.inc/probo/pkg/gid" - "go.probo.inc/probo/pkg/slug" - "go.probo.inc/probo/pkg/uri" -) - -// Heuristic thresholds for matching a CommonThirdParty to an existing -// org ThirdParty. Exported so callers can short-circuit explicitly -// (skip the agent, fall back to creating, etc.) instead of duplicating -// magic numbers. -const ( - // HighConfidenceScore is the floor at which a heuristic match - // is treated as obvious (exact name, suffix-stripped name, slug - // equality). Callers typically link without consulting the - // agent at this score. - HighConfidenceScore = 0.85 - - // MinAgentScore is the floor below which a candidate is - // statistical noise and should not be shown to the - // disambiguation agent. Below this, callers typically prefer - // to create a fresh row over asking the model to pick among - // weak candidates. - MinAgentScore = 0.6 - - // MaxAgentCandidates caps the candidate list shown to the - // disambiguation agent. The list is heuristic-ranked, so the - // top few are the only ones worth the agent's tokens. - MaxAgentCandidates = 5 -) - -// ScoredCandidate is a scored heuristic-match candidate. It is the -// unified currency between the heuristic ranker (RankCandidates) and -// the disambiguation agent (Disambiguate): the agent renders the -// `ThirdParty` fields plus the score directly into its prompt, with -// no intermediate DTO. -type ScoredCandidate struct { - ThirdParty *coredata.ThirdParty - Score float64 -} - -// corporateSuffixes are the legal-form noise words stripped when -// comparing third-party names heuristically. The list is intentionally -// short and conservative: matching "Foo Inc" to "Foo" is safe, but -// stripping "Group" or "Services" would over-match unrelated entries. -// -// Order matters: stripCorporateSuffixes returns on the first match, -// so longer / comma-prefixed forms must come before their shorter -// siblings (", inc." before " inc.", which itself comes before " inc"). -var corporateSuffixes = []string{ - " incorporated", - " corporation", - ", inc.", - ", inc", - " l.l.c.", - " s.a.s.", - " inc.", - " inc", - " llc", - " ltd.", - " ltd", - " limited", - " gmbh", - " s.a.", - " sas", - " sa", - " ag", - " plc", - " corp.", - " corp", - " co.", - " co", - " b.v.", - " bv", -} - -// RankCandidates ranks org ThirdParty rows by how likely each is to -// represent the given CommonThirdParty. Returned slice is sorted by -// descending score; only candidates with score > 0 are kept. Pure -// function: no I/O, deterministic on its inputs. -// -// Scoring (highest match wins; website-host overlap can lift a name -// miss to 0.8): -// -// - exact lowercase name = 1.0 -// - lowercase name with corporate suffix stripped, equal = 0.9 -// - slug equality (slug.Make on the org's name) = 0.85 -// - website host (eTLD+1) overlap with the catalog domain set = 0.8 -func RankCandidates( - commonParty coredata.CommonThirdParty, - commonDomains coredata.CommonThirdPartyDomains, - candidates coredata.ThirdParties, -) []ScoredCandidate { - commonName := strings.ToLower(strings.TrimSpace(commonParty.Name)) - commonStripped := stripCorporateSuffixes(commonName) - commonSlug := commonParty.Slug - - commonHost := "" - if commonParty.WebsiteURL != nil { - commonHost = uri.ExtractDomain(*commonParty.WebsiteURL) - } - - commonDomainSet := make(map[string]struct{}, len(commonDomains)) - for _, d := range commonDomains { - commonDomainSet[strings.ToLower(d.Domain)] = struct{}{} - } - - if commonHost != "" { - commonDomainSet[commonHost] = struct{}{} - } - - scored := make([]ScoredCandidate, 0, len(candidates)) - - for _, tp := range candidates { - score := 0.0 - - orgName := strings.ToLower(strings.TrimSpace(tp.Name)) - orgStripped := stripCorporateSuffixes(orgName) - - switch { - case orgName != "" && orgName == commonName: - score = 1.0 - case orgStripped != "" && orgStripped == commonStripped: - score = 0.9 - case commonSlug != "" && slug.Make(tp.Name) == commonSlug: - score = 0.85 - } - - if tp.WebsiteURL != nil { - orgHost := uri.ExtractDomain(*tp.WebsiteURL) - if orgHost != "" { - if _, hit := commonDomainSet[orgHost]; hit { - if score < 0.8 { - score = 0.8 - } - } - } - } - - if score == 0 { - continue - } - - scored = append(scored, ScoredCandidate{ - ThirdParty: tp, - Score: score, - }) - } - - sort.SliceStable(scored, func(i, j int) bool { - return scored[i].Score > scored[j].Score - }) - - return scored -} - -// stripCorporateSuffixes removes a single trailing legal-form suffix -// from a lowercased name. Only one suffix is stripped to avoid -// mangling names that happen to end in two stop-words (e.g. "Foo Inc -// LLC" → "Foo Inc", not "Foo"). -func stripCorporateSuffixes(lowerName string) string { - for _, s := range corporateSuffixes { - if before, ok := strings.CutSuffix(lowerName, s); ok { - return strings.TrimSpace(before) - } - } - - return lowerName -} - -// LinkToCommon writes common_third_party_id onto an org ThirdParty so -// future matches against the same CommonThirdParty can short-circuit -// to the exact-link path in O(1). No-op when the field is already set -// (to any value) — we never overwrite an existing catalog link because -// a heuristic or agent false-positive must not corrupt a previous, -// possibly more accurate, association. -func LinkToCommon( - ctx context.Context, - tx pg.Tx, - scope coredata.Scoper, - orgThirdParty *coredata.ThirdParty, - commonID gid.GID, -) error { - if orgThirdParty.CommonThirdPartyID != nil { - return nil - } - - orgThirdParty.CommonThirdPartyID = &commonID - - if err := orgThirdParty.Update(ctx, tx, scope); err != nil { - return fmt.Errorf("cannot update third party with common id: %w", err) - } - - return nil -} - -// CreateFromCommon inserts a new org ThirdParty seeded from the catalog -// row (name, category, addresses, URLs, certifications, …). The new row -// has common_third_party_id pointed at commonParty, an empty Countries -// list, ShowOnTrustCenter false, and Level 1 — the caller has -// already confirmed the vendor is actively present on the -// organization's cookie banner, which makes it a first-level third -// party by definition. -// -// Deliberately bypasses any service-level webhook emission: callers -// that need a webhook for the implicit creation should emit it -// themselves. -func CreateFromCommon( - ctx context.Context, - tx pg.Tx, - scope coredata.Scoper, - organizationID gid.GID, - commonParty coredata.CommonThirdParty, -) (*coredata.ThirdParty, error) { - commonID := commonParty.ID - now := time.Now() - - tp := &coredata.ThirdParty{ - ID: gid.New(scope.GetTenantID(), coredata.ThirdPartyEntityType), - OrganizationID: organizationID, - CommonThirdPartyID: &commonID, - Name: commonParty.Name, - Category: commonParty.Category, - HeadquarterAddress: commonParty.HeadquarterAddress, - LegalName: commonParty.LegalName, - WebsiteURL: commonParty.WebsiteURL, - PrivacyPolicyURL: commonParty.PrivacyPolicyURL, - ServiceLevelAgreementURL: commonParty.ServiceLevelAgreementURL, - DataProcessingAgreementURL: commonParty.DataProcessingAgreementURL, - BusinessAssociateAgreementURL: commonParty.BusinessAssociateAgreementURL, - SubprocessorsListURL: commonParty.SubprocessorsListURL, - Certifications: commonParty.Certifications, - Countries: coredata.CountryCodes{}, - StatusPageURL: commonParty.StatusPageURL, - TermsOfServiceURL: commonParty.TermsOfServiceURL, - SecurityPageURL: commonParty.SecurityPageURL, - TrustPageURL: commonParty.TrustPageURL, - ShowOnTrustCenter: false, - Level: 1, - CreatedAt: now, - UpdatedAt: now, - } - - if tp.Certifications == nil { - tp.Certifications = []string{} - } - - if err := tp.Insert(ctx, tx, scope); err != nil { - return nil, fmt.Errorf("cannot insert org third party: %w", err) - } - - return tp, nil -} diff --git a/pkg/thirdparty/match_test.go b/pkg/thirdparty/match_test.go deleted file mode 100644 index 53bfd7d75..000000000 --- a/pkg/thirdparty/match_test.go +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright (c) 2026 Probo Inc . -// -// Permission to use, copy, modify, and/or distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -// PERFORMANCE OF THIS SOFTWARE. - -package thirdparty - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.probo.inc/probo/pkg/coredata" - "go.probo.inc/probo/pkg/gid" -) - -func TestStripCorporateSuffixes(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - in string - want string - }{ - {name: "llc suffix", in: "google llc", want: "google"}, - {name: "comma inc", in: "stripe, inc", want: "stripe"}, - {name: "inc dot", in: "meta inc.", want: "meta"}, - {name: "ltd", in: "deepmind ltd", want: "deepmind"}, - {name: "gmbh", in: "n8n gmbh", want: "n8n"}, - {name: "no suffix", in: "cloudflare", want: "cloudflare"}, - {name: "trailing space", in: "github inc", want: "github"}, - {name: "only one suffix stripped", in: "foo inc llc", want: "foo inc"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - assert.Equal(t, tt.want, stripCorporateSuffixes(tt.in)) - }) - } -} - -func TestRankCandidates(t *testing.T) { - t.Parallel() - - tenantID := gid.NewTenantID() - - mkTP := func(name, website string) *coredata.ThirdParty { - tp := &coredata.ThirdParty{ - ID: gid.New(tenantID, coredata.ThirdPartyEntityType), - Name: name, - } - - if website != "" { - tp.WebsiteURL = new(website) - } - - return tp - } - - t.Run("exact name match scores 1.0", func(t *testing.T) { - t.Parallel() - - common := coredata.CommonThirdParty{Name: "Google", Slug: "google"} - got := RankCandidates(common, nil, coredata.ThirdParties{ - mkTP("Google", ""), - mkTP("Stripe", ""), - }) - - require.Len(t, got, 1) - assert.Equal(t, 1.0, got[0].Score) - assert.Equal(t, "Google", got[0].ThirdParty.Name) - }) - - t.Run("suffix-stripped name scores 0.9", func(t *testing.T) { - t.Parallel() - - common := coredata.CommonThirdParty{Name: "Google", Slug: "google"} - got := RankCandidates(common, nil, coredata.ThirdParties{ - mkTP("Google LLC", ""), - }) - - require.Len(t, got, 1) - assert.Equal(t, 0.9, got[0].Score) - }) - - t.Run("slug equality scores 0.85", func(t *testing.T) { - t.Parallel() - - common := coredata.CommonThirdParty{Name: "Google", Slug: "google"} - got := RankCandidates(common, nil, coredata.ThirdParties{ - mkTP("google!", ""), - }) - - require.Len(t, got, 1) - assert.Equal(t, 0.85, got[0].Score) - }) - - t.Run("website host overlap scores 0.8 when name does not match", func(t *testing.T) { - t.Parallel() - - common := coredata.CommonThirdParty{ - Name: "Google Analytics", - Slug: "google-analytics", - WebsiteURL: new("https://google.com"), - } - - got := RankCandidates(common, nil, coredata.ThirdParties{ - mkTP("Sundar's Search Co", "https://www.google.com/about"), - }) - - require.Len(t, got, 1) - assert.Equal(t, 0.8, got[0].Score) - }) - - t.Run("domain set overlap scores 0.8", func(t *testing.T) { - t.Parallel() - - common := coredata.CommonThirdParty{Name: "Stripe", Slug: "stripe"} - domains := coredata.CommonThirdPartyDomains{ - {Domain: "stripe.com"}, - {Domain: "stripe.network"}, - } - - got := RankCandidates(common, domains, coredata.ThirdParties{ - mkTP("Payment Processor", "https://api.stripe.com/v1"), - }) - - require.Len(t, got, 1) - assert.Equal(t, 0.8, got[0].Score) - }) - - t.Run("no match returns empty", func(t *testing.T) { - t.Parallel() - - common := coredata.CommonThirdParty{Name: "Stripe", Slug: "stripe"} - got := RankCandidates(common, nil, coredata.ThirdParties{ - mkTP("Acme", "https://acme.example"), - mkTP("Widgets Inc", "https://widgets.example"), - }) - - assert.Empty(t, got) - }) - - t.Run("ranks descending by score", func(t *testing.T) { - t.Parallel() - - common := coredata.CommonThirdParty{ - Name: "Google", - Slug: "google", - WebsiteURL: new("https://google.com"), - } - - got := RankCandidates(common, nil, coredata.ThirdParties{ - mkTP("Random", "https://google.com"), - mkTP("Google", ""), - mkTP("Google LLC", ""), - }) - - require.Len(t, got, 3) - assert.Equal(t, "Google", got[0].ThirdParty.Name) - assert.Equal(t, 1.0, got[0].Score) - assert.Equal(t, "Google LLC", got[1].ThirdParty.Name) - assert.Equal(t, 0.9, got[1].Score) - assert.Equal(t, "Random", got[2].ThirdParty.Name) - assert.Equal(t, 0.8, got[2].Score) - }) -}