Add vendor assessment agent

Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com>
2026-04-22 22:36:14 +02:00
parent 25c590ffe6
commit 509d0c88b1
108 changed files with 9445 additions and 645 deletions
--- a/pkg/probod/evidence_describer_config.go
+++ b/pkg/probod/evidence_describer_config.go
@@ -1,38 +0,0 @@
-// Copyright (c) 2025-2026 Probo Inc <hello@getprobo.com>.
-//
-// Permission to use, copy, modify, and/or distribute this software for any
-// purpose with or without fee is hereby granted, provided that the above
-// copyright notice and this permission notice appear in all copies.
-//
-// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
-// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
-// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
-// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-// PERFORMANCE OF THIS SOFTWARE.
-
-package probod
-
-// EvidenceDescriberConfig holds both the worker settings and LLM overrides
-// for the evidence description worker.
-type EvidenceDescriberConfig struct {
-	Interval       int `json:"interval"`    // seconds
-	StaleAfter     int `json:"stale-after"` // seconds
-	MaxConcurrency int `json:"max-concurrency"`
-
-	Provider    string   `json:"provider"`
-	ModelName   string   `json:"model-name"`
-	Temperature *float64 `json:"temperature"`
-	MaxTokens   *int     `json:"max-tokens"`
-}
-
-// LLMConfig extracts the LLM-specific fields as an LLMConfig.
-func (c *EvidenceDescriberConfig) LLMConfig() LLMConfig {
-	return LLMConfig{
-		Provider:    c.Provider,
-		ModelName:   c.ModelName,
-		Temperature: c.Temperature,
-		MaxTokens:   c.MaxTokens,
-	}
-}
--- a/pkg/probod/llm.go
+++ b/pkg/probod/llm.go
@@ -26,6 +26,28 @@ import (
 	llmopenai "go.probo.inc/probo/pkg/llm/openai"
 )

+// resolveAgentClient resolves the agent's effective config from defaults and
+// builds an LLM client for it. The name parameter is used in the logger and
+// in error messages.
+func (impl *Implm) resolveAgentClient(
+	name string,
+	agent LLMAgentConfig,
+	l *log.Logger,
+	tp trace.TracerProvider,
+	r prometheus.Registerer,
+) (LLMAgentConfig, *llm.Client, error) {
+	resolved := impl.cfg.Agents.ResolveAgent(agent)
+	providerCfg, ok := impl.cfg.Agents.Providers[resolved.Provider]
+	if !ok {
+		return LLMAgentConfig{}, nil, fmt.Errorf("unknown LLM provider %q for %s agent", resolved.Provider, name)
+	}
+	client, err := buildLLMClient(providerCfg, l.Named("llm."+name), tp, r)
+	if err != nil {
+		return LLMAgentConfig{}, nil, fmt.Errorf("cannot create %s LLM client: %w", name, err)
+	}
+	return resolved, client, nil
+}
+
 func buildLLMClient(cfg LLMProviderConfig, l *log.Logger, tp trace.TracerProvider, r prometheus.Registerer) (*llm.Client, error) {
 	providerType := cfg.Type
 	if providerType == "" {
--- a/pkg/probod/llm_config.go
+++ b/pkg/probod/llm_config.go
@@ -22,38 +22,50 @@ type (
 		APIKey string `json:"api-key"` // for OpenAI and Anthropic
 	}

-	// LLMConfig holds model parameters for a single LLM consumer. Provider
-	// references one of the keys in LLMSettings.Providers.
-	LLMConfig struct {
-		Provider    string   `json:"provider"` // key into LLMSettings.Providers
+	// LLMAgentConfig holds model parameters for a single agent. Provider
+	// references one of the keys in AgentsConfig.Providers.
+	LLMAgentConfig struct {
+		Provider    string   `json:"provider"` // key into AgentsConfig.Providers
 		ModelName   string   `json:"model-name"`
 		Temperature *float64 `json:"temperature"`
 		MaxTokens   *int     `json:"max-tokens"`
 	}

-	// LLMSettings groups LLM provider credentials and default model
-	// settings. Defaults is used as a fallback when a consumer-specific
-	// field is zero-valued.
-	LLMSettings struct {
-		Providers map[string]LLMProviderConfig `json:"providers"`
-		Defaults  LLMConfig                    `json:"defaults"`
+	// EvidenceDescriberConfig holds worker-side tuning for the evidence
+	// description background worker. LLM parameters for the same worker
+	// live under AgentsConfig.EvidenceDescriber.
+	EvidenceDescriberConfig struct {
+		Interval       int `json:"interval"`    // seconds between polls
+		StaleAfter     int `json:"stale-after"` // seconds before a claim is recycled
+		MaxConcurrency int `json:"max-concurrency"`
+	}
+
+	// AgentsConfig groups LLM provider credentials and per-agent model
+	// settings. Default is used as a fallback when an agent-specific field
+	// is zero-valued.
+	AgentsConfig struct {
+		Providers         map[string]LLMProviderConfig `json:"providers"`
+		Default           LLMAgentConfig               `json:"defaults"`
+		Probo             LLMAgentConfig               `json:"probo"`
+		EvidenceDescriber LLMAgentConfig               `json:"evidence-describer"`
+		VendorAssessor    LLMAgentConfig               `json:"vendor-assessor"`
 	}
 )

-// ResolveLLMConfig returns a fully populated LLMConfig by filling in
-// zero-valued fields from the defaults.
-func (s *LLMSettings) ResolveLLMConfig(cfg LLMConfig) LLMConfig {
-	if cfg.Provider == "" {
-		cfg.Provider = s.Defaults.Provider
+// ResolveAgent returns a fully populated LLMAgentConfig by filling in
+// zero-valued fields from the default config.
+func (c *AgentsConfig) ResolveAgent(agent LLMAgentConfig) LLMAgentConfig {
+	if agent.Provider == "" {
+		agent.Provider = c.Default.Provider
 	}
-	if cfg.ModelName == "" {
-		cfg.ModelName = s.Defaults.ModelName
+	if agent.ModelName == "" {
+		agent.ModelName = c.Default.ModelName
 	}
-	if cfg.Temperature == nil {
-		cfg.Temperature = s.Defaults.Temperature
+	if agent.Temperature == nil {
+		agent.Temperature = c.Default.Temperature
 	}
-	if cfg.MaxTokens == nil {
-		cfg.MaxTokens = s.Defaults.MaxTokens
+	if agent.MaxTokens == nil {
+		agent.MaxTokens = c.Default.MaxTokens
 	}
-	return cfg
+	return agent
 }
--- a/pkg/probod/probod.go
+++ b/pkg/probod/probod.go
@@ -121,10 +121,10 @@ type (
 		AWS               AWSConfig               `json:"aws"`
 		Notifications     NotificationsConfig     `json:"notifications"`
 		Connectors        []ConnectorConfig       `json:"connectors"`
-		LLM               LLMSettings             `json:"llm"`
-		ProboAgent        LLMConfig               `json:"probo-agent"`
+		Agents            AgentsConfig            `json:"llm"`
 		EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"`
 		ChromeDPAddr      string                  `json:"chrome-dp-addr"`
+		SearchEndpoint    string                  `json:"search-endpoint"`
 		CustomDomains     CustomDomainsConfig     `json:"custom-domains"`
 		SCIMBridge        SCIMBridgeConfig        `json:"scim-bridge"`
 		ESign             ESignConfig             `json:"esign"`
@@ -338,24 +338,19 @@ func (impl *Implm) Run(
 		}
 	}

-	proboAgentCfg := impl.cfg.LLM.ResolveLLMConfig(impl.cfg.ProboAgent)
-	proboProviderCfg, ok := impl.cfg.LLM.Providers[proboAgentCfg.Provider]
-	if !ok {
-		return fmt.Errorf("unknown LLM provider %q for probo agent", proboAgentCfg.Provider)
-	}
-	proboLLMClient, err := buildLLMClient(proboProviderCfg, l.Named("llm.probo"), tp, r)
+	proboAgentCfg, proboLLMClient, err := impl.resolveAgentClient("probo", impl.cfg.Agents.Probo, l, tp, r)
 	if err != nil {
-		return fmt.Errorf("cannot create probo LLM client: %w", err)
+		return err
 	}

-	edLLMCfg := impl.cfg.LLM.ResolveLLMConfig(impl.cfg.EvidenceDescriber.LLMConfig())
-	edProviderCfg, ok := impl.cfg.LLM.Providers[edLLMCfg.Provider]
-	if !ok {
-		return fmt.Errorf("unknown LLM provider %q for evidence-describer agent", edLLMCfg.Provider)
-	}
-	evidenceDescriberLLMClient, err := buildLLMClient(edProviderCfg, l.Named("llm.evidence-describer"), tp, r)
+	evidenceDescriberAgentCfg, evidenceDescriberLLMClient, err := impl.resolveAgentClient("evidence-describer", impl.cfg.Agents.EvidenceDescriber, l, tp, r)
 	if err != nil {
-		return fmt.Errorf("cannot create evidence describer LLM client: %w", err)
+		return err
+	}
+
+	vendorAssessor, err := impl.buildVendorAssessor(l, tp, r)
+	if err != nil {
+		return err
 	}

 	fileManagerService := filemanager.NewService(s3Client)
@@ -545,6 +540,7 @@ func (impl *Implm) Run(
 		esignService,
 		defaultConnectorRegistry,
 		time.Duration(impl.cfg.Auth.InvitationConfirmationTokenValidity)*time.Second,
+		vendorAssessor,
 	)
 	if err != nil {
 		return fmt.Errorf("cannot create probo service: %w", err)
@@ -735,9 +731,9 @@ func (impl *Implm) Run(
 	evidenceDescriber := evidencedescriber.New(
 		evidenceDescriberLLMClient,
 		evidencedescriber.Config{
-			Model:     edLLMCfg.ModelName,
-			Temp:      *edLLMCfg.Temperature,
-			MaxTokens: *edLLMCfg.MaxTokens,
+			Model:     evidenceDescriberAgentCfg.ModelName,
+			Temp:      *evidenceDescriberAgentCfg.Temperature,
+			MaxTokens: *evidenceDescriberAgentCfg.MaxTokens,
 		},
 	)
 	evidenceDescriptionWorker := probo.NewEvidenceDescriptionWorker(
--- a/pkg/probod/vendor_assessor.go
+++ b/pkg/probod/vendor_assessor.go
@@ -0,0 +1,58 @@
+// Copyright (c) 2026 Probo Inc <hello@getprobo.com>.
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+package probod
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"go.gearno.de/kit/log"
+	"go.opentelemetry.io/otel/trace"
+	"go.probo.inc/probo/pkg/probo"
+	"go.probo.inc/probo/pkg/vetting"
+)
+
+// buildVendorAssessor wires the vendor assessment agent. It is an opt-in
+// feature: deployments that do not set `llm.vendor-assessor.provider` get a
+// DisabledVendorAssessor that reports the feature as unavailable. The
+// vendor-assessor does not inherit the default provider because its
+// pipeline (LLM + browser + search) is expensive and should not be enabled
+// implicitly.
+func (impl *Implm) buildVendorAssessor(
+	l *log.Logger,
+	tp trace.TracerProvider,
+	r prometheus.Registerer,
+) (probo.VendorAssessor, error) {
+	if impl.cfg.Agents.VendorAssessor.Provider == "" {
+		return probo.DisabledVendorAssessor{}, nil
+	}
+
+	agentCfg, llmClient, err := impl.resolveAgentClient("vendor-assessor", impl.cfg.Agents.VendorAssessor, l, tp, r)
+	if err != nil {
+		return nil, err
+	}
+
+	maxTokens := vetting.DefaultMaxTokens
+	if agentCfg.MaxTokens != nil {
+		maxTokens = *agentCfg.MaxTokens
+	}
+
+	return vetting.NewAssessor(vetting.Config{
+		Client:         llmClient,
+		Model:          agentCfg.ModelName,
+		MaxTokens:      maxTokens,
+		ChromeAddr:     impl.cfg.ChromeDPAddr,
+		SearchEndpoint: impl.cfg.SearchEndpoint,
+		Logger:         l.Named("vendor-assessor"),
+	}), nil
+}