rohit/probo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Drop request sanitization from LLM client

Browse Source 
Remove sanitizeRequest and its call sites so the client no longer
strips sampling parameters based on the model registry. Unsupported
knobs are now avoided by setting the config carefully per model
rather than silently mutating outgoing requests.

Signed-off-by: Émile Ré <emile@probo.com>
This commit is contained in:
Émile Ré
2026-06-01 13:39:42 +02:00
parent 387c92d045
commit 8cf770613a
1 changed files with 0 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
pkg/llm/llm.go
View File

@@ -17,7 +17,6 @@ package llm
import ( import (
"context" "context"
"io" "io"
"strings"
"time" "time"
"go.gearno.de/kit/log" "go.gearno.de/kit/log"
@@ -73,8 +72,6 @@ func NewClient(provider Provider, system string, opts ...Option) *Client {
} }
func (c *Client) ChatCompletion(ctx context.Context, req *ChatCompletionRequest) (*ChatCompletionResponse, error) { func (c *Client) ChatCompletion(ctx context.Context, req *ChatCompletionRequest) (*ChatCompletionResponse, error) {
req = sanitizeRequest(ctx, c.logger, req)
ctx, span := startChatSpan(ctx, c.tracer, c.system, req) ctx, span := startChatSpan(ctx, c.tracer, c.system, req)
c.logger.InfoCtx( c.logger.InfoCtx(
@@ -118,8 +115,6 @@ func (c *Client) ChatCompletion(ctx context.Context, req *ChatCompletionRequest)
} }
func (c *Client) ChatCompletionStream(ctx context.Context, req *ChatCompletionRequest) (ChatCompletionStream, error) { func (c *Client) ChatCompletionStream(ctx context.Context, req *ChatCompletionRequest) (ChatCompletionStream, error) {
req = sanitizeRequest(ctx, c.logger, req)
ctx, span := startChatSpan(ctx, c.tracer, c.system, req) ctx, span := startChatSpan(ctx, c.tracer, c.system, req)
c.logger.InfoCtx( c.logger.InfoCtx(
@@ -145,75 +140,3 @@ func (c *Client) ChatCompletionStream(ctx context.Context, req *ChatCompletionRe
return newTracedStream(stream, span), nil return newTracedStream(stream, span), nil
} }
// sanitizeRequest drops sampling parameters the target model does not
// accept, using the default model registry. Reasoning models (e.g. the
// GPT-5 family) reject an explicit temperature, top_p, or penalties and
// fail the whole request with a 400, so the safest behavior is to omit
// the unsupported knobs rather than propagate a hard error. When the
// model is unknown to the registry the request is left untouched.
func sanitizeRequest(ctx context.Context, logger *log.Logger, req *ChatCompletionRequest) *ChatCompletionRequest {
if req == nil {
return req
}
model, ok := DefaultRegistry().Lookup(req.Model)
if !ok {
return req
}
supports := model.Supports
dropTemperature := req.Temperature != nil && !supports.Temperature
dropTopP := req.TopP != nil && !supports.TopP
dropFrequencyPenalty := req.FrequencyPenalty != nil && !supports.FrequencyPenalty
dropPresencePenalty := req.PresencePenalty != nil && !supports.PresencePenalty
dropStop := len(req.StopSequences) > 0 && !supports.Stop
if !dropTemperature && !dropTopP && !dropFrequencyPenalty && !dropPresencePenalty && !dropStop {
return req
}
sanitized := *req
dropped := make([]string, 0, 5)
if dropTemperature {
sanitized.Temperature = nil
dropped = append(dropped, "temperature")
}
if dropTopP {
sanitized.TopP = nil
dropped = append(dropped, "top_p")
}
if dropFrequencyPenalty {
sanitized.FrequencyPenalty = nil
dropped = append(dropped, "frequency_penalty")
}
if dropPresencePenalty {
sanitized.PresencePenalty = nil
dropped = append(dropped, "presence_penalty")
}
if dropStop {
sanitized.StopSequences = nil
dropped = append(dropped, "stop")
}
logger.WarnCtx(
ctx,
"dropping unsupported sampling parameters for model",
log.String("model", req.Model),
log.String("dropped", strings.Join(dropped, ",")),
)
return &sanitized
}