@@ -76,17 +76,21 @@ func NewProvider(apiKey string, opts ...Option) *Provider {
|
||||
if cfg.httpClient != nil {
|
||||
reqOpts = append(reqOpts, option.WithHTTPClient(cfg.httpClient))
|
||||
}
|
||||
|
||||
if cfg.baseURL != "" {
|
||||
reqOpts = append(reqOpts, option.WithBaseURL(cfg.baseURL))
|
||||
}
|
||||
|
||||
if cfg.requestTimeout > 0 {
|
||||
reqOpts = append(reqOpts, option.WithRequestTimeout(cfg.requestTimeout))
|
||||
}
|
||||
|
||||
if cfg.maxRetries != nil {
|
||||
reqOpts = append(reqOpts, option.WithMaxRetries(*cfg.maxRetries))
|
||||
}
|
||||
|
||||
client := anthropic.NewClient(reqOpts...)
|
||||
|
||||
return &Provider{client: &client}
|
||||
}
|
||||
|
||||
@@ -111,6 +115,7 @@ func (p *Provider) ChatCompletionStream(ctx context.Context, req *llm.ChatComple
|
||||
}
|
||||
|
||||
stream := p.client.Messages.NewStreaming(ctx, params)
|
||||
|
||||
return &anthropicStream{stream: stream}, nil
|
||||
}
|
||||
|
||||
@@ -134,37 +139,46 @@ func buildParams(req *llm.ChatCompletionRequest) (anthropic.MessageNewParams, er
|
||||
for i, s := range system {
|
||||
blocks[i] = anthropic.TextBlockParam{Text: s}
|
||||
}
|
||||
|
||||
params.System = blocks
|
||||
}
|
||||
|
||||
if req.Temperature != nil {
|
||||
params.Temperature = param.NewOpt(*req.Temperature)
|
||||
}
|
||||
|
||||
if req.TopP != nil {
|
||||
params.TopP = param.NewOpt(*req.TopP)
|
||||
}
|
||||
|
||||
if len(req.StopSequences) > 0 {
|
||||
params.StopSequences = req.StopSequences
|
||||
}
|
||||
|
||||
if len(req.Tools) > 0 {
|
||||
params.Tools = buildTools(req.Tools)
|
||||
}
|
||||
|
||||
if req.ToolChoice != nil {
|
||||
params.ToolChoice = buildToolChoice(req.ToolChoice)
|
||||
}
|
||||
|
||||
if req.Thinking != nil && req.Thinking.Enabled {
|
||||
params.Thinking = anthropic.ThinkingConfigParamOfEnabled(int64(req.Thinking.BudgetTokens))
|
||||
}
|
||||
|
||||
if req.ResponseFormat != nil {
|
||||
switch req.ResponseFormat.Type {
|
||||
case llm.ResponseFormatJSONSchema:
|
||||
if req.ResponseFormat.JSONSchema == nil {
|
||||
return anthropic.MessageNewParams{}, fmt.Errorf("cannot apply JSON schema output format: schema is nil")
|
||||
}
|
||||
|
||||
var schema map[string]any
|
||||
if err := json.Unmarshal(req.ResponseFormat.JSONSchema.Schema, &schema); err != nil {
|
||||
return anthropic.MessageNewParams{}, fmt.Errorf("cannot unmarshal JSON schema for output format: %w", err)
|
||||
}
|
||||
|
||||
params.OutputConfig = anthropic.OutputConfigParam{
|
||||
Format: anthropic.JSONOutputFormatParam{Schema: schema},
|
||||
}
|
||||
@@ -186,6 +200,7 @@ func extractSystem(messages []llm.Message) (system []string, rest []llm.Message)
|
||||
rest = append(rest, msg)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -213,9 +228,11 @@ func buildMessages(messages []llm.Message) []anthropic.MessageParam {
|
||||
blocks = append(blocks, buildFilePart(p))
|
||||
}
|
||||
}
|
||||
|
||||
out = append(out, anthropic.NewUserMessage(blocks...))
|
||||
case llm.RoleAssistant:
|
||||
var blocks []anthropic.ContentBlockParamUnion
|
||||
|
||||
for _, p := range msg.Parts {
|
||||
switch part := p.(type) {
|
||||
case llm.ThinkingPart:
|
||||
@@ -226,13 +243,16 @@ func buildMessages(messages []llm.Message) []anthropic.MessageParam {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, tc := range msg.ToolCalls {
|
||||
var input any
|
||||
if err := json.Unmarshal([]byte(tc.Function.Arguments), &input); err != nil || input == nil {
|
||||
input = map[string]any{}
|
||||
}
|
||||
|
||||
blocks = append(blocks, anthropic.NewToolUseBlock(tc.ID, input, tc.Function.Name))
|
||||
}
|
||||
|
||||
out = append(out, anthropic.NewAssistantMessage(blocks...))
|
||||
case llm.RoleTool:
|
||||
out = append(
|
||||
@@ -264,6 +284,7 @@ func buildTools(tools []llm.Tool) []anthropic.ToolUnionParam {
|
||||
if err := json.Unmarshal(t.Parameters, &schema); err == nil {
|
||||
props := schema["properties"]
|
||||
required, _ := schema["required"].([]any)
|
||||
|
||||
reqStrings := make([]string, 0, len(required))
|
||||
for _, r := range required {
|
||||
if s, ok := r.(string); ok {
|
||||
@@ -272,6 +293,7 @@ func buildTools(tools []llm.Tool) []anthropic.ToolUnionParam {
|
||||
}
|
||||
|
||||
extra := make(map[string]any)
|
||||
|
||||
for k, v := range schema {
|
||||
switch k {
|
||||
case "type", "properties", "required":
|
||||
@@ -287,12 +309,14 @@ func buildTools(tools []llm.Tool) []anthropic.ToolUnionParam {
|
||||
if len(extra) > 0 {
|
||||
inputSchema.ExtraFields = extra
|
||||
}
|
||||
|
||||
tool.InputSchema = inputSchema
|
||||
}
|
||||
}
|
||||
|
||||
out[i] = anthropic.ToolUnionParam{OfTool: &tool}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -392,13 +416,16 @@ func parseRetryAfter(resp *http.Response) time.Duration {
|
||||
if resp == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
h := resp.Header.Get("Retry-After")
|
||||
if h == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
if secs, err := strconv.Atoi(h); err == nil {
|
||||
return time.Duration(secs) * time.Second
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -415,12 +442,14 @@ type anthropicStream struct {
|
||||
func (s *anthropicStream) Next() bool {
|
||||
for s.stream.Next() {
|
||||
event := s.stream.Current()
|
||||
|
||||
mapped, ok := s.mapStreamEvent(&event)
|
||||
if ok {
|
||||
s.current = mapped
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -433,6 +462,7 @@ func (s *anthropicStream) Err() error {
|
||||
if err != nil {
|
||||
return mapError(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -448,6 +478,7 @@ func (s *anthropicStream) mapStreamEvent(event *anthropic.MessageStreamEventUnio
|
||||
case "tool_use":
|
||||
s.inToolUse = true
|
||||
tu := cb.AsToolUse()
|
||||
|
||||
return llm.ChatCompletionStreamEvent{
|
||||
Delta: llm.MessageDelta{
|
||||
ToolCalls: []llm.ToolCallDelta{{
|
||||
@@ -460,6 +491,7 @@ func (s *anthropicStream) mapStreamEvent(event *anthropic.MessageStreamEventUnio
|
||||
case "thinking":
|
||||
return llm.ChatCompletionStreamEvent{}, false
|
||||
}
|
||||
|
||||
return llm.ChatCompletionStreamEvent{}, false
|
||||
|
||||
case "content_block_delta":
|
||||
@@ -475,6 +507,7 @@ func (s *anthropicStream) mapStreamEvent(event *anthropic.MessageStreamEventUnio
|
||||
}, true
|
||||
case "signature_delta":
|
||||
s.thinkingSignature = delta.Signature
|
||||
|
||||
return llm.ChatCompletionStreamEvent{
|
||||
Delta: llm.MessageDelta{ThinkingSignature: delta.Signature},
|
||||
}, true
|
||||
@@ -488,6 +521,7 @@ func (s *anthropicStream) mapStreamEvent(event *anthropic.MessageStreamEventUnio
|
||||
},
|
||||
}, true
|
||||
}
|
||||
|
||||
return llm.ChatCompletionStreamEvent{}, false
|
||||
|
||||
case "content_block_stop":
|
||||
@@ -495,10 +529,12 @@ func (s *anthropicStream) mapStreamEvent(event *anthropic.MessageStreamEventUnio
|
||||
s.toolCallIndex++
|
||||
s.inToolUse = false
|
||||
}
|
||||
|
||||
return llm.ChatCompletionStreamEvent{}, false
|
||||
|
||||
case "message_delta":
|
||||
fr := mapStopReason(anthropic.StopReason(event.Delta.StopReason))
|
||||
|
||||
evt := llm.ChatCompletionStreamEvent{
|
||||
FinishReason: &fr,
|
||||
}
|
||||
@@ -508,6 +544,7 @@ func (s *anthropicStream) mapStreamEvent(event *anthropic.MessageStreamEventUnio
|
||||
OutputTokens: int(event.Usage.OutputTokens),
|
||||
}
|
||||
}
|
||||
|
||||
return evt, true
|
||||
|
||||
case "message_start":
|
||||
@@ -520,6 +557,7 @@ func (s *anthropicStream) mapStreamEvent(event *anthropic.MessageStreamEventUnio
|
||||
OutputTokens: int(event.Message.Usage.OutputTokens),
|
||||
}
|
||||
}
|
||||
|
||||
return evt, true
|
||||
|
||||
default:
|
||||
@@ -540,6 +578,7 @@ func buildFilePart(p llm.FilePart) anthropic.ContentBlockParamUnion {
|
||||
if err != nil {
|
||||
return anthropic.NewTextBlock(fmt.Sprintf("[file: %s, type: %s, error decoding content]", p.Filename, p.MimeType))
|
||||
}
|
||||
|
||||
return anthropic.NewDocumentBlock(anthropic.PlainTextSourceParam{
|
||||
Data: string(decoded),
|
||||
})
|
||||
|
||||
@@ -48,6 +48,7 @@ func NewProvider(cfg aws.Config, opts ...Option) *Provider {
|
||||
}
|
||||
|
||||
client := bedrockruntime.NewFromConfig(cfg, fns...)
|
||||
|
||||
return &Provider{client: client}
|
||||
}
|
||||
|
||||
@@ -112,14 +113,17 @@ func buildInferenceConfig(req *llm.ChatCompletionRequest) *types.InferenceConfig
|
||||
v := int32(*req.MaxTokens)
|
||||
cfg.MaxTokens = &v
|
||||
}
|
||||
|
||||
if req.Temperature != nil {
|
||||
v := float32(*req.Temperature)
|
||||
cfg.Temperature = &v
|
||||
}
|
||||
|
||||
if req.TopP != nil {
|
||||
v := float32(*req.TopP)
|
||||
cfg.TopP = &v
|
||||
}
|
||||
|
||||
if len(req.StopSequences) > 0 {
|
||||
cfg.StopSequences = req.StopSequences
|
||||
}
|
||||
@@ -129,6 +133,7 @@ func buildInferenceConfig(req *llm.ChatCompletionRequest) *types.InferenceConfig
|
||||
|
||||
func buildSystem(messages []llm.Message) []types.SystemContentBlock {
|
||||
var system []types.SystemContentBlock
|
||||
|
||||
for _, msg := range messages {
|
||||
if msg.Role == llm.RoleSystem {
|
||||
system = append(
|
||||
@@ -139,6 +144,7 @@ func buildSystem(messages []llm.Message) []types.SystemContentBlock {
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return system
|
||||
}
|
||||
|
||||
@@ -151,11 +157,13 @@ func buildMessages(messages []llm.Message) []types.Message {
|
||||
continue
|
||||
case llm.RoleUser:
|
||||
var content []types.ContentBlock
|
||||
|
||||
for _, p := range msg.Parts {
|
||||
if tp, ok := p.(llm.TextPart); ok {
|
||||
content = append(content, &types.ContentBlockMemberText{Value: tp.Text})
|
||||
}
|
||||
}
|
||||
|
||||
out = append(
|
||||
out, types.Message{
|
||||
Role: types.ConversationRoleUser,
|
||||
@@ -168,8 +176,10 @@ func buildMessages(messages []llm.Message) []types.Message {
|
||||
if text := msg.Text(); text != "" {
|
||||
content = append(content, &types.ContentBlockMemberText{Value: text})
|
||||
}
|
||||
|
||||
for _, tc := range msg.ToolCalls {
|
||||
var input any
|
||||
|
||||
_ = json.Unmarshal([]byte(tc.Function.Arguments), &input)
|
||||
content = append(
|
||||
content,
|
||||
@@ -182,6 +192,7 @@ func buildMessages(messages []llm.Message) []types.Message {
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
out = append(
|
||||
out, types.Message{
|
||||
Role: types.ConversationRoleAssistant,
|
||||
@@ -220,13 +231,16 @@ func buildToolConfig(req *llm.ChatCompletionRequest) *types.ToolConfiguration {
|
||||
}
|
||||
if t.Parameters != nil {
|
||||
var schema any
|
||||
|
||||
_ = json.Unmarshal(t.Parameters, &schema)
|
||||
spec.InputSchema = &types.ToolInputSchemaMemberJson{
|
||||
Value: document.NewLazyDocument(schema),
|
||||
}
|
||||
}
|
||||
|
||||
tools[i] = &types.ToolMemberToolSpec{Value: spec}
|
||||
}
|
||||
|
||||
config.Tools = tools
|
||||
|
||||
if req.ToolChoice != nil {
|
||||
@@ -283,6 +297,7 @@ func mapResponse(output *bedrockruntime.ConverseOutput, model string) *llm.ChatC
|
||||
if b.Value.Input != nil {
|
||||
_ = b.Value.Input.UnmarshalSmithyDocument(&args)
|
||||
}
|
||||
|
||||
argsJSON, _ := json.Marshal(args)
|
||||
resp.Message.ToolCalls = append(resp.Message.ToolCalls, llm.ToolCall{
|
||||
ID: aws.ToString(b.Value.ToolUseId),
|
||||
@@ -321,6 +336,7 @@ func mapError(err error) error {
|
||||
if strings.Contains(msg, "throttling") || strings.Contains(msg, "ThrottlingException") {
|
||||
return &llm.ErrRateLimit{Err: err}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -334,6 +350,7 @@ func mapError(err error) error {
|
||||
if strings.Contains(msg, "context") || strings.Contains(msg, "token") {
|
||||
return &llm.ErrContextLength{Err: err}
|
||||
}
|
||||
|
||||
return err
|
||||
default:
|
||||
return err
|
||||
@@ -368,7 +385,9 @@ func (s *bedrockStream) Next() bool {
|
||||
mapped.Model = s.model
|
||||
s.modelSent = true
|
||||
}
|
||||
|
||||
s.current = mapped
|
||||
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -376,6 +395,7 @@ func (s *bedrockStream) Next() bool {
|
||||
if err := s.eventStream.Err(); err != nil {
|
||||
s.err = mapError(err)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -396,6 +416,7 @@ func (s *bedrockStream) mapEvent(event types.ConverseStreamOutput) (llm.ChatComp
|
||||
case *types.ConverseStreamOutputMemberContentBlockStart:
|
||||
if start, ok := e.Value.Start.(*types.ContentBlockStartMemberToolUse); ok {
|
||||
s.inToolUse = true
|
||||
|
||||
return llm.ChatCompletionStreamEvent{
|
||||
Delta: llm.MessageDelta{
|
||||
ToolCalls: []llm.ToolCallDelta{{
|
||||
@@ -406,7 +427,9 @@ func (s *bedrockStream) mapEvent(event types.ConverseStreamOutput) (llm.ChatComp
|
||||
},
|
||||
}, true
|
||||
}
|
||||
|
||||
s.inToolUse = false
|
||||
|
||||
return llm.ChatCompletionStreamEvent{}, false
|
||||
|
||||
case *types.ConverseStreamOutputMemberContentBlockDelta:
|
||||
@@ -425,6 +448,7 @@ func (s *bedrockStream) mapEvent(event types.ConverseStreamOutput) (llm.ChatComp
|
||||
},
|
||||
}, true
|
||||
}
|
||||
|
||||
return llm.ChatCompletionStreamEvent{}, false
|
||||
|
||||
case *types.ConverseStreamOutputMemberContentBlockStop:
|
||||
@@ -432,10 +456,12 @@ func (s *bedrockStream) mapEvent(event types.ConverseStreamOutput) (llm.ChatComp
|
||||
s.toolIndex++
|
||||
s.inToolUse = false
|
||||
}
|
||||
|
||||
return llm.ChatCompletionStreamEvent{}, false
|
||||
|
||||
case *types.ConverseStreamOutputMemberMessageStop:
|
||||
fr := mapStopReason(e.Value.StopReason)
|
||||
|
||||
return llm.ChatCompletionStreamEvent{
|
||||
FinishReason: &fr,
|
||||
}, true
|
||||
@@ -449,6 +475,7 @@ func (s *bedrockStream) mapEvent(event types.ConverseStreamOutput) (llm.ChatComp
|
||||
},
|
||||
}, true
|
||||
}
|
||||
|
||||
return llm.ChatCompletionStreamEvent{}, false
|
||||
|
||||
default:
|
||||
|
||||
@@ -211,6 +211,7 @@ func (a *StreamAccumulator) Response() *ChatCompletionResponse {
|
||||
Signature: a.thinkingSignature,
|
||||
})
|
||||
}
|
||||
|
||||
parts = append(parts, TextPart{Text: a.content.String()})
|
||||
|
||||
return &ChatCompletionResponse{
|
||||
@@ -232,6 +233,7 @@ func (a *StreamAccumulator) accumulate(event ChatCompletionStreamEvent) {
|
||||
|
||||
a.content.WriteString(event.Delta.Content)
|
||||
a.thinking.WriteString(event.Delta.Thinking)
|
||||
|
||||
if event.Delta.ThinkingSignature != "" {
|
||||
a.thinkingSignature = event.Delta.ThinkingSignature
|
||||
}
|
||||
@@ -246,15 +248,18 @@ func (a *StreamAccumulator) accumulate(event ChatCompletionStreamEvent) {
|
||||
if tcd.ID != "" {
|
||||
tc.ID = tcd.ID
|
||||
}
|
||||
|
||||
if tcd.Name != "" {
|
||||
tc.Function.Name = tcd.Name
|
||||
}
|
||||
|
||||
tc.Function.Arguments += tcd.Arguments
|
||||
}
|
||||
|
||||
if event.Usage != nil {
|
||||
a.usage = *event.Usage
|
||||
}
|
||||
|
||||
if event.FinishReason != nil {
|
||||
a.finishReason = *event.FinishReason
|
||||
}
|
||||
|
||||
@@ -50,6 +50,7 @@ func (e *ErrRateLimit) Error() string {
|
||||
if e.RetryAfter > 0 {
|
||||
return fmt.Sprintf("rate limited (retry after %s): %v", e.RetryAfter, e.Err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("rate limited: %v", e.Err)
|
||||
}
|
||||
|
||||
@@ -59,6 +60,7 @@ func (e *ErrContextLength) Error() string {
|
||||
if e.MaxTokens > 0 {
|
||||
return fmt.Sprintf("context length exceeded (max %d tokens): %v", e.MaxTokens, e.Err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("context length exceeded: %v", e.Err)
|
||||
}
|
||||
|
||||
|
||||
@@ -95,6 +95,7 @@ func (c *Client) ChatCompletion(ctx context.Context, req *ChatCompletionRequest)
|
||||
log.Error(err),
|
||||
)
|
||||
endChatSpan(span, nil, err)
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -109,6 +110,7 @@ func (c *Client) ChatCompletion(ctx context.Context, req *ChatCompletionRequest)
|
||||
)
|
||||
|
||||
endChatSpan(span, resp, nil)
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
@@ -132,6 +134,7 @@ func (c *Client) ChatCompletionStream(ctx context.Context, req *ChatCompletionRe
|
||||
log.Error(err),
|
||||
)
|
||||
endChatSpan(span, nil, err)
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
||||
@@ -60,8 +60,10 @@ func (s *mockStream) Next() bool {
|
||||
if s.idx >= len(s.events) {
|
||||
return false
|
||||
}
|
||||
|
||||
s.current = s.events[s.idx]
|
||||
s.idx++
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -78,6 +80,7 @@ func newTestClient(provider llm.Provider) (*llm.Client, *tracetest.SpanRecorder)
|
||||
"test",
|
||||
llm.WithTracerProvider(tp),
|
||||
)
|
||||
|
||||
return client, recorder
|
||||
}
|
||||
|
||||
@@ -86,10 +89,12 @@ func spanAttrMap(recorder *tracetest.SpanRecorder) map[string]any {
|
||||
if len(spans) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
m := make(map[string]any)
|
||||
for _, a := range spans[0].Attributes() {
|
||||
m[string(a.Key)] = a.Value.AsInterface()
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
@@ -179,6 +184,7 @@ func TestErrors(t *testing.T) {
|
||||
|
||||
t.Run("with retry after", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
e := &llm.ErrRateLimit{RetryAfter: 30 * time.Second, Err: inner}
|
||||
assert.Contains(t, e.Error(), "retry after 30s")
|
||||
assert.Contains(t, e.Error(), "upstream")
|
||||
@@ -187,6 +193,7 @@ func TestErrors(t *testing.T) {
|
||||
|
||||
t.Run("without retry after", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
e := &llm.ErrRateLimit{Err: inner}
|
||||
assert.Contains(t, e.Error(), "rate limited")
|
||||
assert.NotContains(t, e.Error(), "retry after")
|
||||
@@ -195,7 +202,9 @@ func TestErrors(t *testing.T) {
|
||||
|
||||
t.Run("errors.As", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var target *llm.ErrRateLimit
|
||||
|
||||
e := &llm.ErrRateLimit{RetryAfter: 5 * time.Second, Err: inner}
|
||||
require.ErrorAs(t, e, &target)
|
||||
assert.Equal(t, 5*time.Second, target.RetryAfter)
|
||||
@@ -207,6 +216,7 @@ func TestErrors(t *testing.T) {
|
||||
|
||||
t.Run("with max tokens", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
e := &llm.ErrContextLength{MaxTokens: 4096, Err: inner}
|
||||
assert.Contains(t, e.Error(), "4096")
|
||||
assert.ErrorIs(t, e, inner)
|
||||
@@ -214,6 +224,7 @@ func TestErrors(t *testing.T) {
|
||||
|
||||
t.Run("without max tokens", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
e := &llm.ErrContextLength{Err: inner}
|
||||
assert.Contains(t, e.Error(), "context length exceeded")
|
||||
assert.NotContains(t, e.Error(), "max")
|
||||
@@ -223,6 +234,7 @@ func TestErrors(t *testing.T) {
|
||||
|
||||
t.Run("ErrContentFilter", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
e := &llm.ErrContentFilter{Err: inner}
|
||||
assert.Contains(t, e.Error(), "content filtered")
|
||||
assert.ErrorIs(t, e, inner)
|
||||
@@ -230,6 +242,7 @@ func TestErrors(t *testing.T) {
|
||||
|
||||
t.Run("ErrAuthentication", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
e := &llm.ErrAuthentication{Err: inner}
|
||||
assert.Contains(t, e.Error(), "authentication failed")
|
||||
assert.ErrorIs(t, e, inner)
|
||||
@@ -374,12 +387,14 @@ func TestChatCompletionStream(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
var collected []string
|
||||
|
||||
for stream.Next() {
|
||||
e := stream.Event()
|
||||
if e.Delta.Content != "" {
|
||||
collected = append(collected, e.Delta.Content)
|
||||
}
|
||||
}
|
||||
|
||||
require.NoError(t, stream.Err())
|
||||
require.NoError(t, stream.Close())
|
||||
|
||||
@@ -428,6 +443,7 @@ func TestChatCompletionStream(t *testing.T) {
|
||||
|
||||
for stream.Next() {
|
||||
}
|
||||
|
||||
assert.ErrorContains(t, stream.Err(), "connection reset")
|
||||
_ = stream.Close()
|
||||
|
||||
@@ -511,6 +527,7 @@ func TestChatCompletionStream(t *testing.T) {
|
||||
|
||||
for stream.Next() {
|
||||
}
|
||||
|
||||
require.NoError(t, stream.Err())
|
||||
require.NoError(t, stream.Close())
|
||||
|
||||
@@ -521,6 +538,7 @@ func TestChatCompletionStream(t *testing.T) {
|
||||
for _, a := range spans[0].Attributes() {
|
||||
attrs[string(a.Key)] = a.Value.AsInterface()
|
||||
}
|
||||
|
||||
assert.Equal(t, int64(100), attrs["gen_ai.usage.input_tokens"])
|
||||
assert.Equal(t, int64(50), attrs["gen_ai.usage.output_tokens"])
|
||||
assert.Equal(t, []string{"length"}, attrs["gen_ai.response.finish_reasons"])
|
||||
@@ -564,6 +582,7 @@ func TestStreamAccumulator(t *testing.T) {
|
||||
acc := llm.NewStreamAccumulator(&mockStream{events: events})
|
||||
for acc.Next() {
|
||||
}
|
||||
|
||||
require.NoError(t, acc.Err())
|
||||
|
||||
resp := acc.Response()
|
||||
@@ -614,6 +633,7 @@ func TestStreamAccumulator(t *testing.T) {
|
||||
acc := llm.NewStreamAccumulator(&mockStream{events: events})
|
||||
for acc.Next() {
|
||||
}
|
||||
|
||||
require.NoError(t, acc.Err())
|
||||
|
||||
resp := acc.Response()
|
||||
@@ -642,6 +662,7 @@ func TestStreamAccumulator(t *testing.T) {
|
||||
acc := llm.NewStreamAccumulator(&mockStream{events: events})
|
||||
for acc.Next() {
|
||||
}
|
||||
|
||||
require.NoError(t, acc.Err())
|
||||
|
||||
resp := acc.Response()
|
||||
@@ -660,7 +681,9 @@ func TestStreamAccumulator(t *testing.T) {
|
||||
}
|
||||
|
||||
acc := llm.NewStreamAccumulator(&mockStream{events: events})
|
||||
|
||||
var seen []string
|
||||
|
||||
for acc.Next() {
|
||||
e := acc.Event()
|
||||
if e.Delta.Content != "" {
|
||||
|
||||
@@ -165,20 +165,24 @@ func (m *Message) UnmarshalJSON(data []byte) error {
|
||||
|
||||
func (m Message) Text() string {
|
||||
var s strings.Builder
|
||||
|
||||
for _, p := range m.Parts {
|
||||
if tp, ok := p.(TextPart); ok {
|
||||
s.WriteString(tp.Text)
|
||||
}
|
||||
}
|
||||
|
||||
return s.String()
|
||||
}
|
||||
|
||||
func (m Message) Thinking() string {
|
||||
var s strings.Builder
|
||||
|
||||
for _, p := range m.Parts {
|
||||
if tp, ok := p.(ThinkingPart); ok {
|
||||
s.WriteString(tp.Text)
|
||||
}
|
||||
}
|
||||
|
||||
return s.String()
|
||||
}
|
||||
|
||||
@@ -95,6 +95,7 @@ func TestMessageJSONRoundTrip(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
var got Message
|
||||
|
||||
err = json.Unmarshal(data, &got)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.msg, got)
|
||||
|
||||
@@ -87,23 +87,29 @@ func NewProvider(apiKey string, opts ...Option) *Provider {
|
||||
if cfg.httpClient != nil {
|
||||
reqOpts = append(reqOpts, option.WithHTTPClient(cfg.httpClient))
|
||||
}
|
||||
|
||||
if cfg.baseURL != "" {
|
||||
reqOpts = append(reqOpts, option.WithBaseURL(cfg.baseURL))
|
||||
}
|
||||
|
||||
if cfg.organization != "" {
|
||||
reqOpts = append(reqOpts, option.WithOrganization(cfg.organization))
|
||||
}
|
||||
|
||||
if cfg.project != "" {
|
||||
reqOpts = append(reqOpts, option.WithProject(cfg.project))
|
||||
}
|
||||
|
||||
if cfg.requestTimeout > 0 {
|
||||
reqOpts = append(reqOpts, option.WithRequestTimeout(cfg.requestTimeout))
|
||||
}
|
||||
|
||||
if cfg.maxRetries != nil {
|
||||
reqOpts = append(reqOpts, option.WithMaxRetries(*cfg.maxRetries))
|
||||
}
|
||||
|
||||
client := openai.NewClient(reqOpts...)
|
||||
|
||||
return &Provider{client: &client}
|
||||
}
|
||||
|
||||
@@ -125,6 +131,7 @@ func (p *Provider) ChatCompletionStream(ctx context.Context, req *llm.ChatComple
|
||||
}
|
||||
|
||||
stream := p.client.Chat.Completions.NewStreaming(ctx, params)
|
||||
|
||||
return &openaiStream{stream: stream}, nil
|
||||
}
|
||||
|
||||
@@ -137,35 +144,45 @@ func buildParams(req *llm.ChatCompletionRequest) openai.ChatCompletionNewParams
|
||||
if req.MaxTokens != nil {
|
||||
params.MaxCompletionTokens = param.NewOpt(int64(*req.MaxTokens))
|
||||
}
|
||||
|
||||
if req.Temperature != nil {
|
||||
params.Temperature = param.NewOpt(*req.Temperature)
|
||||
}
|
||||
|
||||
if req.TopP != nil {
|
||||
params.TopP = param.NewOpt(*req.TopP)
|
||||
}
|
||||
|
||||
if req.FrequencyPenalty != nil {
|
||||
params.FrequencyPenalty = param.NewOpt(*req.FrequencyPenalty)
|
||||
}
|
||||
|
||||
if req.PresencePenalty != nil {
|
||||
params.PresencePenalty = param.NewOpt(*req.PresencePenalty)
|
||||
}
|
||||
|
||||
if len(req.StopSequences) > 0 {
|
||||
params.Stop = openai.ChatCompletionNewParamsStopUnion{
|
||||
OfStringArray: req.StopSequences,
|
||||
}
|
||||
}
|
||||
|
||||
if len(req.Tools) > 0 {
|
||||
params.Tools = buildTools(req.Tools)
|
||||
}
|
||||
|
||||
if req.ToolChoice != nil {
|
||||
params.ToolChoice = buildToolChoice(req.ToolChoice)
|
||||
}
|
||||
|
||||
if req.ParallelToolCalls != nil {
|
||||
params.ParallelToolCalls = param.NewOpt(*req.ParallelToolCalls)
|
||||
}
|
||||
|
||||
if req.ResponseFormat != nil {
|
||||
params.ResponseFormat = buildResponseFormat(req.ResponseFormat)
|
||||
}
|
||||
|
||||
if req.Thinking != nil && req.Thinking.Enabled && isReasoningModel(req.Model) {
|
||||
switch {
|
||||
case req.Thinking.BudgetTokens <= 1024:
|
||||
@@ -205,6 +222,7 @@ func buildMessages(messages []llm.Message) []openai.ChatCompletionMessageParamUn
|
||||
parts = append(parts, buildFilePart(p))
|
||||
}
|
||||
}
|
||||
|
||||
out = append(out, openai.UserMessage(parts))
|
||||
case llm.RoleAssistant:
|
||||
m := openai.ChatCompletionAssistantMessageParam{
|
||||
@@ -224,6 +242,7 @@ func buildMessages(messages []llm.Message) []openai.ChatCompletionMessageParamUn
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out = append(out, openai.ChatCompletionMessageParamUnion{OfAssistant: &m})
|
||||
case llm.RoleTool:
|
||||
out = append(out, openai.ToolMessage(msg.Text(), msg.ToolCallID))
|
||||
@@ -247,8 +266,10 @@ func buildTools(tools []llm.Tool) []openai.ChatCompletionToolParam {
|
||||
fn.Parameters = params
|
||||
}
|
||||
}
|
||||
|
||||
out[i] = openai.ChatCompletionToolParam{Function: fn}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -294,13 +315,16 @@ func buildResponseFormat(rf *llm.ResponseFormat) openai.ChatCompletionNewParamsR
|
||||
if rf.JSONSchema.Description != "" {
|
||||
schema.Description = param.NewOpt(rf.JSONSchema.Description)
|
||||
}
|
||||
|
||||
if rf.JSONSchema.Schema != nil {
|
||||
schema.Schema = rf.JSONSchema.Schema
|
||||
}
|
||||
|
||||
return openai.ChatCompletionNewParamsResponseFormatUnion{
|
||||
OfJSONSchema: &shared.ResponseFormatJSONSchemaParam{JSONSchema: schema},
|
||||
}
|
||||
}
|
||||
|
||||
return openai.ChatCompletionNewParamsResponseFormatUnion{}
|
||||
default:
|
||||
return openai.ChatCompletionNewParamsResponseFormatUnion{}
|
||||
@@ -319,6 +343,7 @@ func mapResponse(c *openai.ChatCompletion) *llm.ChatCompletionResponse {
|
||||
if len(c.Choices) > 0 {
|
||||
choice := c.Choices[0]
|
||||
resp.FinishReason = mapFinishReason(choice.FinishReason)
|
||||
|
||||
resp.Message = llm.Message{
|
||||
Role: llm.RoleAssistant,
|
||||
Parts: []llm.Part{llm.TextPart{Text: choice.Message.Content}},
|
||||
@@ -371,9 +396,11 @@ func mapError(err error) error {
|
||||
if apiErr.Code == "context_length_exceeded" {
|
||||
return &llm.ErrContextLength{Err: err}
|
||||
}
|
||||
|
||||
if apiErr.Code == "content_filter" {
|
||||
return &llm.ErrContentFilter{Err: err}
|
||||
}
|
||||
|
||||
return err
|
||||
default:
|
||||
return err
|
||||
@@ -384,13 +411,16 @@ func parseRetryAfter(resp *http.Response) time.Duration {
|
||||
if resp == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
h := resp.Header.Get("Retry-After")
|
||||
if h == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
if secs, err := strconv.Atoi(h); err == nil {
|
||||
return time.Duration(secs) * time.Second
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -407,6 +437,7 @@ func (s *openaiStream) Next() bool {
|
||||
|
||||
chunk := s.stream.Current()
|
||||
s.current = mapChunkToEvent(&chunk)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -419,6 +450,7 @@ func (s *openaiStream) Err() error {
|
||||
if err != nil {
|
||||
return mapError(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -474,6 +506,7 @@ func isReasoningModel(model string) bool {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -488,6 +521,7 @@ func buildFilePart(p llm.FilePart) openai.ChatCompletionContentPartUnionParam {
|
||||
if err != nil {
|
||||
return openai.TextContentPart(fmt.Sprintf("[file: %s, type: %s, error decoding content]", p.Filename, p.MimeType))
|
||||
}
|
||||
|
||||
return openai.TextContentPart(fmt.Sprintf("File: %s\n\n%s", p.Filename, string(decoded)))
|
||||
default:
|
||||
return openai.FileContentPart(openai.ChatCompletionContentPartFileFileParam{
|
||||
|
||||
@@ -66,6 +66,7 @@ func NewRegistry(models map[string]ModelDefinition) *Registry {
|
||||
m.ID = id
|
||||
r.index(&m)
|
||||
}
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -74,6 +75,7 @@ func DefaultRegistry() *Registry {
|
||||
defaultRegistryOnce.Do(func() {
|
||||
defaultRegistry = NewRegistry(generatedModels)
|
||||
})
|
||||
|
||||
return defaultRegistry
|
||||
}
|
||||
|
||||
@@ -84,9 +86,11 @@ func (r *Registry) Lookup(modelID string) (ModelDefinition, bool) {
|
||||
if m, ok := r.byID[modelID]; ok {
|
||||
return *m, true
|
||||
}
|
||||
|
||||
if m, ok := r.byID[normalizeModelID(modelID)]; ok {
|
||||
return *m, true
|
||||
}
|
||||
|
||||
return ModelDefinition{}, false
|
||||
}
|
||||
|
||||
@@ -114,5 +118,6 @@ func normalizeModelID(id string) string {
|
||||
if idx := strings.IndexByte(id, '/'); idx >= 0 {
|
||||
id = id[idx+1:]
|
||||
}
|
||||
|
||||
return strings.ReplaceAll(id, ".", "-")
|
||||
}
|
||||
|
||||
@@ -36,12 +36,15 @@ func startChatSpan(ctx context.Context, tracer trace.Tracer, system string, req
|
||||
if req.Temperature != nil {
|
||||
attrs = append(attrs, semconv.GenAIRequestTemperature(*req.Temperature))
|
||||
}
|
||||
|
||||
if req.MaxTokens != nil {
|
||||
attrs = append(attrs, semconv.GenAIRequestMaxTokens(*req.MaxTokens))
|
||||
}
|
||||
|
||||
if req.TopP != nil {
|
||||
attrs = append(attrs, semconv.GenAIRequestTopP(*req.TopP))
|
||||
}
|
||||
|
||||
if len(req.StopSequences) > 0 {
|
||||
attrs = append(attrs, semconv.GenAIRequestStopSequences(req.StopSequences...))
|
||||
}
|
||||
@@ -59,6 +62,7 @@ func endChatSpan(span trace.Span, resp *ChatCompletionResponse, err error) {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
span.End()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -95,13 +99,16 @@ func (s *tracedStream) Next() bool {
|
||||
s.finalizeSpan()
|
||||
return false
|
||||
}
|
||||
|
||||
s.lastEvent = s.inner.Event()
|
||||
if s.lastEvent.FinishReason != nil {
|
||||
s.finishReason = s.lastEvent.FinishReason
|
||||
}
|
||||
|
||||
if s.lastEvent.Usage != nil {
|
||||
s.usage = s.lastEvent.Usage
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -119,7 +126,9 @@ func (s *tracedStream) Close() error {
|
||||
s.span.RecordError(err)
|
||||
s.span.SetStatus(codes.Error, err.Error())
|
||||
}
|
||||
|
||||
s.finalizeSpan()
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -129,6 +138,7 @@ func (s *tracedStream) finalizeSpan() {
|
||||
s.span.RecordError(err)
|
||||
s.span.SetStatus(codes.Error, err.Error())
|
||||
s.span.End()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -139,12 +149,15 @@ func (s *tracedStream) finalizeSpan() {
|
||||
semconv.GenAIUsageOutputTokens(s.usage.OutputTokens),
|
||||
)
|
||||
}
|
||||
|
||||
if s.finishReason != nil {
|
||||
attrs = append(attrs, semconv.GenAIResponseFinishReasons(string(*s.finishReason)))
|
||||
}
|
||||
|
||||
if len(attrs) > 0 {
|
||||
s.span.SetAttributes(attrs...)
|
||||
}
|
||||
|
||||
s.span.End()
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user