Skip to content

feat: openai: allow assistant content to be string #486

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mathetake merged 4 commits into from
Mar 13, 2025
Merged

feat: openai: allow assistant content to be string #486

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e161b71
all for backwards compatibility for assistant content
aabchoo Mar 13, 2025
6544a5e
remove new case that was not covered previously
aabchoo Mar 13, 2025
269709d
remove comments + test case for content as string
aabchoo Mar 13, 2025
9983184
Merge branch 'main' into aaron/string-or-assistant-content-openai
yuzisun Mar 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion internal/apischema/openai/openai.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ package openai

import (
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
Expand Down Expand Up @@ -149,6 +150,28 @@ func (c *ChatCompletionContentPartUserUnionParam) UnmarshalJSON(data []byte) err
return nil
}

type StringOrAssistantRoleContentUnion struct {
Value interface{}
}

func (s *StringOrAssistantRoleContentUnion) UnmarshalJSON(data []byte) error {
var str string
err := json.Unmarshal(data, &str)
if err == nil {
s.Value = str
return nil
}

var content ChatCompletionAssistantMessageParamContent
err = json.Unmarshal(data, &content)
if err == nil {
s.Value = content
return nil
}

return errors.New("cannot unmarshal JSON data as string or assistant content parts")
}

type StringOrArray struct {
Value interface{}
}
Expand Down Expand Up @@ -335,7 +358,7 @@ type ChatCompletionAssistantMessageParam struct {
Audio ChatCompletionAssistantMessageParamAudio `json:"audio,omitempty"`
// The contents of the assistant message. Required unless `tool_calls` or
// `function_call` is specified.
Content ChatCompletionAssistantMessageParamContent `json:"content"`
Content StringOrAssistantRoleContentUnion `json:"content"`
// An optional name for the participant. Provides the model information to
// differentiate between participants of the same role.
Name string `json:"name,omitempty"`
Expand Down
30 changes: 29 additions & 1 deletion internal/apischema/openai/openai_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,35 @@ func TestOpenAIChatCompletionMessageUnmarshal(t *testing.T) {
{
Value: ChatCompletionAssistantMessageParam{
Role: ChatMessageRoleAssistant,
Content: ChatCompletionAssistantMessageParamContent{Text: ptr.To("you are a helpful assistant")},
Content: StringOrAssistantRoleContentUnion{Value: ChatCompletionAssistantMessageParamContent{Text: ptr.To("you are a helpful assistant")}},
},
Type: ChatMessageRoleAssistant,
},
},
},
},
{
name: "assistant message string",
in: []byte(`{"model": "gpu-o4",
"messages": [
{"role": "assistant", "content": "you are a helpful assistant"},
{"role": "assistant", "content": "{'text': 'you are a helpful assistant'}"}
]}
`),
out: &ChatCompletionRequest{
Model: "gpu-o4",
Messages: []ChatCompletionMessageParamUnion{
{
Value: ChatCompletionAssistantMessageParam{
Role: ChatMessageRoleAssistant,
Content: StringOrAssistantRoleContentUnion{Value: "you are a helpful assistant"},
},
Type: ChatMessageRoleAssistant,
},
{
Value: ChatCompletionAssistantMessageParam{
Role: ChatMessageRoleAssistant,
Content: StringOrAssistantRoleContentUnion{Value: "{'text': 'you are a helpful assistant'}"},
},
Type: ChatMessageRoleAssistant,
},
Expand Down
13 changes: 8 additions & 5 deletions internal/extproc/translator/openai_awsbedrock.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,11 +245,14 @@ func (o *openAIToAWSBedrockTranslatorV1ChatCompletion) openAIMessageToBedrockMes
) (*awsbedrock.Message, error) {
var bedrockMessage *awsbedrock.Message
contentBlocks := make([]*awsbedrock.ContentBlock, 0)
if openAiMessage.Content.Type == openai.ChatCompletionAssistantMessageParamContentTypeRefusal {
contentBlocks = append(contentBlocks, &awsbedrock.ContentBlock{Text: openAiMessage.Content.Refusal})
} else if openAiMessage.Content.Text != nil {
// TODO: we are sometimes missing the content (should fix)
contentBlocks = append(contentBlocks, &awsbedrock.ContentBlock{Text: openAiMessage.Content.Text})
if v, ok := openAiMessage.Content.Value.(string); ok {
contentBlocks = append(contentBlocks, &awsbedrock.ContentBlock{Text: &v})
} else if content, ok := openAiMessage.Content.Value.(openai.ChatCompletionAssistantMessageParamContent); ok {
if content.Type == openai.ChatCompletionAssistantMessageParamContentTypeRefusal {
contentBlocks = append(contentBlocks, &awsbedrock.ContentBlock{Text: content.Refusal})
} else if content.Text != nil {
contentBlocks = append(contentBlocks, &awsbedrock.ContentBlock{Text: content.Text})
}
}
bedrockMessage = &awsbedrock.Message{
Role: role,
Expand Down
22 changes: 20 additions & 2 deletions internal/extproc/translator/openai_awsbedrock_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,11 @@ func TestOpenAIToAWSBedrockTranslatorV1ChatCompletion_RequestBody(t *testing.T)
},
{
Value: openai.ChatCompletionAssistantMessageParam{
Content: openai.ChatCompletionAssistantMessageParamContent{
Text: ptr.To("I dunno"),
Content: openai.StringOrAssistantRoleContentUnion{
Copy link
Copy Markdown
Contributor

@yuzisun yuzisun Mar 13, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you help add the missing unmarshall test which is why this issue is not caught

Copy link
Copy Markdown
Contributor Author

@aabchoo aabchoo Mar 13, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The issue was content being parsed as string even though it was sent as AssistantContentType. Added a test, but the behavior is unideal since the unmarshalling now sees it as valid:

https://github.com/envoyproxy/ai-gateway/pull/486/files#diff-3387664749ae0034b39372c41c009a68c9b68b7ee9ae5886babe0a4f846e542fR168

Value: openai.ChatCompletionAssistantMessageParamContent{
Type: openai.ChatCompletionAssistantMessageParamContentTypeText,
Text: ptr.To("I dunno"),
},
},
ToolCalls: []openai.ChatCompletionMessageToolCallParam{
{
Expand All @@ -97,6 +100,13 @@ func TestOpenAIToAWSBedrockTranslatorV1ChatCompletion_RequestBody(t *testing.T)
},
}, Type: openai.ChatMessageRoleAssistant,
},
{
Value: openai.ChatCompletionAssistantMessageParam{
Content: openai.StringOrAssistantRoleContentUnion{
Value: "I also dunno",
},
}, Type: openai.ChatMessageRoleAssistant,
},
},
},
output: awsbedrock.ConverseInput{
Expand Down Expand Up @@ -164,6 +174,14 @@ func TestOpenAIToAWSBedrockTranslatorV1ChatCompletion_RequestBody(t *testing.T)
},
},
},
{
Role: openai.ChatMessageRoleAssistant,
Content: []*awsbedrock.ContentBlock{
{
Text: ptr.To("I also dunno"),
},
},
},
},
},
},
Expand Down