Skip to content

Commit 984857e

Browse files
daniel-lxsdaniel-lxs
committed
feat: enhance image URL handling in convertToOpenAiMessages and improve saveApiConversationHistory error handling
1 parent cdca46e commit 984857e

File tree

4 files changed

+94
-6
lines changed

4 files changed

+94
-6
lines changed

‎src/api/transform/__tests__/openai-format.spec.ts‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,50 @@ describe("convertToOpenAiMessages", () => {
7676
})
7777
})
7878

79+
it("should preserve AI SDK image data URLs without double-prefixing", () => {
80+
const messages: any[] = [
81+
{
82+
role: "user",
83+
content: [
84+
{
85+
type: "image",
86+
image: "data:image/png;base64,already_encoded",
87+
mediaType: "image/png",
88+
},
89+
],
90+
},
91+
]
92+
93+
const openAiMessages = convertToOpenAiMessages(messages)
94+
const content = openAiMessages[0].content as Array<{ type: string; image_url?: { url: string } }>
95+
expect(content[0]).toEqual({
96+
type: "image_url",
97+
image_url: { url: "data:image/png;base64,already_encoded" },
98+
})
99+
})
100+
101+
it("should preserve AI SDK image http URLs without converting to data URLs", () => {
102+
const messages: any[] = [
103+
{
104+
role: "user",
105+
content: [
106+
{
107+
type: "image",
108+
image: "https://example.com/image.png",
109+
mediaType: "image/png",
110+
},
111+
],
112+
},
113+
]
114+
115+
const openAiMessages = convertToOpenAiMessages(messages)
116+
const content = openAiMessages[0].content as Array<{ type: string; image_url?: { url: string } }>
117+
expect(content[0]).toEqual({
118+
type: "image_url",
119+
image_url: { url: "https://example.com/image.png" },
120+
})
121+
})
122+
79123
it("should handle assistant messages with tool use (no normalization without normalizeToolCallId)", () => {
80124
const anthropicMessages: any[] = [
81125
{

‎src/api/transform/openai-format.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -331,9 +331,17 @@ export function convertToOpenAiMessages(
331331
mediaType?: string
332332
source?: { media_type?: string; data?: string }
333333
}): string => {
334-
// AI SDK format: { type: "image", image: base64, mediaType: mimeType }
335-
if (part.image && part.mediaType) {
336-
return `data:${part.mediaType};base64,${part.image}`
334+
// AI SDK format:
335+
// - raw base64 + mediaType: construct data URL
336+
// - existing data/http(s) URL in image: pass through unchanged
337+
if (part.image) {
338+
const image = part.image.trim()
339+
if (image.startsWith("data:") || /^https?:\/\//i.test(image)) {
340+
return image
341+
}
342+
if (part.mediaType) {
343+
return `data:${part.mediaType};base64,${image}`
344+
}
337345
}
338346
// Legacy Anthropic format: { type: "image", source: { media_type, data } }
339347
if (part.source?.media_type && part.source?.data) {

‎src/core/task/Task.ts‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1245,11 +1245,16 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
12451245

12461246
private async saveApiConversationHistory(): Promise<boolean> {
12471247
try {
1248-
await saveRooMessages({
1248+
const saved = await saveRooMessages({
12491249
messages: structuredClone(this.apiConversationHistory),
12501250
taskId: this.taskId,
12511251
globalStoragePath: this.globalStoragePath,
12521252
})
1253+
// saveRooMessages historically returned void in some tests/mocks; treat only explicit false as failure.
1254+
if (saved === false) {
1255+
console.error("Failed to save API conversation history: saveRooMessages returned false")
1256+
return false
1257+
}
12531258
return true
12541259
} catch (error) {
12551260
console.error("Failed to save API conversation history:", error)
@@ -3777,9 +3782,18 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
37773782
content: [...this.pendingToolResults],
37783783
ts: Date.now(),
37793784
}
3785+
const previousHistoryLength = this.apiConversationHistory.length
37803786
this.apiConversationHistory.push(toolMessage)
3781-
await this.saveApiConversationHistory()
3782-
this.pendingToolResults = []
3787+
const saved = await this.saveApiConversationHistory()
3788+
if (saved) {
3789+
this.pendingToolResults = []
3790+
} else {
3791+
// Keep pending results for retry and roll back in-memory insertion to avoid duplicates.
3792+
this.apiConversationHistory = this.apiConversationHistory.slice(0, previousHistoryLength)
3793+
console.warn(
3794+
`[Task#${this.taskId}] Failed to persist pending tool results in main loop; keeping pending results for retry`,
3795+
)
3796+
}
37833797
}
37843798

37853799
// Push to stack if there's content OR if we're paused waiting for a subtask.

‎src/core/task/__tests__/Task.persistence.spec.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,28 @@ describe("Task persistence", () => {
298298
vi.useRealTimers()
299299
})
300300

301+
it("returns false when saveRooMessages resolves false", async () => {
302+
vi.useFakeTimers()
303+
304+
mockSaveRooMessages.mockResolvedValue(false)
305+
306+
const task = new Task({
307+
provider: mockProvider,
308+
apiConfiguration: mockApiConfig,
309+
task: "test task",
310+
startTask: false,
311+
})
312+
313+
const promise = task.retrySaveApiConversationHistory()
314+
await vi.runAllTimersAsync()
315+
const result = await promise
316+
317+
expect(result).toBe(false)
318+
expect(mockSaveRooMessages).toHaveBeenCalledTimes(3)
319+
320+
vi.useRealTimers()
321+
})
322+
301323
it("succeeds on 2nd retry attempt", async () => {
302324
vi.useFakeTimers()
303325

0 commit comments

Comments
 (0)