fix(llm): preserve structured tool results as text across providers

Jasmine Zhang · Jasmine Zhang · commit 63ecf452addc · 2026-06-26T10:04:31.000+08:00
diff --git a/src/llm/providers/google-shared.convert.test.ts b/src/llm/providers/google-shared.convert.test.ts
@@ -375,4 +375,29 @@ describe("google-shared convertMessages", () => {
     expect(asRecord(toolCall.functionCall).id).toBeUndefined();
     expect(asRecord(toolResponse.functionResponse).id).toBeUndefined();
   });
+
+  it("serializes structured tool results into function responses", () => {
+    const model = makeModel("gemini-1.5-pro");
+    const context = {
+      messages: [
+        {
+          role: "toolResult",
+          toolCallId: "call_1",
+          toolName: "session_status",
+          content: [{ type: "json", payload: { sessionKey: "current", status: "ok" } }],
+          isError: false,
+          timestamp: 0,
+        },
+      ],
+    } as unknown as Context;
+
+    const contents = convertMessagesForTest(model, context);
+    const toolResponsePart = contents[0]?.parts?.find(
+      (part) => typeof part === "object" && part !== null && "functionResponse" in part,
+    );
+    const toolResponse = requireRecordProperty(asRecord(toolResponsePart), "functionResponse");
+    expect(asRecord(toolResponse.response).output).toBe(
+      '{"type":"json","payload":{"sessionKey":"current","status":"ok"}}',
+    );
+  });
 });
diff --git a/src/llm/providers/google-shared.ts b/src/llm/providers/google-shared.ts
@@ -32,6 +32,7 @@ import type {
 } from "../types.js";
 import type { AssistantMessageEventStream } from "../utils/event-stream.js";
 import { sanitizeSurrogates } from "../utils/sanitize-unicode.js";
+import { extractToolResultText } from "./tool-result-text.js";
 import { transformMessages } from "./transform-messages.js";
 
 export type GoogleApiType = "google-generative-ai" | "google-vertex";
@@ -278,8 +279,7 @@ export function convertMessages<T extends GoogleApiType>(
       });
     } else if (msg.role === "toolResult") {
       // Extract text and image content
-      const textContent = msg.content.filter((c): c is TextContent => c.type === "text");
-      const textResult = textContent.map((c) => c.text).join("\n");
+      const textResult = extractToolResultText(msg.content as Array<Record<string, unknown>>);
       const imageContent = model.input.includes("image")
         ? msg.content.filter((c): c is ImageContent => c.type === "image")
         : [];
diff --git a/src/llm/providers/openai-completions.test.ts b/src/llm/providers/openai-completions.test.ts
@@ -40,7 +40,11 @@ vi.mock("openai", () => {
   return { default: MockOpenAI };
 });
 
-import { streamOpenAICompletions, streamSimpleOpenAICompletions } from "./openai-completions.js";
+import {
+  convertMessages,
+  streamOpenAICompletions,
+  streamSimpleOpenAICompletions,
+} from "./openai-completions.js";
 
 const model = {
   id: "gpt-5.5",
@@ -1059,3 +1063,32 @@ describe("openai-completions stop-reason tool-call guard", () => {
     expect(result.content.filter((b) => b.type === "toolCall")).toStrictEqual([]);
   });
 });
+
+describe("convertMessages", () => {
+  it("serializes structured tool results as tool text", () => {
+    const params = convertMessages(
+      model,
+      {
+        messages: [
+          {
+            role: "toolResult",
+            toolCallId: "call_1",
+            toolName: "session_status",
+            content: [{ type: "json", payload: { sessionKey: "current", status: "ok" } }],
+            isError: false,
+            timestamp: 0,
+          },
+        ],
+      } as Context,
+      false,
+    );
+
+    expect(params).toContainEqual(
+      expect.objectContaining({
+        role: "tool",
+        tool_call_id: "call_1",
+        content: '{"type":"json","payload":{"sessionKey":"current","status":"ok"}}',
+      }),
+    );
+  });
+});
diff --git a/src/llm/providers/openai-completions.ts b/src/llm/providers/openai-completions.ts
@@ -51,6 +51,7 @@ import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copi
 import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.js";
 import { mapOpenAIStopReason } from "./openai-stop-reason.js";
 import { buildBaseOptions } from "./simple-options.js";
+import { extractToolResultText } from "./tool-result-text.js";
 import { transformMessages } from "./transform-messages.js";
 
 /**
@@ -1128,10 +1129,7 @@ export function convertMessages(
         const toolMsg = transformedMessages[j] as ToolResultMessage;
 
         // Extract text and image content
-        const textResult = toolMsg.content
-          .filter(isTextContentBlock)
-          .map((block) => block.text)
-          .join("\n");
+        const textResult = extractToolResultText(toolMsg.content as Array<Record<string, unknown>>);
         const hasImages = toolMsg.content.some((c) => c.type === "image");
 
         // Always send tool result with text (or placeholder if only images)
diff --git a/src/llm/providers/openai-responses-shared.test.ts b/src/llm/providers/openai-responses-shared.test.ts
@@ -527,6 +527,39 @@ describe("convertResponsesMessages", () => {
       summary: [],
     });
   });
+
+  it("serializes structured tool results as text instead of image placeholders", () => {
+    const input = convertResponsesMessages(
+      nativeOpenAIModel,
+      {
+        systemPrompt: "system",
+        messages: [
+          {
+            role: "toolResult",
+            toolCallId: "call_structured",
+            toolName: "session_status",
+            content: [
+              {
+                type: "json",
+                payload: { sessionKey: "current", model: "openai/gpt-5.4", status: "ok" },
+              },
+            ],
+            isError: false,
+            timestamp: 1,
+          },
+        ],
+      } satisfies Context,
+      allowedToolCallProviders,
+      { includeSystemPrompt: false, replayResponsesItemIds: false },
+    ) as unknown as Array<Record<string, unknown>>;
+
+    expect(input).toContainEqual({
+      type: "function_call_output",
+      call_id: "call_structured",
+      output:
+        '{"type":"json","payload":{"sessionKey":"current","model":"openai/gpt-5.4","status":"ok"}}',
+    });
+  });
 });
 
 describe("processResponsesStream", () => {
diff --git a/src/llm/providers/openai-responses-shared.ts b/src/llm/providers/openai-responses-shared.ts
@@ -45,6 +45,7 @@ import { headersToRecord } from "../utils/headers.js";
 import { parseStreamingJson } from "../utils/json-parse.js";
 import { sanitizeSurrogates } from "../utils/sanitize-unicode.js";
 import { convertResponsesToolPayload, convertResponsesTools } from "./openai-responses-tools.js";
+import { extractToolResultText } from "./tool-result-text.js";
 import { transformMessages } from "./transform-messages.js";
 
 // =============================================================================
@@ -372,10 +373,7 @@ export function convertResponsesMessages<TApi extends Api>(
       }
       messages.push(...output);
     } else if (msg.role === "toolResult") {
-      const textResult = msg.content
-        .filter((c): c is TextContent => c.type === "text")
-        .map((c) => c.text)
-        .join("\n");
+      const textResult = extractToolResultText(msg.content as Array<Record<string, unknown>>);
       const hasImages = msg.content.some((c): c is ImageContent => c.type === "image");
       const hasText = textResult.length > 0;
       const [callId] = msg.toolCallId.split("|");
diff --git a/src/llm/providers/tool-result-text.ts b/src/llm/providers/tool-result-text.ts
@@ -0,0 +1,57 @@
+import { sanitizeSurrogates } from "../utils/sanitize-unicode.js";
+
+type ToolResultContentBlock = Record<string, unknown>;
+
+function stringifyStructuredBlock(block: ToolResultContentBlock): string | undefined {
+  const seen = new WeakSet<object>();
+  try {
+    const serialized = JSON.stringify(block, (_key, value) => {
+      if (typeof value === "string") {
+        return value.replace(
+          /data:[^"'\\\s]+/gi,
+          (match) => `[inline data URI: ${match.length} chars]`,
+        );
+      }
+      if (!value || typeof value !== "object") {
+        return value;
+      }
+      if (seen.has(value)) {
+        return "[Circular]";
+      }
+      seen.add(value);
+      return value;
+    });
+    if (!serialized || serialized === "{}") {
+      return undefined;
+    }
+    return serialized.length > 1_000
+      ? `${serialized.slice(0, 1_000)}... (${serialized.length} chars)`
+      : serialized;
+  } catch {
+    return undefined;
+  }
+}
+
+export function extractToolResultText(blocks: readonly ToolResultContentBlock[]): string {
+  const parts: string[] = [];
+  for (const block of blocks) {
+    if (!block || typeof block !== "object") {
+      continue;
+    }
+    if (block.type === "image") {
+      continue;
+    }
+    if (block.type === "text") {
+      const text = typeof block.text === "string" ? block.text : "";
+      if (text) {
+        parts.push(text);
+      }
+      continue;
+    }
+    const structured = stringifyStructuredBlock(block);
+    if (structured) {
+      parts.push(structured);
+    }
+  }
+  return sanitizeSurrogates(parts.join("\n"));
+}
