Skip to content

Commit f055995

Browse files
only21milcodex
andauthored
fix(chat): mark tool images beyond the preview limit (pingdotgg#23595)
Co-authored-by: GPT-6.1 Sol <[email protected]>
1 parent 60b35a1 commit f055995

2 files changed

Lines changed: 66 additions & 7 deletions

File tree

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import { describe, expect, it } from "vite-plus/test";
2+
import { ThreadId, TurnItemId, type OrchestrationV2TurnItem } from "@t3tools/contracts";
3+
import * as DateTime from "effect/DateTime";
4+
5+
import { turnItemOutputImages, turnItemOutputText } from "./itemDetail.ts";
6+
7+
function toolItem(output: unknown): OrchestrationV2TurnItem {
8+
return {
9+
id: TurnItemId.make("images"),
10+
threadId: ThreadId.make("thread"),
11+
runId: null,
12+
nodeId: null,
13+
providerThreadId: null,
14+
providerTurnId: null,
15+
nativeItemRef: null,
16+
parentItemId: null,
17+
ordinal: 1,
18+
status: "completed",
19+
title: null,
20+
startedAt: null,
21+
completedAt: null,
22+
updatedAt: DateTime.makeUnsafe("2026-10-01T00:00:00Z"),
23+
type: "dynamic_tool",
24+
toolName: "screenshots",
25+
input: {},
26+
output,
27+
};
28+
}
29+
30+
describe("tool image output parsing", () => {
31+
it.each(["array", "content"] as const)(
32+
"keeps markers for images beyond the served positions in %s output",
33+
(shape) => {
34+
// Detail responses retain image metadata after the server strips image bytes.
35+
const image = { type: "image", mimeType: "image/png" };
36+
const blocks = Array.from({ length: 8 }, () => ({ ...image }));
37+
const wrap = (content: unknown[]) => (shape === "array" ? content : { content });
38+
const withinLimit = toolItem(wrap(blocks));
39+
expect(turnItemOutputText(withinLimit)).toBeNull();
40+
expect(turnItemOutputImages(withinLimit)).toHaveLength(8);
41+
42+
const overflow = toolItem(
43+
wrap([
44+
...blocks,
45+
{ type: "text", text: "Two additional screenshots:" },
46+
{ ...image, sizeBytes: 100 },
47+
{ type: "image", source: { type: "base64", media_type: "image/jpeg" } },
48+
]),
49+
);
50+
expect(turnItemOutputImages(overflow)).toHaveLength(8);
51+
expect(turnItemOutputText(overflow)).toBe("Two additional screenshots:\n[image]\n[image]");
52+
},
53+
);
54+
});

‎packages/client-runtime/src/work-log/itemDetail.ts‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import type { AssetResource, OrchestrationV2TurnItem } from "@t3tools/contracts";
2-
import { readToolOutputImage, toolOutputImages } from "@t3tools/shared/toolOutput";
2+
import { toolOutputImageBlocks, toolOutputImages } from "@t3tools/shared/toolOutput";
33
import * as DateTime from "effect/DateTime";
44

55
const MAX_TEXT_BLOCK_DEPTH = 4;
@@ -12,28 +12,33 @@ function isSummarizedValue(value: unknown): boolean {
1212
return isRecord(value) && value.truncated === true && typeof value.summary === "string";
1313
}
1414

15-
function textFromBlocks(value: unknown, depth: number): string | null {
15+
function textFromBlocks(
16+
value: unknown,
17+
depth: number,
18+
imageBlocks: ReadonlyArray<unknown>,
19+
): string | null {
1620
if (depth > MAX_TEXT_BLOCK_DEPTH) return null;
1721
if (typeof value === "string") return value;
1822
if (Array.isArray(value)) {
19-
const parts = value.map((block) => textFromBlocks(block, depth + 1));
23+
const parts = value.map((block) => textFromBlocks(block, depth + 1, imageBlocks));
2024
return parts.every((part) => part !== null)
2125
? parts.filter((part) => part !== "").join("\n")
2226
: null;
2327
}
2428
if (!isRecord(value)) return null;
2529
if (value.type === "text" && typeof value.text === "string") return value.text;
26-
if (value.type === "image") return readToolOutputImage(value) ? "" : "[image]";
30+
if (value.type === "image") return imageBlocks.includes(value) ? "" : "[image]";
2731
if (value.type === "resource_link" && typeof value.uri === "string") return value.uri;
2832
if (value.type === "resource" && isRecord(value.resource)) {
2933
const resource = value.resource;
3034
if (typeof resource.text === "string") return resource.text;
3135
if (typeof resource.uri === "string") return resource.uri;
3236
}
3337
const keys = Object.keys(value).filter((key) => key !== "isError" && key !== "is_error");
34-
if (keys.length === 1 && keys[0] === "content") return textFromBlocks(value.content, depth + 1);
38+
if (keys.length === 1 && keys[0] === "content")
39+
return textFromBlocks(value.content, depth + 1, imageBlocks);
3540
if (keys.length === 2 && keys.includes("content") && keys.includes("structuredContent")) {
36-
const text = textFromBlocks(value.content, depth + 1);
41+
const text = textFromBlocks(value.content, depth + 1, imageBlocks);
3742
return text?.trim() ? text : null;
3843
}
3944
return null;
@@ -61,7 +66,7 @@ function prettyJsonText(text: string): string {
6166
function formatToolValue(value: unknown, hasImagePreview = false): string | null {
6267
if (value === undefined || value === null) return null;
6368
if (hasImagePreview && isRecord(value) && value.type === "image") return null;
64-
const text = textFromBlocks(value, 0);
69+
const text = textFromBlocks(value, 0, toolOutputImageBlocks(value));
6570
if (text !== null) return text.trim() ? prettyJsonText(text) : null;
6671
let json: string | undefined;
6772
try {

0 commit comments

Comments
 (0)