Skip to content

Commit d51bc58

Browse files
authored
fix(tools): show images from MCP and OpenCode outputs (#30389)
Preserve MCP and OpenCode image outputs so tool details show screenshots before their inputs. Highlight HTML preview input and valid JSON arguments and results across clients, while keeping native file text readable. Sanitize CI test logs to avoid a Unicode runner crash without changing test exit codes. Validation: all CI checks passed, 729 affected tests and the full 156-test Pi suite passed locally, web and iOS details verified. Model: GPT 6.1 Sol. Harness: Codex.
1 parent a185dd9 commit d51bc58

20 files changed

Lines changed: 401 additions & 105 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,12 +272,18 @@ jobs:
272272
env:
273273
TEST_PACKAGES: ${{ needs.changes.outputs.test_packages }}
274274
run: |
275+
set -o pipefail
275276
filters=(--filter '!t3' --filter '!@t3tools/monorepo' --filter '!@t3tools/web')
276277
if [[ -n "$TEST_PACKAGES" ]]; then
277278
filters=()
278279
for package in $TEST_PACKAGES; do filters+=(--filter "$package"); done
279280
fi
280-
vp run --parallel --concurrency-limit 4 "${filters[@]}" test
281+
# The runner's trace writer can crash on surrogate pairs in test output.
282+
vp run --parallel --concurrency-limit 4 "${filters[@]}" test 2>&1 | python3 -u -c '
283+
import sys
284+
for line in sys.stdin:
285+
sys.stdout.write(line.encode("ascii", "backslashreplace").decode("ascii"))
286+
'
281287
282288
# The web suite is the largest outside the server. In the shared Test job it
283289
# started last and ran alone for about 40s after everything else finished.

‎apps/mobile/src/features/threads/thread-work-log.tsx‎

Lines changed: 43 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ import {
6969
workEntryRowLabel,
7070
} from "../../lib/threadActivity";
7171
import {
72+
isJsonToolText,
7273
toolCallLines,
7374
turnItemOutputImages,
7475
turnItemOutputText,
@@ -874,14 +875,16 @@ function WorkspacePreparationRetryButton(props: {
874875
);
875876
}
876877

877-
function ReplCodeText({
878+
function ToolCodeText({
878879
code,
880+
language,
879881
theme,
880882
}: {
881883
readonly code: string;
884+
readonly language: "typescript" | "html" | "json";
882885
readonly theme: "light" | "dark";
883886
}) {
884-
const lines = useMarkdownCodeHighlight({ code, language: "typescript", theme, enabled: true });
887+
const lines = useMarkdownCodeHighlight({ code, language, theme, enabled: true });
885888
let offset = 0;
886889
const highlighted = lines?.flatMap((line, lineIndex) => {
887890
const separator = lineIndex > 0 ? "\n" : "";
@@ -1255,20 +1258,32 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
12551258
) : call ? (
12561259
[
12571260
call.command,
1258-
...(call.args ?? []).map(([key, value]) =>
1259-
key === "code" &&
1260-
shownItem.type === "dynamic_tool" &&
1261-
isJavaScriptReplTool(shownItem.toolName) ? (
1261+
...(call.args ?? []).map(([key, value]) => {
1262+
const language =
1263+
key === "html"
1264+
? "html"
1265+
: key === "code" &&
1266+
shownItem.type === "dynamic_tool" &&
1267+
isJavaScriptReplTool(shownItem.toolName)
1268+
? "typescript"
1269+
: isJsonToolText(value)
1270+
? "json"
1271+
: null;
1272+
return language ? (
12621273
<View key={key}>
12631274
<Text className="font-mono text-2xs leading-normal text-foreground-muted">
12641275
{key}
12651276
</Text>
1266-
<ReplCodeText code={value} theme={props.themeAppearance} />
1277+
<ToolCodeText
1278+
code={value}
1279+
language={language}
1280+
theme={props.themeAppearance}
1281+
/>
12671282
</View>
12681283
) : (
12691284
`${key} ${value}`
1270-
),
1271-
),
1285+
);
1286+
}),
12721287
call.argsText,
12731288
]
12741289
.filter((line) => Boolean(line))
@@ -1281,15 +1296,23 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
12811296
selectable
12821297
className="font-mono text-2xs leading-normal text-foreground"
12831298
>
1284-
{line}
1299+
{isJsonToolText(line) ? (
1300+
<ToolCodeText code={line} language="json" theme={props.themeAppearance} />
1301+
) : (
1302+
line
1303+
)}
12851304
</Text>
12861305
) : (
12871306
line
12881307
) /* oxlint-enable react/no-array-index-key */,
12891308
)
12901309
) : fullDetail ? (
12911310
<Text selectable className="font-mono text-2xs leading-normal text-foreground-muted">
1292-
{fullDetail}
1311+
{isJsonToolText(fullDetail) ? (
1312+
<ToolCodeText code={fullDetail} language="json" theme={props.themeAppearance} />
1313+
) : (
1314+
fullDetail
1315+
)}
12931316
</Text>
12941317
) : null}
12951318
{fetchedOutput ? (
@@ -1300,7 +1323,15 @@ const ThreadWorkLogRow = function ThreadWorkLogRow(
13001323
(!call || call.command || call.args || call.argsText) && "mt-1.5",
13011324
)}
13021325
>
1303-
{fetchedOutput}
1326+
{isJsonToolText(fetchedOutput) ? (
1327+
<ToolCodeText
1328+
code={fetchedOutput}
1329+
language="json"
1330+
theme={props.themeAppearance}
1331+
/>
1332+
) : (
1333+
fetchedOutput
1334+
)}
13041335
</Text>
13051336
) : null}
13061337
{streamsOutput ? null : exitLine}

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import type {
2626
} from "@anthropic-ai/claude-agent-sdk/sdk-tools";
2727
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
2828
import * as NodeServices from "@effect/platform-node/NodeServices";
29+
import { toolOutputImages } from "@t3tools/shared/toolOutput";
2930
import {
3031
ChatAttachmentId,
3132
type ChatAttachment,
@@ -4844,11 +4845,19 @@ describe("ClaudeAdapterV2 background wake turns", () => {
48444845
{
48454846
type: "tool_result",
48464847
tool_use_id: id,
4847-
content: "Weather result",
4848+
content: [
4849+
{ type: "text", text: "Weather result" },
4850+
{
4851+
type: "image",
4852+
source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" },
4853+
},
4854+
],
48484855
is_error: isError,
48494856
},
48504857
],
48514858
},
4859+
tool_use_result:
4860+
title === undefined ? { screenshot: { mimeType: "image/png" } } : undefined,
48524861
}),
48534862
);
48544863
yield* Queue.offer(
@@ -4881,6 +4890,9 @@ describe("ClaudeAdapterV2 background wake turns", () => {
48814890
assert.deepEqual(item.input, { city: "Berlin" });
48824891
}
48834892
assert.equal(new Set(items.map((item) => item.id)).size, 1);
4893+
assert.deepEqual(toolOutputImages(items.at(-1)?.output), [
4894+
{ mimeType: "image/png", data: "iVBORw0KGgo=" },
4895+
]);
48844896
}).pipe(
48854897
Effect.scoped,
48864898
Effect.provide(

‎apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import * as DocumentText from "../../provider/DocumentText.ts";
2+
import { mcpToolResultOutput, toolOutputImages } from "@t3tools/shared/toolOutput";
23
import { makeProviderTextDeltaCoalescer } from "./ProviderTextDeltaCoalescer.ts";
34
import { claudeTaskOutputRoot, tailClaudeTaskOutput } from "./ClaudeTaskOutputTail.ts";
45
import {
@@ -3084,7 +3085,14 @@ function claudeNativeToolOutputValue(output: ClaudeNativeToolOutput): unknown |
30843085
case "none":
30853086
return undefined;
30863087
case "content_block":
3088+
return output.value;
30873089
case "structured_tool_use_result":
3090+
if (toolOutputImages(output.value).length === 0) {
3091+
return mcpToolResultOutput({
3092+
content: output.fallbackValue,
3093+
structuredContent: output.value,
3094+
});
3095+
}
30883096
return output.value;
30893097
default:
30903098
return assertNever(output);

‎apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ import {
5050
import { getModelSelectionStringOptionValue, modelSelectionsEqual } from "@t3tools/shared/model";
5151
import { appendGeneratedImageEmbeds } from "@t3tools/shared/generatedImageMarkdown";
5252
import { resolveSpawnCommand } from "@t3tools/shared/shell";
53+
import { mcpToolResultOutput } from "@t3tools/shared/toolOutput";
5354
import type {
5455
ChatAttachment,
5556
OrchestrationV2AppThread,
@@ -490,9 +491,7 @@ function codexMcpToolOutput(
490491
const resultOutput =
491492
item.result === null || item.result === undefined
492493
? undefined
493-
: item.result.structuredContent !== null && item.result.structuredContent !== undefined
494-
? item.result.structuredContent
495-
: item.result.content;
494+
: mcpToolResultOutput(item.result);
496495

497496
if (item.error === null || item.error === undefined) {
498497
return resultOutput;

‎apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts‎

Lines changed: 62 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import * as NodeURL from "node:url";
2+
import { toolOutputImages } from "@t3tools/shared/toolOutput";
23
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
34
import * as NodeServices from "@effect/platform-node/NodeServices";
45
import { assert, it } from "@effect/vitest";
@@ -2715,48 +2716,67 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
27152716
}).pipe(Effect.scoped),
27162717
);
27172718

2718-
it.effect("keeps thread-scoped T3 waits as dynamic tools through their lifecycle", () =>
2719-
Effect.gen(function* () {
2720-
const tool = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call-wait" };
2721-
const name = "t3-code-web_delegate_task";
2722-
const input = { task: "Review", mode: "wait" };
2723-
const { runtime, thread } = yield* resumed([
2724-
out("session.prompt", { sessionID: SESSION, text: "<any>" }),
2725-
promptAccepted,
2726-
event("session.tool.input.started", { ...tool, name }),
2727-
event("session.tool.called", { ...tool, name, input, executed: true }),
2728-
event("session.tool.success", {
2729-
...tool,
2730-
content: [{ type: "text", text: "Reviewed" }],
2731-
metadata: {},
2732-
executed: true,
2733-
}),
2734-
event("session.execution.succeeded", { sessionID: SESSION }),
2735-
]);
2736-
const events = yield* runtime.events.pipe(
2737-
Stream.takeUntil((event) => event.type === "turn.terminal"),
2738-
Stream.runCollect,
2739-
Effect.forkScoped,
2740-
);
2741-
yield* runtime.startTurn(turnInput(thread));
2742-
const items = (yield* Fiber.join(events)).flatMap((event) =>
2743-
event.type === "turn_item.updated" &&
2744-
event.turnItem.nativeItemRef?.nativeId?.endsWith(tool.id)
2745-
? [event.turnItem]
2746-
: [],
2747-
);
2748-
assert.deepEqual(
2749-
items.map((item) => item.type),
2750-
["dynamic_tool", "dynamic_tool", "dynamic_tool"],
2751-
);
2752-
assert.deepEqual(
2753-
items.map((item) => item.status),
2754-
["running", "running", "completed"],
2755-
);
2756-
const last = items.at(-1);
2757-
assert.equal(last?.type === "dynamic_tool" ? last.toolName : null, name);
2758-
assert.deepEqual(last?.type === "dynamic_tool" ? last.input : null, input);
2759-
}).pipe(Effect.scoped),
2719+
it.effect.each(["delegate_task", "html_preview", "mcp__acme__edit_document"])(
2720+
"keeps MCP %s as a dynamic tool through its lifecycle",
2721+
(toolName) =>
2722+
Effect.gen(function* () {
2723+
const tool = { sessionID: SESSION, assistantMessageID: "msg_assistant", id: "call-wait" };
2724+
const name = toolName.startsWith("mcp__") ? toolName : `t3-code-web_${toolName}`;
2725+
const input =
2726+
toolName !== "delegate_task"
2727+
? { html: "<p>Preview</p>" }
2728+
: { task: "Review", mode: "wait" };
2729+
const { runtime, thread } = yield* resumed([
2730+
out("session.prompt", { sessionID: SESSION, text: "<any>" }),
2731+
promptAccepted,
2732+
event("session.tool.input.started", { ...tool, name }),
2733+
event("session.tool.called", { ...tool, name, input, executed: true }),
2734+
event(
2735+
"session.tool.success",
2736+
{
2737+
...tool,
2738+
content: [
2739+
{ type: "text", text: "Reviewed" },
2740+
...(toolName !== "delegate_task"
2741+
? [{ type: "file", uri: "data:image/png;base64,iVBORw0KGgo=", mime: "image/png" }]
2742+
: []),
2743+
],
2744+
metadata: {},
2745+
executed: true,
2746+
},
2747+
undefined,
2748+
2,
2749+
),
2750+
event("session.execution.succeeded", { sessionID: SESSION }),
2751+
]);
2752+
const events = yield* runtime.events.pipe(
2753+
Stream.takeUntil((event) => event.type === "turn.terminal"),
2754+
Stream.runCollect,
2755+
Effect.forkScoped,
2756+
);
2757+
yield* runtime.startTurn(turnInput(thread));
2758+
const items = (yield* Fiber.join(events)).flatMap((event) =>
2759+
event.type === "turn_item.updated" &&
2760+
event.turnItem.nativeItemRef?.nativeId?.endsWith(tool.id)
2761+
? [event.turnItem]
2762+
: [],
2763+
);
2764+
assert.deepEqual(
2765+
items.map((item) => item.type),
2766+
["dynamic_tool", "dynamic_tool", "dynamic_tool"],
2767+
);
2768+
assert.deepEqual(
2769+
items.map((item) => item.status),
2770+
["running", "running", "completed"],
2771+
);
2772+
const last = items.at(-1);
2773+
assert.equal(last?.type === "dynamic_tool" ? last.toolName : null, name);
2774+
assert.deepEqual(last?.type === "dynamic_tool" ? last.input : null, input);
2775+
assert.deepEqual(
2776+
toolOutputImages(last?.type === "dynamic_tool" ? last.output : null),
2777+
toolName !== "delegate_task" ? [{ mimeType: "image/png", data: "iVBORw0KGgo=" }] : [],
2778+
);
2779+
}).pipe(Effect.scoped),
27602780
);
27612781

27622782
it.effect("stores a shell command that exits non-zero as failed", () =>

0 commit comments

Comments
 (0)