Skip to content

Commit 5eb8773

Browse files
t3dotggclaude
andauthored
fix: tool screenshots show as images, not base64 text (#16199)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 2a68a77 commit 5eb8773

17 files changed

Lines changed: 551 additions & 21 deletions

File tree

‎apps/mobile/src/features/threads/ThreadFeed.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1933,6 +1933,7 @@ function renderFeedEntry(
19331933
onToggleRow={props.onToggleWorkRow}
19341934
renderImage={props.renderViewedImage}
19351935
renderReasoning={props.renderReasoning}
1936+
onPressPreview={props.onPressPreview}
19361937
/>
19371938
);
19381939
}

‎apps/mobile/src/features/threads/ThreadMarkdownImage.tsx‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,10 @@ function ThreadMarkdownImageRequest(props: {
178178
/** Environment-hosted image that loads through a signed asset URL. */
179179
export function ThreadMarkdownImage(props: {
180180
readonly environmentId: EnvironmentId;
181-
readonly resource: Extract<AssetResource, { readonly _tag: "attachment" | "media-file" }>;
181+
readonly resource: Extract<
182+
AssetResource,
183+
{ readonly _tag: "attachment" | "media-file" | "tool-output-image" }
184+
>;
182185
readonly alt: string | null;
183186
readonly srcFragment?: string;
184187
readonly actionsSource?: MediaActionsSource;
@@ -192,7 +195,9 @@ export function ThreadMarkdownImage(props: {
192195
sourceKey={
193196
props.resource._tag === "attachment"
194197
? `attachment:${props.resource.attachmentId}`
195-
: `workspace:${props.resource.path}`
198+
: props.resource._tag === "tool-output-image"
199+
? `tool-output:${props.resource.itemId}:${props.resource.index}`
200+
: `workspace:${props.resource.path}`
196201
}
197202
unavailable={assetUrl._tag === "Failure"}
198203
knownSize={assetUrl._tag === "Success" ? assetUrl.imageDimensions : undefined}

‎apps/mobile/src/features/threads/thread-work-log.tsx‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,11 @@ import {
6464
type ThreadFeedActivity,
6565
workEntryRowLabel,
6666
} from "../../lib/threadActivity";
67-
import { toolCallLines, turnItemOutputText } from "@t3tools/client-runtime/work-log/item-detail";
67+
import {
68+
toolCallLines,
69+
turnItemOutputImages,
70+
turnItemOutputText,
71+
} from "@t3tools/client-runtime/work-log/item-detail";
6872
import { useTurnItemDetail } from "../../state/queries";
6973
import {
7074
resolveThreadWorkGroupInitialScroll,
@@ -80,6 +84,8 @@ import {
8084
import { resolveWorkGroupScrollAnchor } from "@t3tools/client-runtime/work-log/scroll-anchor";
8185
import { notificationChildThreadId } from "@t3tools/client-runtime/state/thread-execution";
8286
import type { MarkdownImageRenderer } from "../../native/SelectableMarkdownText";
87+
import type { FilePreviewSource } from "../../components/FilePreviewModal";
88+
import { ThreadMarkdownImage } from "./ThreadMarkdownImage";
8389
import Animated, {
8490
cancelAnimation,
8591
Easing,
@@ -462,6 +468,7 @@ interface ThreadWorkLogProps {
462468
readonly onToggleRow: (rowId: string, anchorKey: string) => void;
463469
readonly renderImage: MarkdownImageRenderer;
464470
readonly renderReasoning: (text: string) => ReactNode;
471+
readonly onPressPreview: (source: FilePreviewSource) => void;
465472
}
466473

467474
export function ThreadWorkLog(props: ThreadWorkLogProps) {
@@ -479,6 +486,7 @@ export function ThreadWorkLog(props: ThreadWorkLogProps) {
479486
onToggleRow={props.onToggleRow}
480487
renderImage={props.renderImage}
481488
renderReasoning={props.renderReasoning}
489+
onPressPreview={props.onPressPreview}
482490
themeAppearance={props.themeAppearance}
483491
/>
484492
),
@@ -492,6 +500,7 @@ export function ThreadWorkLog(props: ThreadWorkLogProps) {
492500
props.onToggleRow,
493501
props.renderImage,
494502
props.renderReasoning,
503+
props.onPressPreview,
495504
props.themeAppearance,
496505
],
497506
);
@@ -951,12 +960,13 @@ const ThreadWorkLogRow = memo(function ThreadWorkLogRow(
951960
? formatItemFullDetail(row.projectedItem, fetchedItem)
952961
: row.getFullDetail()
953962
: null;
963+
const outputImages = expanded && fetchedItem ? turnItemOutputImages(fetchedItem) : [];
954964
const fetchedOutput = !expanded
955965
? null
956966
: shownItem.type === "file_search" || shownItem.type === "web_search"
957967
? turnItemOutputText(shownItem)
958968
: fetchedItem
959-
? (turnItemOutputText(fetchedItem) ?? "No output.")
969+
? (turnItemOutputText(fetchedItem) ?? (outputImages.length > 0 ? null : "No output."))
960970
: fetchedDetail.error
961971
? `Couldn't load output: ${fetchedDetail.error}`
962972
: row.fetchesDetail
@@ -1116,6 +1126,7 @@ const ThreadWorkLogRow = memo(function ThreadWorkLogRow(
11161126
call ||
11171127
fetchedOutput ||
11181128
viewedImagePath ||
1129+
outputImages.length > 0 ||
11191130
row.workEntry.questionAnswer) ? (
11201131
<Animated.View
11211132
entering={WORK_LOG_DETAIL_ENTER_TRANSITION}
@@ -1134,6 +1145,16 @@ const ThreadWorkLogRow = memo(function ThreadWorkLogRow(
11341145
{props.renderImage({ href: viewedImagePath, alt: null, title: null })}
11351146
</View>
11361147
) : null}
1148+
{outputImages.map((resource) => (
1149+
<View key={resource.index} className="pb-1.5">
1150+
<ThreadMarkdownImage
1151+
environmentId={props.environmentId}
1152+
resource={resource}
1153+
alt={null}
1154+
onPressPreview={props.onPressPreview}
1155+
/>
1156+
</View>
1157+
))}
11371158
<ScrollView
11381159
nestedScrollEnabled
11391160
directionalLockEnabled

‎apps/server/src/assets/AssetAccess.test.ts‎

Lines changed: 99 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,16 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
33
import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform";
44
import * as NodeFSP from "node:fs/promises";
55
import * as NodeOS from "node:os";
6-
import { AssetAccessError, AssetPreviewTypeValidationError, ThreadId } from "@t3tools/contracts";
6+
import {
7+
AssetAccessError,
8+
AssetPreviewTypeValidationError,
9+
ThreadId,
10+
TurnItemId,
11+
} from "@t3tools/contracts";
712
import { PROJECT_FAVICON_FALLBACK_MARKER } from "@t3tools/shared/projectFavicon";
813
import { describe, expect, it } from "@effect/vitest";
914
import * as Crypto from "effect/Crypto";
15+
import * as DateTime from "effect/DateTime";
1016
import * as Effect from "effect/Effect";
1117
import * as FileSystem from "effect/FileSystem";
1218
import * as Layer from "effect/Layer";
@@ -20,6 +26,7 @@ import { vi } from "vite-plus/test";
2026

2127
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
2228
import * as ServerConfig from "../config.ts";
29+
import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
2330
import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts";
2431
import * as T3ProjectFileLoader from "../project/T3ProjectFileLoader.ts";
2532
import * as WorkspacePaths from "../workspace/WorkspacePaths.ts";
@@ -44,8 +51,70 @@ vi.mock("node:os", async (importOriginal) => {
4451
const configLayer = ServerConfig.ServerConfig.layerTest(process.cwd(), {
4552
prefix: "t3-asset-access-test-",
4653
});
54+
// A PNG header is enough for the dimension read: signature, then IHDR width and height.
55+
const screenshotPng = new Uint8Array(24);
56+
screenshotPng.set([
57+
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13, 0x49, 0x48, 0x44, 0x52,
58+
]);
59+
new DataView(screenshotPng.buffer).setUint32(16, 390);
60+
new DataView(screenshotPng.buffer).setUint32(20, 844);
61+
const screenshotItem = {
62+
id: TurnItemId.make("tool-screenshot"),
63+
type: "dynamic_tool" as const,
64+
threadId: ThreadId.make("thread-1"),
65+
runId: null,
66+
nodeId: null,
67+
providerThreadId: null,
68+
providerTurnId: null,
69+
nativeItemRef: null,
70+
parentItemId: null,
71+
ordinal: 1,
72+
status: "completed" as const,
73+
title: null,
74+
toolName: "mcp__t3-code__device_screenshot",
75+
input: { deviceId: "phone" },
76+
output: {
77+
content: [
78+
{
79+
type: "image",
80+
source: {
81+
type: "base64",
82+
media_type: "image/png",
83+
data: Buffer.from(screenshotPng).toString("base64"),
84+
},
85+
},
86+
],
87+
},
88+
startedAt: DateTime.makeUnsafe("2026-10-05T00:00:00.000Z"),
89+
completedAt: DateTime.makeUnsafe("2026-10-05T00:00:01.000Z"),
90+
updatedAt: DateTime.makeUnsafe("2026-10-05T00:00:01.000Z"),
91+
};
92+
93+
const oversizedScreenshotItem = {
94+
...screenshotItem,
95+
id: TurnItemId.make("tool-screenshot-oversized"),
96+
output: {
97+
content: [
98+
{
99+
type: "image",
100+
source: { type: "base64", media_type: "image/png", data: "A".repeat(14 * 1024 * 1024) },
101+
},
102+
],
103+
},
104+
};
105+
47106
const testLayer = Layer.mergeAll(
48107
NodeHttpPlatform.layer,
108+
Layer.mock(Orchestrator.OrchestratorV2)({
109+
getTurnItem: ({ itemId }) =>
110+
Effect.succeed(
111+
itemId === screenshotItem.id
112+
? screenshotItem
113+
: itemId === oversizedScreenshotItem.id
114+
? oversizedScreenshotItem
115+
: null,
116+
),
117+
}),
49118
configLayer,
50119
WorkspacePaths.layer,
51120
ProjectFaviconResolver.layer.pipe(
@@ -103,6 +172,35 @@ describe("AssetAccess", () => {
103172
);
104173
});
105174

175+
it.effect("serves an image a tool returned inline from the stored item", () =>
176+
Effect.gen(function* () {
177+
const resource = {
178+
_tag: "tool-output-image" as const,
179+
threadId: screenshotItem.threadId,
180+
itemId: screenshotItem.id,
181+
index: 0,
182+
};
183+
const result = yield* issueAssetUrl({ resource });
184+
expect(result.imageDimensions).toEqual({ width: 390, height: 844 });
185+
const suffix = result.relativeUrl.slice(`${ASSET_ROUTE_PREFIX}/`.length);
186+
const separator = suffix.indexOf("/");
187+
expect(yield* resolveAsset(suffix.slice(0, separator), suffix.slice(separator + 1))).toEqual({
188+
kind: "bytes",
189+
mimeType: "image/png",
190+
bytes: Buffer.from(screenshotPng),
191+
});
192+
const missing = yield* issueAssetUrl({ resource: { ...resource, index: 1 } }).pipe(
193+
Effect.flip,
194+
);
195+
expect(missing._tag).toBe("AssetWorkspaceAssetNotFoundError");
196+
// Larger than a provider turn accepts, so it is never decoded.
197+
const oversized = yield* issueAssetUrl({
198+
resource: { ...resource, itemId: oversizedScreenshotItem.id },
199+
}).pipe(Effect.flip);
200+
expect(oversized._tag).toBe("AssetWorkspaceAssetNotFoundError");
201+
}).pipe(Effect.provide(testLayer)),
202+
);
203+
106204
it.effect("issues exact URLs for media and browser documents outside the workspace", () =>
107205
Effect.gen(function* () {
108206
const fs = yield* FileSystem.FileSystem;

‎apps/server/src/assets/AssetAccess.ts‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import type { AssetResource } from "@t3tools/contracts";
22
import {
33
AssetAttachmentNotFoundError,
4+
PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
45
AssetGitHubMediaUrlValidationError,
56
AssetPreviewTypeValidationError,
67
AssetProjectFaviconInspectionError,
@@ -13,7 +14,9 @@ import {
1314
AssetWorkspacePathValidationError,
1415
AssetWorkspaceResolutionError,
1516
AssetWorkspaceRootNormalizationError,
17+
ThreadId,
1618
ToolActivityNativeAppReference,
19+
TurnItemId,
1720
} from "@t3tools/contracts";
1821
import {
1922
audioMimeTypeFromExtension,
@@ -30,6 +33,7 @@ import {
3033
} from "@t3tools/shared/imageDimensions";
3134
import { githubMediaFetchUrl, githubMediaFileName } from "@t3tools/shared/githubMedia";
3235
import { PROJECT_FAVICON_FALLBACK_MARKER } from "@t3tools/shared/projectFavicon";
36+
import { toolOutputImages } from "@t3tools/shared/toolOutput";
3337
import * as Clock from "effect/Clock";
3438
import * as Crypto from "effect/Crypto";
3539
import * as Effect from "effect/Effect";
@@ -49,6 +53,7 @@ import {
4953
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
5054
import { parseAttachmentFileExtension, resolveAttachmentPathById } from "../attachmentStore.ts";
5155
import * as ServerConfig from "../config.ts";
56+
import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
5257
import { expandHomePathWith } from "../pathExpansion.ts";
5358
import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts";
5459
import * as WorkspacePaths from "../workspace/WorkspacePaths.ts";
@@ -138,6 +143,14 @@ const AssetClaimsSchema = Schema.Union([
138143
app: ToolActivityNativeAppReference,
139144
expiresAt: Schema.Number,
140145
}),
146+
Schema.Struct({
147+
version: Schema.Literal(1),
148+
kind: Schema.Literal("tool-output-image"),
149+
threadId: ThreadId,
150+
itemId: TurnItemId,
151+
index: Schema.Number,
152+
expiresAt: Schema.Number,
153+
}),
141154
Schema.Struct({
142155
version: Schema.Literal(1),
143156
kind: Schema.Literal("github-media"),
@@ -162,6 +175,11 @@ export type ResolvedAsset =
162175
readonly mimeType?: string;
163176
readonly file?: OpenMediaFile;
164177
}
178+
| {
179+
readonly kind: "bytes";
180+
readonly bytes: Uint8Array;
181+
readonly mimeType: string;
182+
}
165183
| {
166184
readonly kind: "github-media";
167185
readonly url: string;
@@ -198,6 +216,27 @@ const optionOnNotFound = <A, R>(
198216
}),
199217
);
200218

219+
// The largest image a provider turn accepts, as base64 (4 characters per 3 bytes).
220+
const MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH = Math.ceil(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / 3) * 4;
221+
222+
/**
223+
* Decodes one image a tool returned inline; null when the stored item has no
224+
* such image, or it is larger than a provider turn accepts.
225+
*/
226+
const readToolOutputImage = Effect.fn("AssetAccess.readToolOutputImage")(function* (input: {
227+
readonly threadId: ThreadId;
228+
readonly itemId: TurnItemId;
229+
readonly index: number;
230+
}) {
231+
const orchestrator = yield* Orchestrator.OrchestratorV2;
232+
const item = yield* orchestrator.getTurnItem(input);
233+
const image =
234+
item?.type === "dynamic_tool" ? toolOutputImages(item.output)[input.index] : undefined;
235+
return image?.data === undefined || image.data.length > MAX_TOOL_OUTPUT_IMAGE_BASE64_LENGTH
236+
? null
237+
: { mimeType: image.mimeType, bytes: Buffer.from(image.data, "base64") };
238+
});
239+
201240
const resolveCanonicalFile = Effect.fn("AssetAccess.resolveCanonicalFile")(function* (
202241
filePath: string,
203242
) {
@@ -672,6 +711,28 @@ export const issueAssetUrl = Effect.fn("AssetAccess.issueAssetUrl")(function* (i
672711
}
673712
break;
674713
}
714+
case "tool-output-image": {
715+
const image = yield* readToolOutputImage(input.resource).pipe(
716+
Effect.mapError(
717+
(cause) => new AssetWorkspaceAssetInspectionError({ resource: input.resource, cause }),
718+
),
719+
);
720+
if (image === null) {
721+
return yield* new AssetWorkspaceAssetNotFoundError({ resource: input.resource });
722+
}
723+
claims = {
724+
version: 1,
725+
kind: "tool-output-image",
726+
threadId: input.resource.threadId,
727+
itemId: input.resource.itemId,
728+
index: input.resource.index,
729+
expiresAt,
730+
};
731+
// The allowed types are all `image/<extension>`.
732+
fileName = `image-${input.resource.index + 1}.${image.mimeType.slice("image/".length)}`;
733+
imageDimensions = readImageDimensions(image.bytes);
734+
break;
735+
}
675736
case "native-app-icon": {
676737
claims = {
677738
version: 1,
@@ -806,6 +867,20 @@ export const resolveAsset = Effect.fn("AssetAccess.resolveAsset")(function* (
806867
} satisfies ResolvedAsset;
807868
}
808869

870+
if (claims.kind === "tool-output-image") {
871+
const image = yield* readToolOutputImage(claims).pipe(
872+
Effect.tapError((cause) =>
873+
Effect.logError("Failed to read tool output image.", {
874+
threadId: claims.threadId,
875+
itemId: claims.itemId,
876+
cause,
877+
}),
878+
),
879+
Effect.orElseSucceed(() => null),
880+
);
881+
return image ? ({ kind: "bytes", ...image } satisfies ResolvedAsset) : null;
882+
}
883+
809884
if (claims.kind === "native-app-icon") {
810885
const nativeAppIconResolver = yield* NativeAppIconResolver.NativeAppIconResolver;
811886
const iconPath = yield* nativeAppIconResolver.resolve(claims.app);

0 commit comments

Comments
 (0)