Skip to content

Commit 4f27a84

Browse files
t3dotggclaude
andauthored
fix(mcp): preview snapshots fit in the agent's tool output again (pingdotgg#13558)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 9957349 commit 4f27a84

3 files changed

Lines changed: 125 additions & 40 deletions

File tree

‎apps/server/src/mcp/McpHttpServer.test.ts‎

Lines changed: 58 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,10 @@ it.effect.each([
248248
const metadata = { ...page, title: `Snapshot ${call}`, screenshot };
249249
const { accessibilityTree: _tree, ...boundedMetadata } = metadata;
250250
expect(snapshot.isError).toBe(false);
251-
expect(snapshot.structuredContent).toEqual(metadata);
251+
expect(snapshot.structuredContent).toEqual({
252+
...boundedMetadata,
253+
omitted: ["accessibilityTree (use interactiveElements locators or preview_evaluate)"],
254+
});
252255
const [identity, text, ...rest] = snapshot.content;
253256
expect(identity?.type === "text" ? decodeJsonText(identity.text) : null).toEqual({
254257
url: page.url,
@@ -287,7 +290,8 @@ it.effect.each([
287290
"text",
288291
"image",
289292
]);
290-
expect(nextDefault.structuredContent).toEqual({ ...page, title: "Snapshot 7", screenshot });
293+
expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7", screenshot });
294+
expect(nextDefault.structuredContent).not.toHaveProperty("accessibilityTree");
291295
expect(requests).toBe(7);
292296
}),
293297
).pipe(Effect.provide(TestLayer)),
@@ -341,6 +345,15 @@ it.effect("saves the snapshot PNG on request and reports its path", () =>
341345

342346
const unsaved = yield* callSnapshot({});
343347
expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath");
348+
349+
// A save without the image skips the page dump.
350+
const pathOnly = yield* callSnapshot({ save: true, includeImage: false });
351+
const saved = pathOnly.structuredContent as { readonly screenshotPath: string };
352+
expect(saved).toEqual({ url: snapshotResult.url, screenshotPath: expect.any(String) });
353+
expect(Buffer.from(yield* fileSystem.readFile(saved.screenshotPath)).toString()).toBe("png");
354+
const [only, ...others] = pathOnly.content;
355+
expect(others).toEqual([]);
356+
expect(only?.type === "text" ? decodeJsonText(only.text) : null).toEqual(saved);
344357
}),
345358
).pipe(Effect.provide(TestLayer)),
346359
);
@@ -460,9 +473,10 @@ it.effect("keeps the snapshot text under the agent's output ceiling", () =>
460473
expect(parsed.consoleEntries[0]?.text).toBe("entry 60");
461474
expect(notice?.type === "text" ? notice.text : "").toContain("accessibilityTree");
462475
expect(notice?.type === "text" ? notice.text : "").toContain("60 older console entries");
463-
// The structured result is untouched; only the text the agent reads is bounded.
464-
expect(snapshot.structuredContent).toMatchObject({
465-
accessibilityTree: oversized.accessibilityTree,
476+
// Claude Code shows the model structuredContent instead of the text, so it is bounded too.
477+
expect(snapshot.structuredContent).toEqual({
478+
...parsed,
479+
omitted: expect.arrayContaining(["60 older console entries"]),
466480
});
467481
}),
468482
).pipe(Effect.provide(TestLayer)),
@@ -500,6 +514,45 @@ it.effect("bounds the snapshot text even when nothing but logs and the title are
500514
).pipe(Effect.provide(TestLayer)),
501515
);
502516

517+
it.effect("bounds page text made of wide characters before dropping locators", () =>
518+
Effect.scoped(
519+
Effect.gen(function* () {
520+
// The character caps alone leave 8,000 three-byte characters, about 24 KB.
521+
yield* serveSnapshots("mcp-wide-text-client", {
522+
...snapshotResult,
523+
visibleText: "界".repeat(9_000),
524+
interactiveElements: Array.from({ length: 20 }, (_, i) => ({
525+
tag: "button",
526+
role: "button",
527+
name: `Button ${i}`,
528+
selector: `#button-${i}`,
529+
x: 0,
530+
y: 0,
531+
width: 10,
532+
height: 10,
533+
})),
534+
});
535+
536+
const snapshot = yield* callSnapshot({ includeImage: false });
537+
538+
const [, text, notice] = snapshot.content;
539+
const body = text?.type === "text" ? text.text : "";
540+
expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
541+
McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
542+
);
543+
const parsed = decodeJsonText(body) as {
544+
readonly visibleText: string;
545+
readonly interactiveElements: ReadonlyArray<unknown>;
546+
};
547+
expect(parsed.visibleText).toMatch(/^界+…$/);
548+
expect(parsed.interactiveElements).toHaveLength(20);
549+
expect(notice?.type === "text" ? notice.text : "").toContain(
550+
"visibleText after 4000 characters",
551+
);
552+
}),
553+
).pipe(Effect.provide(TestLayer)),
554+
);
555+
503556
it.effect("sheds log entries before locators when every list is full", () =>
504557
Effect.scoped(
505558
Effect.gen(function* () {

‎apps/server/src/mcp/McpHttpServer.ts‎

Lines changed: 65 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -114,12 +114,15 @@ const McpAuthMiddlewareLive = HttpRouter.middleware<{
114114
}>()(makeMcpAuthMiddleware).layer;
115115

116116
/**
117-
* Claude Code drops every MCP result above 25k tokens (~100 KB of text) and
118-
* hands the agent a truncation notice instead, so a snapshot that carries the
119-
* full accessibility tree and 20 KB of page text loses its locators too. Keep
120-
* the text under that ceiling and tell the agent what was cut.
117+
* Claude Code moves an MCP result above its output limit to a file and hands
118+
* the agent a notice instead, so a snapshot that carries the full
119+
* accessibility tree and page text loses its locators too. Claude Code also
120+
* shows the model `structuredContent` in place of the text blocks when a
121+
* result has both, so both carry the same bounded snapshot. Keep it near
122+
* 20 KB and tell the agent what was cut. The short `omitted` notes may go a
123+
* little over; the provider limit is far above this.
121124
*/
122-
export const MAX_SNAPSHOT_TEXT_BYTES = 60_000;
125+
export const MAX_SNAPSHOT_TEXT_BYTES = 20_000;
123126
const MAX_SNAPSHOT_VISIBLE_TEXT_CHARS = 8_000;
124127
const MAX_SNAPSHOT_ELEMENT_NAME_CHARS = 200;
125128
const MAX_SNAPSHOT_LOG_ENTRIES = 40;
@@ -164,12 +167,11 @@ type SnapshotMetadata = {
164167
/**
165168
* Drops the accessibility tree, shortens page text, element names, identifiers,
166169
* and log strings, keeps only the newest log entries, and finally sheds
167-
* interactive elements until the JSON fits. Returns the text plus notes on
168-
* what is missing so the agent can reach for preview_evaluate.
170+
* interactive elements until the JSON fits. Returns the bounded value, its
171+
* text, and notes on what is missing so the agent can reach for
172+
* preview_evaluate.
169173
*/
170-
const boundSnapshotMetadata = (
171-
metadata: SnapshotMetadata,
172-
): { readonly text: string; readonly omitted: ReadonlyArray<string> } => {
174+
const boundSnapshotMetadata = (metadata: SnapshotMetadata) => {
173175
const omitted: Array<string> = [];
174176
const { accessibilityTree, ...withoutTree } = metadata;
175177
if (accessibilityTree !== undefined) {
@@ -198,16 +200,10 @@ const boundSnapshotMetadata = (
198200
) {
199201
omitted.push(`element names longer than ${MAX_SNAPSHOT_ELEMENT_NAME_CHARS} characters`);
200202
}
201-
if (metadata.visibleText.length > MAX_SNAPSHOT_VISIBLE_TEXT_CHARS) {
202-
omitted.push(
203-
`visibleText after ${MAX_SNAPSHOT_VISIBLE_TEXT_CHARS} characters (use preview_evaluate for more)`,
204-
);
205-
}
206203
const bounded = {
207204
...withoutTree,
208205
url: cutText(metadata.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
209206
title: cutText(metadata.title, MAX_SNAPSHOT_IDENTIFIER_CHARS),
210-
visibleText: cutText(metadata.visibleText, MAX_SNAPSHOT_VISIBLE_TEXT_CHARS),
211207
interactiveElements: metadata.interactiveElements.map((element) => ({
212208
...element,
213209
name: cutText(element.name, MAX_SNAPSHOT_ELEMENT_NAME_CHARS),
@@ -218,9 +214,10 @@ const boundSnapshotMetadata = (
218214
};
219215

220216
// Per-field caps do not sum below the ceiling: three log arrays of 40 capped
221-
// entries alone can pass 60 KB. Shed the least useful lists first, halving
222-
// one list per round, until the JSON fits. With every list empty the rest
223-
// is bounded by the identifier and visibleText caps, so this terminates.
217+
// entries alone can pass 60 KB, and the caps count characters, not bytes.
218+
// Halve one thing per round until the JSON fits: logs first, then page
219+
// text, then the locators. The identifier caps bound the rest, so this
220+
// terminates.
224221
const shedOrder = [
225222
"actionTimeline",
226223
"networkEntries",
@@ -239,31 +236,51 @@ const boundSnapshotMetadata = (
239236
networkEntries: 0,
240237
actionTimeline: 0,
241238
};
242-
let text = encodeJsonText({ ...bounded, ...lists });
239+
let visibleTextChars = Math.min(metadata.visibleText.length, MAX_SNAPSHOT_VISIBLE_TEXT_CHARS);
240+
const value = () => ({
241+
...bounded,
242+
visibleText: cutText(metadata.visibleText, visibleTextChars),
243+
...lists,
244+
});
245+
let text = encodeJsonText(value());
243246
while (utf8Length(text) > MAX_SNAPSHOT_TEXT_BYTES) {
244247
// Elements carry the locators, so they go last; logs shed newest-last.
245248
const key =
246249
shedOrder.find(
247250
(candidate) => candidate !== "interactiveElements" && lists[candidate].length > 0,
248-
) ?? (lists.interactiveElements.length > 0 ? "interactiveElements" : undefined);
251+
) ??
252+
(visibleTextChars > 0
253+
? "visibleText"
254+
: lists.interactiveElements.length > 0
255+
? "interactiveElements"
256+
: undefined);
249257
if (key === undefined) break;
250-
const keep = Math.floor(lists[key].length / 2);
251-
dropped[key] += lists[key].length - keep;
252-
// slice(-0) keeps everything, so spell out the empty case.
253-
lists[key] =
254-
keep === 0
255-
? []
256-
: key === "interactiveElements"
257-
? lists[key].slice(0, keep)
258-
: lists[key].slice(-keep);
259-
text = encodeJsonText({ ...bounded, ...lists });
258+
if (key === "visibleText") {
259+
visibleTextChars = Math.floor(visibleTextChars / 2);
260+
} else {
261+
const keep = Math.floor(lists[key].length / 2);
262+
dropped[key] += lists[key].length - keep;
263+
// slice(-0) keeps everything, so spell out the empty case.
264+
lists[key] =
265+
keep === 0
266+
? []
267+
: key === "interactiveElements"
268+
? lists[key].slice(0, keep)
269+
: lists[key].slice(-keep);
270+
}
271+
text = encodeJsonText(value());
272+
}
273+
if (visibleTextChars < metadata.visibleText.length) {
274+
omitted.push(
275+
`visibleText after ${visibleTextChars} characters (use preview_evaluate for more)`,
276+
);
260277
}
261278
for (const key of shedOrder) {
262279
if (dropped[key] > 0) {
263280
omitted.push(`${dropped[key]} of ${bounded[key].length} ${key}`);
264281
}
265282
}
266-
return { text, omitted };
283+
return { value: value(), text, omitted };
267284
};
268285

269286
export class PreviewScreenshotSaveError extends Schema.TaggedError<PreviewScreenshotSaveError>()(
@@ -396,6 +413,18 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
396413
const png = new Uint8Array(Buffer.from(screenshot.data, "base64"));
397414
const screenshotPath =
398415
payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined;
416+
if (screenshotPath !== undefined && payload?.includeImage === false) {
417+
// The agent only wants a file to show the user. The url keeps the site icon on the tool row.
418+
const saved = {
419+
url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
420+
screenshotPath,
421+
};
422+
return new McpSchema.CallToolResult({
423+
isError: false,
424+
structuredContent: saved,
425+
content: [{ type: "text", text: encodeJsonText(saved) }],
426+
});
427+
}
399428
const metadata = {
400429
...page,
401430
screenshot: {
@@ -408,7 +437,10 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
408437
const bounded = boundSnapshotMetadata(metadata);
409438
return new McpSchema.CallToolResult({
410439
isError: false,
411-
structuredContent: metadata,
440+
structuredContent:
441+
bounded.omitted.length === 0
442+
? bounded.value
443+
: { ...bounded.value, omitted: bounded.omitted },
412444
content: [
413445
// Keep the page identity readable even if a provider truncates the snapshot.
414446
{

‎apps/server/src/mcp/toolkits/preview/tools.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ const PreviewSetAppearanceTool = safeBrowserTool(
119119
export const PreviewSnapshotTool = readonlyBrowserTool(
120120
Tool.make("preview_snapshot", {
121121
description:
122-
"Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",
122+
"Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",
123123
parameters: Schema.Struct({
124124
...PreviewAutomationTabTargetInput.fields,
125125
includeImage: Schema.optional(
@@ -131,7 +131,7 @@ export const PreviewSnapshotTool = readonlyBrowserTool(
131131
save: Schema.optional(
132132
Schema.Boolean.annotate({
133133
description:
134-
"Write the screenshot PNG to disk and return its absolute path as screenshotPath. Defaults to false.",
134+
"Write the screenshot PNG to disk and return its absolute path as screenshotPath. With includeImage=false, return only the url and screenshotPath. Defaults to false.",
135135
}),
136136
),
137137
}),

0 commit comments

Comments
 (0)