Skip to content

Commit 7a64017

Browse files
t3dotggclaude
andcommitted
fix(mcp): keep preview snapshots usable by the agent and let it save them
preview_snapshot text grew past Claude Code's 25k token MCP ceiling, so the harness replaced the whole result with a truncation notice. preview_evaluate returned arrays as structuredContent, which the harness rejects as malformed. Neither tool wrote a file the agent could embed for the user. Bound the snapshot text, wrap evaluate results in {value}, cap element names at the source, and add save=true to write the PNG to browser-artifacts. Co-Authored-By: Claude Fable 5.1 <[email protected]>
1 parent 8b2838e commit 7a64017

6 files changed

Lines changed: 428 additions & 67 deletions

File tree

‎apps/desktop/src/preview/Manager.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,12 @@ const ZOOM_EPSILON = 0.001;
109109
const MAX_EVALUATION_BYTES = 64_000;
110110
const MAX_VISIBLE_TEXT_LENGTH = 20_000;
111111
const MAX_INTERACTIVE_ELEMENTS = 200;
112+
/**
113+
* A `[role]` container's innerText is its whole subtree, which turned one
114+
* snapshot's element list into 60 KB of repeated page text. Names are labels,
115+
* not content, so cap them where they are read.
116+
*/
117+
const MAX_INTERACTIVE_ELEMENT_NAME_LENGTH = 200;
112118
const MAX_SCREENSHOT_WIDTH = 1280;
113119
/** How long an armed tab keeps the exclusive display-media slot before another tab may take it. */
114120
const RECORDING_ARM_GRACE_MS = 10_000;
@@ -3538,7 +3544,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
35383544
return {
35393545
tag: element.tagName.toLowerCase(),
35403546
role: element.getAttribute("role"),
3541-
name: element.getAttribute("aria-label") || element.innerText || element.getAttribute("name") || "",
3547+
name: (element.getAttribute("aria-label") || element.innerText || element.getAttribute("name") || "").slice(0, ${MAX_INTERACTIVE_ELEMENT_NAME_LENGTH}),
35423548
selector: selectorFor(element),
35433549
x: rect.x,
35443550
y: rect.y,

‎apps/server/src/config.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ export interface ServerDerivedPaths {
3838
readonly providerStatusCacheDir: string;
3939
readonly worktreesDir: string;
4040
readonly attachmentsDir: string;
41+
/** Screenshots the agent asks the collaborative browser to keep for the user. */
42+
readonly browserArtifactsDir: string;
4143
readonly logsDir: string;
4244
readonly serverLogPath: string;
4345
readonly serverTracePath: string;
@@ -125,6 +127,7 @@ export const deriveServerPaths = Effect.fn(function* (
125127
providerStatusCacheDir,
126128
worktreesDir: join(baseDir, "worktrees"),
127129
attachmentsDir,
130+
browserArtifactsDir: join(stateDir, "browser-artifacts"),
128131
logsDir,
129132
serverLogPath: join(logsDir, "server.log"),
130133
serverTracePath: join(logsDir, "server.trace.ndjson"),

‎apps/server/src/mcp/McpHttpServer.test.ts‎

Lines changed: 217 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@ import * as NodeServices from "@effect/platform-node/NodeServices";
44
import { EnvironmentId, PreviewTabId, ProviderInstanceId, ThreadId } from "@t3tools/contracts";
55
import * as Deferred from "effect/Deferred";
66
import * as Effect from "effect/Effect";
7+
import * as FileSystem from "effect/FileSystem";
78
import * as Layer from "effect/Layer";
8-
import * as Schema from "effect/Schema";
9+
import * as Path from "effect/Path";
910
import * as Stream from "effect/Stream";
1011
import { McpProtocol, McpSchema, McpServer } from "effect/unstable/ai";
1112
import { HttpBody, HttpClient, HttpRouter, HttpServerResponse } from "effect/unstable/http";
1213

14+
import * as ServerConfig from "../config.ts";
1315
import * as McpHttpServer from "./McpHttpServer.ts";
1416
import * as McpInvocationContext from "./McpInvocationContext.ts";
1517
import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts";
@@ -18,7 +20,6 @@ const environmentId = EnvironmentId.make("environment-mcp-test");
1820
const threadId = ThreadId.make("thread-mcp-test");
1921
const tabId = PreviewTabId.make("tab-mcp-test");
2022
const alternateTabId = PreviewTabId.make("tab-mcp-alternate");
21-
const encodeJsonText = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
2223
const invocation = {
2324
environmentId,
2425
threadId,
@@ -39,9 +40,62 @@ const client = McpSchema.McpServerClient.of({
3940
});
4041
const TestLayer = McpHttpServer.PreviewToolkitRegistrationLive.pipe(
4142
Layer.provideMerge(McpServer.McpServer.layer),
42-
Layer.provideMerge(PreviewAutomationBroker.layer.pipe(Layer.provide(NodeServices.layer))),
43+
Layer.provideMerge(PreviewAutomationBroker.layer),
44+
Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-mcp-http-server-test-" })),
45+
Layer.provideMerge(NodeServices.layer),
4346
);
4447

48+
const snapshotResult = {
49+
url: "http://example.test/",
50+
title: "Example",
51+
loading: false,
52+
visibleText: "Example",
53+
interactiveElements: [],
54+
accessibilityTree: {},
55+
consoleEntries: [],
56+
networkEntries: [],
57+
actionTimeline: [],
58+
screenshot: {
59+
mimeType: "image/png",
60+
data: Buffer.from("png").toString("base64"),
61+
width: 10,
62+
height: 5,
63+
},
64+
};
65+
66+
/** Answers every snapshot request on a fresh broker host with the given result. */
67+
const serveSnapshots = (clientId: string, result: unknown) =>
68+
Effect.gen(function* () {
69+
const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
70+
const connected = yield* Deferred.make<void>();
71+
const inputs: Array<unknown> = [];
72+
const events = yield* broker.connect({ clientId, environmentId });
73+
yield* Stream.runForEach(events, (event) => {
74+
if (event.type === "connected") return Deferred.succeed(connected, undefined);
75+
inputs.push(event.request.input);
76+
return broker.respond({
77+
clientId,
78+
connectionId: event.connectionId,
79+
requestId: event.request.requestId,
80+
ok: true,
81+
result,
82+
});
83+
}).pipe(Effect.forkScoped);
84+
yield* Deferred.await(connected);
85+
return inputs;
86+
});
87+
88+
const callSnapshot = (args: Record<string, unknown>) =>
89+
Effect.gen(function* () {
90+
const server = yield* McpServer.McpServer;
91+
return yield* server
92+
.callTool({ name: "preview_snapshot", arguments: args })
93+
.pipe(
94+
Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
95+
Effect.provideService(McpSchema.McpServerClient, client),
96+
);
97+
});
98+
4599
it("normalizes empty successful notification responses to accepted", () => {
46100
const notificationResponse = McpHttpServer.normalizeMcpHttpResponse(
47101
HttpServerResponse.text("", { status: 200, contentType: "application/json" }),
@@ -90,7 +144,9 @@ it.effect.each([{}, { includeImage: false }])(
90144
);
91145

92146
expect(snapshot.isError).toBe(true);
93-
expect(snapshot.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]);
147+
expect(snapshot.content).toEqual([
148+
{ type: "text", text: "Preview snapshot failed: PreviewAutomationExecutionError." },
149+
]);
94150
expect(snapshot.structuredContent).toEqual({
95151
error: {
96152
_tag: "PreviewAutomationExecutionError",
@@ -172,10 +228,16 @@ it.effect.each([
172228
Effect.provideService(McpSchema.McpServerClient, client),
173229
);
174230
const metadata = { ...page, title: `Snapshot ${call}`, screenshot };
231+
const { accessibilityTree: _tree, ...boundedMetadata } = metadata;
175232
expect(snapshot.isError).toBe(false);
176233
expect(snapshot.structuredContent).toEqual(metadata);
177-
expect(snapshot.content).toEqual([
178-
{ type: "text", text: encodeJsonText(snapshot.structuredContent) },
234+
const [text, ...rest] = snapshot.content;
235+
expect(text?.type === "text" ? JSON.parse(text.text) : null).toEqual(boundedMetadata);
236+
expect(rest).toEqual([
237+
{
238+
type: "text",
239+
text: "Snapshot text was bounded. Omitted: accessibilityTree (use interactiveElements locators or preview_evaluate).",
240+
},
179241
...(images
180242
? [
181243
{
@@ -198,7 +260,7 @@ it.effect.each([
198260
Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
199261
Effect.provideService(McpSchema.McpServerClient, client),
200262
);
201-
expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "image"]);
263+
expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "text", "image"]);
202264
expect(nextDefault.structuredContent).toEqual({ ...page, title: "Snapshot 7", screenshot });
203265
expect(requests).toBe(7);
204266
}),
@@ -219,14 +281,132 @@ it.effect("rejects non-boolean snapshot image options before selecting a browser
219281
Effect.provideService(McpSchema.McpServerClient, client),
220282
);
221283
expect(result.isError).toBe(true);
222-
expect(result.content).toEqual([{ type: "text", text: "Preview snapshot failed." }]);
284+
expect(result.content).toEqual([{ type: "text", text: "Preview snapshot failed: AiError." }]);
223285
expect(result.structuredContent).toEqual({
224286
error: { _tag: "AiError", operation: "snapshot", failureCount: 1 },
225287
});
226288
}
227289
}).pipe(Effect.provide(TestLayer)),
228290
);
229291

292+
it.effect("saves the snapshot PNG on request and reports its path", () =>
293+
Effect.scoped(
294+
Effect.gen(function* () {
295+
const config = yield* ServerConfig.ServerConfig;
296+
const fileSystem = yield* FileSystem.FileSystem;
297+
const path = yield* Path.Path;
298+
const inputs = yield* serveSnapshots("mcp-save-client", snapshotResult);
299+
300+
const snapshot = yield* callSnapshot({ save: true });
301+
302+
expect(snapshot.isError).toBe(false);
303+
// The browser never receives the server-only `save` flag.
304+
expect(inputs).toEqual([{}]);
305+
const structured = snapshot.structuredContent as { readonly screenshotPath?: string };
306+
const screenshotPath = structured.screenshotPath;
307+
expect(typeof screenshotPath).toBe("string");
308+
expect(path.dirname(screenshotPath!)).toBe(config.browserArtifactsDir);
309+
expect(path.basename(screenshotPath!)).toMatch(
310+
/^browser-screenshot-example-test-[0-9a-z]+\.png$/,
311+
);
312+
expect(Buffer.from(yield* fileSystem.readFile(screenshotPath!)).toString()).toBe("png");
313+
const text = snapshot.content.find((content) => content.type === "text");
314+
expect(text?.type === "text" ? text.text : "").toContain(screenshotPath);
315+
316+
const unsaved = yield* callSnapshot({});
317+
expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath");
318+
}),
319+
).pipe(Effect.provide(TestLayer)),
320+
);
321+
322+
it.effect("reports a tagged error when the screenshot cannot be saved", () =>
323+
Effect.scoped(
324+
Effect.gen(function* () {
325+
const config = yield* ServerConfig.ServerConfig;
326+
const fileSystem = yield* FileSystem.FileSystem;
327+
// A regular file where the artifacts directory should be makes every write fail.
328+
yield* fileSystem.writeFileString(config.browserArtifactsDir, "");
329+
yield* serveSnapshots("mcp-save-failure-client", snapshotResult);
330+
331+
const snapshot = yield* callSnapshot({ save: true });
332+
333+
expect(snapshot.isError).toBe(true);
334+
expect(snapshot.content).toEqual([
335+
{ type: "text", text: "Preview snapshot failed: PreviewScreenshotSaveError." },
336+
]);
337+
expect(snapshot.structuredContent).toEqual({
338+
error: { _tag: "PreviewScreenshotSaveError", operation: "snapshot", failureCount: 1 },
339+
});
340+
}),
341+
).pipe(Effect.provide(TestLayer)),
342+
);
343+
344+
it.effect("keeps the snapshot text under the agent's output ceiling", () =>
345+
Effect.scoped(
346+
Effect.gen(function* () {
347+
// Mirrors the real failure: a [role] container whose innerText is the whole
348+
// project list, repeated for several elements, plus a big AX tree.
349+
const pageText = "/Users/theo/Code/project\nClaude, Codex · 79 threads\n".repeat(600);
350+
const element = (name: string, index: number) => ({
351+
tag: "div",
352+
role: "presentation",
353+
name,
354+
selector: `div:nth-of-type(${index})`,
355+
x: 0,
356+
y: 0,
357+
width: 10,
358+
height: 10,
359+
});
360+
const oversized = {
361+
...snapshotResult,
362+
visibleText: pageText,
363+
interactiveElements: [
364+
element(pageText, 1),
365+
element(pageText, 2),
366+
element(pageText, 3),
367+
element("Continue", 4),
368+
],
369+
accessibilityTree: { nodes: Array.from({ length: 2_000 }, (_, i) => ({ nodeId: `${i}` })) },
370+
consoleEntries: Array.from({ length: 100 }, (_, i) => ({
371+
level: "log",
372+
text: `entry ${i}`,
373+
timestamp: "t",
374+
})),
375+
};
376+
yield* serveSnapshots("mcp-bounded-client", oversized);
377+
378+
const snapshot = yield* callSnapshot({ includeImage: false });
379+
380+
expect(snapshot.isError).toBe(false);
381+
const [text, notice] = snapshot.content;
382+
expect(text?.type).toBe("text");
383+
const body = text?.type === "text" ? text.text : "";
384+
expect(Buffer.byteLength(body, "utf8")).toBeLessThanOrEqual(
385+
McpHttpServer.MAX_SNAPSHOT_TEXT_BYTES,
386+
);
387+
const parsed = JSON.parse(body) as {
388+
readonly accessibilityTree?: unknown;
389+
readonly visibleText: string;
390+
readonly interactiveElements: ReadonlyArray<{ readonly name: string }>;
391+
readonly consoleEntries: ReadonlyArray<{ readonly text: string }>;
392+
};
393+
expect(parsed.accessibilityTree).toBeUndefined();
394+
expect(parsed.visibleText.length).toBeLessThanOrEqual(8_001);
395+
expect(parsed.interactiveElements).toHaveLength(4);
396+
expect(parsed.interactiveElements[0]?.name.length).toBeLessThanOrEqual(201);
397+
expect(parsed.interactiveElements[3]?.name).toBe("Continue");
398+
expect(parsed.consoleEntries).toHaveLength(40);
399+
expect(parsed.consoleEntries[0]?.text).toBe("entry 60");
400+
expect(notice?.type === "text" ? notice.text : "").toContain("accessibilityTree");
401+
expect(notice?.type === "text" ? notice.text : "").toContain("60 older console entries");
402+
// The structured result is untouched; only the text the agent reads is bounded.
403+
expect(snapshot.structuredContent).toMatchObject({
404+
accessibilityTree: oversized.accessibilityTree,
405+
});
406+
}),
407+
).pipe(Effect.provide(TestLayer)),
408+
);
409+
230410
it.effect("terminates HTTP MCP sessions with DELETE", () =>
231411
Effect.scoped(
232412
Effect.gen(function* () {
@@ -304,33 +484,19 @@ it.effect("registers annotated tools and preserves authenticated request context
304484
ok: true,
305485
result:
306486
event.request.operation === "snapshot"
307-
? {
308-
url: "http://example.test/",
309-
title: "Example",
310-
loading: false,
311-
visibleText: "Example",
312-
interactiveElements: [],
313-
accessibilityTree: {},
314-
consoleEntries: [],
315-
networkEntries: [],
316-
actionTimeline: [],
317-
screenshot: {
318-
mimeType: "image/png",
319-
data: Buffer.from("png").toString("base64"),
320-
width: 10,
321-
height: 5,
322-
},
323-
}
324-
: event.request.operation === "press"
325-
? undefined
326-
: {
327-
available: true,
328-
visible: true,
329-
tabId,
330-
url: "http://example.test/",
331-
title: "Example",
332-
loading: false,
333-
},
487+
? snapshotResult
488+
: event.request.operation === "evaluate"
489+
? ["Connect", "Continue"]
490+
: event.request.operation === "press"
491+
? undefined
492+
: {
493+
available: true,
494+
visible: true,
495+
tabId,
496+
url: "http://example.test/",
497+
title: "Example",
498+
loading: false,
499+
},
334500
});
335501
}).pipe(Effect.forkScoped);
336502
yield* Effect.yieldNow;
@@ -395,6 +561,22 @@ it.effect("registers annotated tools and preserves authenticated request context
395561
alternateTabId,
396562
);
397563

564+
// Arrays and primitives are wrapped so structuredContent stays a JSON object.
565+
// Claude Code rejects the whole result otherwise.
566+
const evaluateTool = server.tools.find(({ tool }) => tool.name === "preview_evaluate");
567+
expect(evaluateTool?.tool.outputSchema).toMatchObject({ type: "object" });
568+
const evaluated = yield* server
569+
.callTool({ name: "preview_evaluate", arguments: { expression: "buttons()" } })
570+
.pipe(
571+
Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
572+
Effect.provideService(McpSchema.McpServerClient, client),
573+
);
574+
expect(evaluated.isError).toBe(false);
575+
expect(evaluated.structuredContent).toEqual({ value: ["Connect", "Continue"] });
576+
expect(evaluated.content).toEqual([
577+
{ type: "text", text: JSON.stringify({ value: ["Connect", "Continue"] }) },
578+
]);
579+
398580
const actionRequests = [
399581
{ name: "preview_click", arguments: { x: 10, y: 10 } },
400582
{ name: "preview_type", arguments: { text: "Hello" } },

0 commit comments

Comments
 (0)