Skip to content

Commit 0365909

Browse files
committed
fix(mcp): default preview_snapshot to text-only output
Embeds the PNG only on explicit includeImage=true. Inline images persist in provider session history and some providers reject them outright, bricking the session on every later request. save=true already returns a screenshotPath file reference with text-only output. Fixes #11295.
1 parent 54084ae commit 0365909

3 files changed

Lines changed: 29 additions & 13 deletions

File tree

‎apps/server/src/mcp/McpHttpServer.test.ts‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ it.effect("tells the agent how to fall back when no desktop app can run the snap
229229
);
230230

231231
it.effect.each([
232-
{ mode: "default", input: {}, images: true },
232+
{ mode: "default", input: {}, images: false },
233233
{ mode: "explicit image", input: { includeImage: true }, images: true },
234234
{ mode: "text only", input: { includeImage: false }, images: false },
235235
])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images }) =>
@@ -336,12 +336,7 @@ it.effect.each([
336336
Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
337337
Effect.provideService(McpSchema.McpServerClient, client),
338338
);
339-
expect(nextDefault.content.map((content) => content.type)).toEqual([
340-
"text",
341-
"text",
342-
"text",
343-
"image",
344-
]);
339+
expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "text", "text"]);
345340
expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7", screenshot });
346341
expect(nextDefault.structuredContent).not.toHaveProperty("accessibilityTree");
347342
expect(requests).toBe(7);
@@ -394,6 +389,12 @@ it.effect("saves the snapshot PNG on request and reports its path", () =>
394389
expect(Buffer.from(yield* fileSystem.readFile(screenshotPath!)).toString()).toBe("png");
395390
const [, text] = snapshot.content;
396391
expect(text?.type === "text" ? text.text : "").toContain(screenshotPath);
392+
// Saving returns a file reference, not history-bricking bytes: no image
393+
// content unless includeImage is explicitly true.
394+
expect(snapshot.content.map((content) => content.type)).not.toContain("image");
395+
396+
const withImage = yield* callSnapshot({ save: true, includeImage: true });
397+
expect(withImage.content.map((content) => content.type)).toContain("image");
397398

398399
const unsaved = yield* callSnapshot({});
399400
expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath");
@@ -821,10 +822,22 @@ it.effect("registers annotated tools and preserves authenticated request context
821822
Effect.provideService(McpSchema.McpServerClient, client),
822823
);
823824
expect(snapshot.isError).toBe(false);
824-
expect(snapshot.content.some((content) => content.type === "image")).toBe(true);
825+
// Text-first by default: no inline image bytes in the result.
826+
expect(snapshot.content.some((content) => content.type === "image")).toBe(false);
825827
expect(snapshot.structuredContent).toMatchObject({
826828
screenshot: { mimeType: "image/png", width: 10, height: 5 },
827829
});
830+
const snapshotWithImage = yield* server
831+
.callTool({
832+
name: "preview_snapshot",
833+
arguments: { tabId: alternateTabId, includeImage: true },
834+
})
835+
.pipe(
836+
Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
837+
Effect.provideService(McpSchema.McpServerClient, client),
838+
);
839+
expect(snapshotWithImage.isError).toBe(false);
840+
expect(snapshotWithImage.content.some((content) => content.type === "image")).toBe(true);
828841
expect(routedRequests.find(({ operation }) => operation === "snapshot")?.tabId).toBe(
829842
alternateTabId,
830843
);

‎apps/server/src/mcp/McpHttpServer.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -465,9 +465,12 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
465465
text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`,
466466
},
467467
]),
468-
...(payload?.includeImage === false
469-
? []
470-
: [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]),
468+
// Text-first by default: inline images persist in provider
469+
// session history and some providers reject them outright,
470+
// bricking the session. Embed only on explicit opt-in.
471+
...(payload?.includeImage === true
472+
? [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]
473+
: []),
471474
],
472475
});
473476
}),

‎apps/server/src/mcp/toolkits/preview/tools.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,13 +119,13 @@ const PreviewSetAppearanceTool = safeBrowserTool(
119119
export const PreviewSnapshotTool = readonlyBrowserTool(
120120
Tool.make("preview_snapshot", {
121121
description:
122-
"Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",
122+
"Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and screenshot metadata as text. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=true only when the model needs vision: the PNG is embedded in the tool result and persists in session history, which some providers reject. Set save=true to also write the PNG to disk and get screenshotPath back; with an explicit includeImage=false, save=true returns only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",
123123
parameters: Schema.Struct({
124124
...PreviewAutomationTabTargetInput.fields,
125125
includeImage: Schema.optional(
126126
Schema.Boolean.annotate({
127127
description:
128-
"Include the PNG image in the tool response. Defaults to true. Set false for text-only output.",
128+
"Include the PNG image in the tool response. Defaults to false. Set true only when the model needs vision; inline images persist in session history and some providers reject them.",
129129
}),
130130
),
131131
save: Schema.optional(

0 commit comments

Comments
 (0)