Skip to content

Commit 2162f0c

Browse files
TonybynMp4claude
andcommitted
fix(server): saved preview screenshots keep full resolution
Saving a preview snapshot wrote the same 1280px-capped image the agent receives, so captures from scaled displays and headless tabs were shrunk and their text came out soft. The saved file now uses the uncapped capture, while the agent's image stays capped. Fixes #28251 Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 80a2351 commit 2162f0c

7 files changed

Lines changed: 94 additions & 12 deletions

File tree

‎apps/server/src/mcp/McpHttpServer.test.ts‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import {
1919
import * as Context from "effect/Context";
2020
import * as Deferred from "effect/Deferred";
2121
import * as Effect from "effect/Effect";
22+
import * as FileSystem from "effect/FileSystem";
2223
import * as Layer from "effect/Layer";
2324
import * as Schema from "effect/Schema";
2425
import * as Stream from "effect/Stream";
@@ -425,6 +426,57 @@ it.effect.each([
425426
).pipe(Effect.provide(layerTest)),
426427
);
427428

429+
it.effect("saves the uncapped screenshot while the agent sees the capped one", () =>
430+
Effect.scoped(
431+
Effect.gen(function* () {
432+
const server = yield* McpServer.McpServer;
433+
const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker;
434+
const connected = yield* Deferred.make<void>();
435+
const capped =
436+
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=";
437+
const full =
438+
"iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAAAAABX3VL4AAAADklEQVR4nGNgYGBgYAAAAAYAAXKCmeoAAAAASUVORK5CYII=";
439+
const events = yield* broker.connect({ clientId: "mcp-save-client", environmentId });
440+
yield* Stream.runForEach(events, (event) => {
441+
if (event.type === "connected") return Deferred.succeed(connected, undefined);
442+
expect(event.request.input).toEqual({ fullScreenshot: true });
443+
return broker.respond({
444+
clientId: "mcp-save-client",
445+
connectionId: event.connectionId,
446+
requestId: event.request.requestId,
447+
ok: true,
448+
result: {
449+
...snapshotResult,
450+
screenshot: { mimeType: "image/png", data: capped, width: 1, height: 1 },
451+
fullScreenshot: { data: full, width: 2, height: 2 },
452+
},
453+
});
454+
}).pipe(Effect.forkScoped);
455+
yield* Deferred.await(connected);
456+
457+
const snapshot = yield* server
458+
.callTool({
459+
name: "preview_snapshot",
460+
arguments: { tabId: alternateTabId, save: true, includeImage: true },
461+
})
462+
.pipe(
463+
Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
464+
Effect.provideService(McpSchema.McpServerClient, client),
465+
);
466+
const { screenshotPath } = snapshot.structuredContent as { readonly screenshotPath: string };
467+
const saved = yield* (yield* FileSystem.FileSystem).readFile(screenshotPath);
468+
469+
expect(Buffer.from(saved).toString("base64")).toBe(full);
470+
expect(snapshot.structuredContent).not.toHaveProperty("fullScreenshot");
471+
expect(snapshot.content.at(-1)).toEqual({
472+
type: "image",
473+
mimeType: "image/png",
474+
data: new Uint8Array(Buffer.from(capped, "base64")),
475+
});
476+
}),
477+
).pipe(Effect.provide(layerTest)),
478+
);
479+
428480
it.effect("refuses preview tools to a client outside a thread before they run", () =>
429481
Effect.gen(function* () {
430482
const server = yield* McpServer.McpServer;

‎apps/server/src/mcp/McpHttpServer.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -541,11 +541,19 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
541541
readonly width: number;
542542
readonly height: number;
543543
};
544+
readonly fullScreenshot?: { readonly data: string };
544545
};
545-
const { screenshot, ...page } = snapshot;
546+
const { screenshot, fullScreenshot, ...page } = snapshot;
546547
const png = new Uint8Array(Buffer.from(screenshot.data, "base64"));
547548
const screenshotPath =
548-
payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined;
549+
payload?.save === true
550+
? yield* saveScreenshot(
551+
snapshot.url,
552+
fullScreenshot
553+
? new Uint8Array(Buffer.from(fullScreenshot.data, "base64"))
554+
: png,
555+
)
556+
: undefined;
549557
// Providers replay images on later requests, and some reject them; include images only when requested.
550558
const includeImage = payload?.includeImage === true;
551559
if (screenshotPath !== undefined && !includeImage) {

‎apps/server/src/mcp/toolkits/preview/handlers.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -208,8 +208,12 @@ const handlers = {
208208
invokeTargeted<PreviewAutomationSetColorSchemeResult>("setColorScheme", input),
209209
),
210210
preview_snapshot: McpToolAccess.readsAsCaller((input) => {
211-
const { includeImage: _includeImage, save: _save, ...operationInput } = input ?? {};
212-
return invokeTargeted<PreviewAutomationSnapshot>("snapshot", operationInput);
211+
const { includeImage: _includeImage, save, ...operationInput } = input ?? {};
212+
// The saved file is for people, so it keeps the resolution the agent's image is capped below.
213+
return invokeTargeted<PreviewAutomationSnapshot>("snapshot", {
214+
...operationInput,
215+
...(save === true ? { fullScreenshot: true } : {}),
216+
});
213217
}),
214218
preview_act: McpToolAccess.actsAsCaller(({ action, ...input }) =>
215219
decodeActInput[action](input).pipe(

‎apps/server/src/preview/ServerBrowser.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import {
3030
type PreviewAutomationScrollInput,
3131
type PreviewAutomationSelectInput,
3232
type PreviewAutomationSetColorSchemeInput,
33+
type PreviewAutomationSnapshotInput,
3334
type PreviewAutomationStatus,
3435
type PreviewAutomationTypeInput,
3536
type PreviewAutomationUploadInput,
@@ -2156,6 +2157,7 @@ const make = Effect.gen(function* () {
21562157
ServerBrowserPage.snapshot({
21572158
...tab,
21582159
renderScale: tab.desktop ? "display" : RENDER_SCALE,
2160+
fullScreenshot: (input as PreviewAutomationSnapshotInput | undefined)?.fullScreenshot,
21592161
}),
21602162
);
21612163
}

‎apps/server/src/preview/ServerBrowserPage.test.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -283,10 +283,11 @@ describe("server browser snapshot screenshot", () => {
283283
const screenshotOf = async (html: string) => {
284284
const page = await context.newPage();
285285
await page.setContent(html);
286-
const { screenshot } = await ServerBrowserPage.snapshot({
286+
const { screenshot, fullScreenshot } = await ServerBrowserPage.snapshot({
287287
page,
288288
cdp: await context.newCDPSession(page),
289289
renderScale: 2,
290+
fullScreenshot: true,
290291
consoleEntries: [],
291292
networkEntries: [],
292293
actionTimeline: [],
@@ -309,7 +310,7 @@ describe("server browser snapshot screenshot", () => {
309310
Array.from(context.getImageData(x * bitmap.width, y * bitmap.height, 1, 1).data),
310311
);
311312
})()`)) as Array<[number, number, number, number]>;
312-
return { screenshot, pixels };
313+
return { screenshot, fullScreenshot, pixels };
313314
};
314315
const colorOf = ([red, green, blue]: readonly [number, number, number, number]) =>
315316
blue - red > 100 ? "blue" : red - blue > 100 ? "red" : `rgb(${red}, ${green}, ${blue})`;
@@ -324,12 +325,13 @@ describe("server browser snapshot screenshot", () => {
324325
});
325326

326327
it("captures a plain page at the shrunk size with its content in place", async () => {
327-
const { screenshot, pixels } = await screenshotOf(halves(""));
328+
const { screenshot, fullScreenshot, pixels } = await screenshotOf(halves(""));
328329

329330
expect({
330331
size: [screenshot.width, screenshot.height],
332+
fullSize: [fullScreenshot?.width, fullScreenshot?.height],
331333
colors: pixels.map(colorOf),
332-
}).toEqual({ size: [1280, 800], colors: ["blue", "red", "red"] });
334+
}).toEqual({ size: [1280, 800], fullSize: [2560, 1600], colors: ["blue", "red", "red"] });
333335
});
334336

335337
it("shrinks the screenshot of an XML document, whose createElement makes no canvas", async () => {

‎apps/server/src/preview/ServerBrowserPage.ts‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,7 @@ type SnapshotInput = {
499499
readonly cdp: CDPSession;
500500
/** Headless pages render at the scale the server set; a desktop page at its display's. */
501501
readonly renderScale: number | "display";
502+
readonly fullScreenshot?: boolean | undefined;
502503
readonly consoleEntries: ReadonlyArray<PreviewAutomationConsoleEntry>;
503504
readonly networkEntries: ReadonlyArray<PreviewAutomationNetworkEntry>;
504505
readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"];
@@ -542,9 +543,10 @@ const capture = async (
542543
"url" | "title" | "loading" | "visibleText" | "interactiveElements"
543544
>
544545
>,
545-
captureViewport(input.cdp, { format: "png" }).then((png) =>
546-
scale < 1 ? shrinkPng(input.cdp, png, scale) : png,
547-
),
546+
captureViewport(input.cdp, { format: "png" }).then(async (full) => ({
547+
full,
548+
data: scale < 1 ? await shrinkPng(input.cdp, full, scale) : full,
549+
})),
548550
]);
549551
void pageAndScreenshot.catch(constVoid);
550552
// Leaves time for the second read; a frame that never loads holds the tree until then.
@@ -568,7 +570,7 @@ const capture = async (
568570
void iframe.dispose().catch(constVoid);
569571
if (frame && unread.includes(frame)) throw tooBusy();
570572
}
571-
const [page, data] = await pageAndScreenshot;
573+
const [page, { full, data }] = await pageAndScreenshot;
572574
if (state.generation !== generation) throw pageChanged();
573575
const mask = (text: string) => (dotted ? text.replace(dotted, MASKED_VALUE) : text);
574576
// Redacted before the cut, so a payload crossing it can't leave its start showing, and before
@@ -594,6 +596,9 @@ const capture = async (
594596
data,
595597
...pngSize(data),
596598
},
599+
...(input.fullScreenshot && full !== data
600+
? { fullScreenshot: { data: full, ...pngSize(full) } }
601+
: {}),
597602
};
598603
};
599604

‎packages/contracts/src/previewAutomation.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -552,9 +552,18 @@ export const PreviewAutomationSnapshot = Schema.Struct({
552552
width: Schema.Int,
553553
height: Schema.Int,
554554
}),
555+
/** The capture before `screenshot` was capped for the agent, when requested and larger. */
556+
fullScreenshot: Schema.optional(
557+
Schema.Struct({ data: Schema.String, width: Schema.Int, height: Schema.Int }),
558+
),
555559
});
556560
export type PreviewAutomationSnapshot = typeof PreviewAutomationSnapshot.Type;
557561

562+
export const PreviewAutomationSnapshotInput = Schema.Struct({
563+
fullScreenshot: Schema.optional(Schema.Boolean),
564+
});
565+
export type PreviewAutomationSnapshotInput = typeof PreviewAutomationSnapshotInput.Type;
566+
558567
export const PreviewAutomationRecordingStatus = Schema.Struct({
559568
tabId: PreviewTabId,
560569
recording: Schema.Boolean,

0 commit comments

Comments
 (0)