Skip to content

Commit ff9ee0d

Browse files
ScottN-PVclaude
andcommitted
fix(server): size desktop-drawn preview snapshots by the page's own scale and viewport
Snapshots assumed every tab renders at 2x in a 1280x800 viewport. A tab the desktop draws keeps the display's scale and its panel's viewport, so at 150% scaling the PNG came out 960x600 while the result reported 1280x800, and a viewport of another size was clipped wrongly. For those tabs the snapshot now reads devicePixelRatio and the viewport from the page and divides out the zoom the desktop applied. The reported size is read from the PNG header. Headless tabs capture as before. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent bfec238 commit ff9ee0d

4 files changed

Lines changed: 255 additions & 11 deletions

File tree

‎apps/server/src/preview/ServerBrowser.test.ts‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1026,6 +1026,129 @@ it.live("viewers see the agent's pointer move to its target and click there", ()
10261026
).pipe(Effect.provide(layer)),
10271027
);
10281028

1029+
/** Makes a fake desktop page report its ratio and CSS viewport, as a real one would. */
1030+
const reportPage = (
1031+
context: ReturnType<typeof makeContext>,
1032+
page: { readonly ratio: number; readonly width: number; readonly height: number },
1033+
) => {
1034+
const evaluate = context.page.evaluate.getMockImplementation()!;
1035+
context.page.evaluate.mockImplementation(((script?: unknown) =>
1036+
typeof script === "string" && script.includes("devicePixelRatio")
1037+
? Promise.resolve(page)
1038+
: evaluate()) as typeof evaluate);
1039+
};
1040+
1041+
/** The clip of each scaled capture a context's sessions were asked for. Viewer stills are full size. */
1042+
const captureClips = (context: ReturnType<typeof makeContext>) =>
1043+
context.sessions.flatMap((session) =>
1044+
session.send.mock.calls.flatMap(([method, input]) => {
1045+
const clip = (
1046+
input as
1047+
| {
1048+
readonly clip?: {
1049+
readonly width: number;
1050+
readonly height: number;
1051+
readonly scale: number;
1052+
};
1053+
}
1054+
| undefined
1055+
)?.clip;
1056+
return method === "Page.captureScreenshot" && clip !== undefined
1057+
? [{ width: clip.width, height: clip.height, scale: clip.scale }]
1058+
: [];
1059+
}),
1060+
);
1061+
1062+
it.live("sizes a snapshot by the display's ratio for a tab the desktop renders", () =>
1063+
Effect.scoped(
1064+
Effect.gen(function* () {
1065+
desktopRendersNext = true;
1066+
const { browser, broker, tabId } = yield* ready;
1067+
yield* browser.attachViewer(viewerInput(tabId, false));
1068+
const context = desktopConnections[0]!.context;
1069+
// A 150% display, with the page at the panel's default 1280x800.
1070+
reportPage(context, { ratio: 1.5, width: 1280, height: 800 });
1071+
const snapshot = yield* broker.invoke<PreviewAutomationSnapshot>({
1072+
scope,
1073+
tabId,
1074+
operation: "snapshot",
1075+
input: {},
1076+
});
1077+
expect(captureClips(context)).toEqual([{ width: 1280, height: 800, scale: 1280 / 1920 }]);
1078+
expect(snapshot.screenshot).toMatchObject({ width: 1280, height: 800 });
1079+
}),
1080+
).pipe(Effect.provide(layer)),
1081+
);
1082+
1083+
it.live("captures the viewport a desktop-drawn page actually has", () =>
1084+
Effect.scoped(
1085+
Effect.gen(function* () {
1086+
desktopRendersNext = true;
1087+
const { browser, broker, tabId } = yield* ready;
1088+
yield* browser.attachViewer(viewerInput(tabId, false));
1089+
const context = desktopConnections[0]!.context;
1090+
// The desktop window was zoomed out, so the page lays out smaller than 1280x800.
1091+
reportPage(context, { ratio: 1.5, width: 1067, height: 667 });
1092+
const snapshot = yield* broker.invoke<PreviewAutomationSnapshot>({
1093+
scope,
1094+
tabId,
1095+
operation: "snapshot",
1096+
input: {},
1097+
});
1098+
expect(captureClips(context)).toEqual([
1099+
{ width: 1067, height: 667, scale: 1280 / (1067 * 1.5) },
1100+
]);
1101+
expect(snapshot.screenshot).toMatchObject({ width: 1280, height: 800 });
1102+
}),
1103+
).pipe(Effect.provide(layer)),
1104+
);
1105+
1106+
it.live("leaves page zoom out of a desktop-drawn tab's snapshot scale", () =>
1107+
Effect.scoped(
1108+
Effect.gen(function* () {
1109+
desktopRendersNext = true;
1110+
const { browser, broker, tabId } = yield* ready;
1111+
const manager = yield* Manager.PreviewManager;
1112+
yield* browser.attachViewer(viewerInput(tabId, false));
1113+
const context = desktopConnections[0]!.context;
1114+
yield* manager.adjust({ threadId: scope.thread.threadId, tabId, zoomFactor: 2 });
1115+
// A 150% display with the page at 200% zoom: it lays out at half size in CSS pixels.
1116+
reportPage(context, { ratio: 3, width: 640, height: 400 });
1117+
const snapshot = yield* broker.invoke<PreviewAutomationSnapshot>({
1118+
scope,
1119+
tabId,
1120+
operation: "snapshot",
1121+
input: {},
1122+
});
1123+
expect(captureClips(context)).toEqual([{ width: 1280, height: 800, scale: 1280 / 1920 }]);
1124+
expect(snapshot.screenshot).toMatchObject({ width: 1280, height: 800 });
1125+
// The desktop zooms its own page; the server sends it no emulation.
1126+
expect(
1127+
context.sessions.flatMap((session) => session.send.mock.calls.map(([method]) => method)),
1128+
).not.toContain("Emulation.setDeviceMetricsOverride");
1129+
}),
1130+
).pipe(Effect.provide(layer)),
1131+
);
1132+
1133+
it.live("keeps a headless tab's snapshot at its own render scale", () =>
1134+
Effect.scoped(
1135+
Effect.gen(function* () {
1136+
const { broker, tabId } = yield* ready;
1137+
const snapshot = yield* broker.invoke<PreviewAutomationSnapshot>({
1138+
scope,
1139+
tabId,
1140+
operation: "snapshot",
1141+
input: {},
1142+
});
1143+
expect(captureClips(contexts[0]!)).toEqual([{ width: 1280, height: 800, scale: 0.5 }]);
1144+
expect(contexts[0]!.page.evaluate).not.toHaveBeenCalledWith(
1145+
expect.stringContaining("devicePixelRatio"),
1146+
);
1147+
expect(snapshot.screenshot).toMatchObject({ width: 1280, height: 800 });
1148+
}),
1149+
).pipe(Effect.provide(layer)),
1150+
);
1151+
10291152
it.live("drives the desktop's own page for a tab the desktop renders", () =>
10301153
Effect.scoped(
10311154
Effect.gen(function* () {

‎apps/server/src/preview/ServerBrowser.ts‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -631,10 +631,13 @@ const make = Effect.gen(function* () {
631631
await tab.page.emulateMedia({ colorScheme: colorScheme === "system" ? null : colorScheme });
632632
}
633633
const zoomFactor = snapshot.zoomFactor ?? 1;
634-
if (zoomFactor !== tab.zoomFactor && !tab.desktop) {
634+
if (zoomFactor !== tab.zoomFactor) {
635635
tab.zoomFactor = zoomFactor;
636-
await applyZoom(tab);
637-
broadcastViewport(tab);
636+
// The desktop applies the zoom to a page it draws; it is kept here for snapshot sizing.
637+
if (!tab.desktop) {
638+
await applyZoom(tab);
639+
broadcastViewport(tab);
640+
}
638641
}
639642
};
640643

@@ -1270,6 +1273,26 @@ const make = Effect.gen(function* () {
12701273
return started;
12711274
};
12721275

1276+
/**
1277+
* What sizes a snapshot. Chromium sizes a capture in device-independent pixels times the
1278+
* display's scale, without page zoom. A headless tab renders at RENDER_SCALE in the viewport
1279+
* Playwright set. A tab the desktop draws has neither: its page reports the display's scale
1280+
* times the zoom the desktop applied, and its viewport in CSS pixels.
1281+
*/
1282+
const snapshotRendering = async (tab: ServerTab) => {
1283+
if (!tab.desktop) return { renderScale: RENDER_SCALE };
1284+
const page = (await tab.page.evaluate(
1285+
"({ ratio: devicePixelRatio, width: innerWidth, height: innerHeight })",
1286+
)) as { readonly ratio: number; readonly width: number; readonly height: number };
1287+
return {
1288+
renderScale: page.ratio / tab.zoomFactor,
1289+
viewport: {
1290+
width: Math.round(page.width * tab.zoomFactor),
1291+
height: Math.round(page.height * tab.zoomFactor),
1292+
},
1293+
};
1294+
};
1295+
12731296
// Scaled captures repaint every screencast; pause them to avoid leaking that frame.
12741297
const withScreencastsPaused = <A>(tab: ServerTab, capture: () => Promise<A>): Promise<A> =>
12751298
withCaptureLock(tab, async () => {
@@ -1653,8 +1676,8 @@ const make = Effect.gen(function* () {
16531676
return { tabId: tab.tabId, colorScheme };
16541677
}
16551678
case "snapshot": {
1656-
return withScreencastsPaused(tab, () =>
1657-
ServerBrowserPage.snapshot({ ...tab, renderScale: RENDER_SCALE }),
1679+
return withScreencastsPaused(tab, async () =>
1680+
ServerBrowserPage.snapshot({ ...tab, ...(await snapshotRendering(tab)) }),
16581681
);
16591682
}
16601683
case "click": {

‎apps/server/src/preview/ServerBrowserPage.test.ts‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,3 +246,78 @@ describe("server browser element refs", () => {
246246
expect(await page.evaluate("window.seen")).toEqual(["click", "hover", "drop"]);
247247
});
248248
});
249+
250+
describe("server browser snapshot size", () => {
251+
/** Width and height from the PNG's IHDR chunk. */
252+
const pngSize = (data: string) => {
253+
const bytes = Buffer.from(data, "base64");
254+
return { width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20) };
255+
};
256+
257+
// Chromium sizes a scaled capture by the window's real ratio, so each case launches at its own,
258+
// as a display does for a tab the desktop draws.
259+
it.each([1, 1.5, 2])(
260+
"captures a 1280x800 viewport at 1280x800 when the render scale is the page's %sx",
261+
async (deviceScaleFactor) => {
262+
const browser = await chromium.launch({
263+
headless: true,
264+
args: [`--force-device-scale-factor=${deviceScaleFactor}`],
265+
});
266+
try {
267+
const context = await browser.newContext({
268+
viewport: { width: 1280, height: 800 },
269+
deviceScaleFactor,
270+
});
271+
const page = await context.newPage();
272+
const cdp = await context.newCDPSession(page);
273+
await page.setContent("<p>sized</p>");
274+
// A just-launched browser can refuse a capture until it has painted.
275+
await page.evaluate(
276+
"new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
277+
);
278+
const result = await ServerBrowserPage.snapshot({
279+
page,
280+
cdp,
281+
renderScale: deviceScaleFactor,
282+
consoleEntries: [],
283+
networkEntries: [],
284+
actionTimeline: [],
285+
});
286+
expect(pngSize(result.screenshot.data)).toEqual({ width: 1280, height: 800 });
287+
expect(result.screenshot).toMatchObject({ width: 1280, height: 800 });
288+
} finally {
289+
await browser.close();
290+
}
291+
},
292+
);
293+
294+
it("reports the captured image's size when the given viewport is a pixel off", async () => {
295+
const browser = await chromium.launch({
296+
headless: true,
297+
args: ["--force-device-scale-factor=1"],
298+
});
299+
try {
300+
const context = await browser.newContext({ viewport: { width: 539, height: 939 } });
301+
const page = await context.newPage();
302+
const cdp = await context.newCDPSession(page);
303+
await page.setContent("<p>sized</p>");
304+
await page.evaluate(
305+
"new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(resolve)))",
306+
);
307+
// A zoomed desktop page's integer CSS size times its zoom can miss its real width by one.
308+
const result = await ServerBrowserPage.snapshot({
309+
page,
310+
cdp,
311+
renderScale: 1,
312+
viewport: { width: 538, height: 940 },
313+
consoleEntries: [],
314+
networkEntries: [],
315+
actionTimeline: [],
316+
});
317+
expect(pngSize(result.screenshot.data)).toEqual({ width: 539, height: 939 });
318+
expect(result.screenshot).toMatchObject({ width: 539, height: 939 });
319+
} finally {
320+
await browser.close();
321+
}
322+
});
323+
});

‎apps/server/src/preview/ServerBrowserPage.ts‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -150,15 +150,34 @@ const SNAPSHOT_SCRIPT = `(() => {
150150
};
151151
})()`;
152152

153+
/**
154+
* Width and height from a PNG's header, so the reported size is the image's own. Null for data
155+
* that is not a PNG.
156+
*/
157+
const pngSize = (base64: string) => {
158+
const header = Buffer.from(base64.slice(0, 32), "base64");
159+
return header.length >= 24 && header.toString("latin1", 12, 16) === "IHDR"
160+
? { width: header.readUInt32BE(16), height: header.readUInt32BE(20) }
161+
: null;
162+
};
163+
164+
/** A page's viewport in device-independent pixels, the units a capture clip is given in. */
165+
type Viewport = { readonly width: number; readonly height: number };
166+
153167
// Scaled captures repaint live screencasts, so callers pause them. Clips use document offsets.
154168
export const captureViewport = async (
155169
page: Page,
156170
cdp: CDPSession,
157-
options: { readonly format: "png" | "jpeg"; readonly quality?: number; readonly scale: number },
171+
options: {
172+
readonly format: "png" | "jpeg";
173+
readonly quality?: number;
174+
readonly scale: number;
175+
readonly viewport?: Viewport;
176+
},
158177
) => {
159178
let clip;
160179
if (options.scale < 1) {
161-
const viewport = page.viewportSize() ?? { width: 1280, height: 800 };
180+
const viewport = options.viewport ?? page.viewportSize() ?? { width: 1280, height: 800 };
162181
const { cssVisualViewport } = await cdp.send("Page.getLayoutMetrics");
163182
clip = {
164183
x: cssVisualViewport.pageX,
@@ -179,11 +198,13 @@ export const snapshot = async (input: {
179198
readonly page: Page;
180199
readonly cdp: CDPSession;
181200
readonly renderScale: number;
201+
/** For a page Playwright did not size, such as one the desktop draws. */
202+
readonly viewport?: Viewport;
182203
readonly consoleEntries: ReadonlyArray<PreviewAutomationConsoleEntry>;
183204
readonly networkEntries: ReadonlyArray<PreviewAutomationNetworkEntry>;
184205
readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"];
185206
}): Promise<PreviewAutomationSnapshot> => {
186-
const viewport = input.page.viewportSize() ?? { width: 1280, height: 800 };
207+
const viewport = input.viewport ?? input.page.viewportSize() ?? { width: 1280, height: 800 };
187208
const scale = Math.min(1, MAX_SCREENSHOT_WIDTH / (viewport.width * input.renderScale));
188209
const state = refsFor(input.page);
189210
invalidateRefs(input.page);
@@ -196,7 +217,7 @@ export const snapshot = async (input: {
196217
>
197218
>,
198219
input.page.ariaSnapshot({ mode: "ai", boxes: true, timeout: DEFAULT_TIMEOUT_MS }),
199-
captureViewport(input.page, input.cdp, { format: "png", scale }),
220+
captureViewport(input.page, input.cdp, { format: "png", scale, viewport }),
200221
]);
201222
if (state.generation !== generation) {
202223
throw new ServerBrowserOperationError(
@@ -220,8 +241,10 @@ export const snapshot = async (input: {
220241
screenshot: {
221242
mimeType: "image/png",
222243
data,
223-
width: Math.round(viewport.width * input.renderScale * scale),
224-
height: Math.round(viewport.height * input.renderScale * scale),
244+
...(pngSize(data) ?? {
245+
width: Math.round(viewport.width * input.renderScale * scale),
246+
height: Math.round(viewport.height * input.renderScale * scale),
247+
}),
225248
},
226249
};
227250
};

0 commit comments

Comments
 (0)