Skip to content

Commit 2763bbe

Browse files
fix(server): preview snapshots preserve blurred backdrops (#24618)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 3e70f55 commit 2763bbe

4 files changed

Lines changed: 154 additions & 48 deletions

File tree

‎apps/server/src/preview/ServerBrowser.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@ function makeSession(frame?: ReturnType<typeof makeFrame>) {
9696
// A snapshot sizes its screenshot from the PNG header; viewers get JPEG frames as-is.
9797
return { data: (input as { format: string }).format === "png" ? PNG_HEADER : "ZnJhbWU=" };
9898
}
99+
// Snapshots shrink their screenshot in the page; the header stands in for the result.
100+
if (method === "Runtime.callFunctionOn") return { result: { value: PNG_HEADER } };
99101
if (method === "Page.getFrameTree") return { frameTree: { frame: { id: "main" } } };
100102
if (method === "Page.createIsolatedWorld") {
101103
return {

‎apps/server/src/preview/ServerBrowser.ts‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1467,10 +1467,9 @@ const make = Effect.gen(function* () {
14671467
try {
14681468
await encoder.evaluate(ServerBrowserPage.recordingEncoderScript(recordingFrameRate));
14691469
// Seed idle pages before returning so an immediate stop has a frame to encode.
1470-
const firstFrame = await ServerBrowserPage.captureViewport(tab.page, tab.cdp, {
1470+
const firstFrame = await ServerBrowserPage.captureViewport(tab.cdp, {
14711471
format: "jpeg",
14721472
quality: RECORDING_SCREENCAST.quality,
1473-
scale: 1,
14741473
});
14751474
await encoder.evaluate(
14761475
([data, width]) =>
@@ -1522,7 +1521,6 @@ const make = Effect.gen(function* () {
15221521
return started;
15231522
};
15241523

1525-
// Scaled captures repaint every screencast; pause them to avoid leaking that frame.
15261524
const withScreencastsPaused = <A>(tab: ServerTab, capture: () => Promise<A>): Promise<A> =>
15271525
withCaptureLock(tab, async () => {
15281526
tab.capturing += 1;
@@ -2333,14 +2331,9 @@ const make = Effect.gen(function* () {
23332331
}
23342332
broadcastControl(tab);
23352333
pushFileChooser(tab);
2336-
// Full scale: a scaled capture would flash in every other viewer.
23372334
const pushStill = async () => {
23382335
const data = await withCaptureLock(tab, () =>
2339-
ServerBrowserPage.captureViewport(tab.page, session, {
2340-
format: "jpeg",
2341-
quality,
2342-
scale: 1,
2343-
}),
2336+
ServerBrowserPage.captureViewport(session, { format: "jpeg", quality }),
23442337
).catch(() => null);
23452338
if (data) viewer.push({ _tag: "frame", data: Buffer.from(data, "base64"), ack: Effect.void });
23462339
};

‎apps/server/src/preview/ServerBrowserPage.test.ts‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,109 @@ describe("server browser element refs", () => {
245245
});
246246
});
247247

248+
describe("server browser snapshot screenshot", () => {
249+
let browser: Browser;
250+
let context: BrowserContext;
251+
252+
beforeAll(async () => {
253+
// ServerBrowserContexts launches with these, and the capture scale depends on them.
254+
browser = await chromium.launch({
255+
headless: true,
256+
args: ["--disable-gpu", "--force-device-scale-factor=2"],
257+
});
258+
});
259+
afterAll(async () => {
260+
await browser?.close();
261+
});
262+
beforeEach(async () => {
263+
context = await browser.newContext({
264+
viewport: { width: 1280, height: 800 },
265+
deviceScaleFactor: 2,
266+
});
267+
});
268+
afterEach(async () => {
269+
await context.close();
270+
});
271+
272+
// Blue on the left half and red on the right, so a mirrored left half shows up on the right.
273+
const halves = (overlay: string) => `
274+
<style>
275+
body { margin: 0; background: white; }
276+
.half { position: absolute; top: 0; width: 640px; height: 800px; }
277+
.overlay { position: fixed; inset: 0; ${overlay} }
278+
</style>
279+
<div class="half" style="left: 0; background: #2563eb"></div>
280+
<div class="half" style="left: 640px; background: #ef4444"></div>
281+
<div class="overlay"></div>`;
282+
283+
const screenshotOf = async (html: string) => {
284+
const page = await context.newPage();
285+
await page.setContent(html);
286+
const { screenshot } = await ServerBrowserPage.snapshot({
287+
page,
288+
cdp: await context.newCDPSession(page),
289+
renderScale: 2,
290+
consoleEntries: [],
291+
networkEntries: [],
292+
actionTimeline: [],
293+
});
294+
// The server has no DOM types, so the decoder runs as page source.
295+
const points = [
296+
[0.15, 0.75],
297+
[0.85, 0.25],
298+
[0.85, 0.75],
299+
];
300+
const pixels = (await page.evaluate(`(async () => {
301+
const bytes = Uint8Array.from(atob(${JSON.stringify(screenshot.data)}), (char) => char.charCodeAt(0));
302+
const bitmap = await createImageBitmap(new Blob([bytes], { type: "image/png" }));
303+
const canvas = document.createElement("canvas");
304+
canvas.width = bitmap.width;
305+
canvas.height = bitmap.height;
306+
const context = canvas.getContext("2d");
307+
context.drawImage(bitmap, 0, 0);
308+
return ${JSON.stringify(points)}.map(([x, y]) =>
309+
Array.from(context.getImageData(x * bitmap.width, y * bitmap.height, 1, 1).data),
310+
);
311+
})()`)) as Array<[number, number, number, number]>;
312+
return { screenshot, pixels };
313+
};
314+
const colorOf = ([red, green, blue]: readonly [number, number, number, number]) =>
315+
blue - red > 100 ? "blue" : red - blue > 100 ? "red" : `rgb(${red}, ${green}, ${blue})`;
316+
317+
it("captures a blurred backdrop-filter page as rendered, not mirrored", async () => {
318+
const { screenshot, pixels } = await screenshotOf(halves("backdrop-filter: blur(4px);"));
319+
320+
expect({
321+
size: [screenshot.width, screenshot.height],
322+
colors: pixels.map(colorOf),
323+
}).toEqual({ size: [1280, 800], colors: ["blue", "red", "red"] });
324+
});
325+
326+
it("captures a plain page at the shrunk size with its content in place", async () => {
327+
const { screenshot, pixels } = await screenshotOf(halves(""));
328+
329+
expect({
330+
size: [screenshot.width, screenshot.height],
331+
colors: pixels.map(colorOf),
332+
}).toEqual({ size: [1280, 800], colors: ["blue", "red", "red"] });
333+
});
334+
335+
it("shrinks the screenshot of an XML document, whose createElement makes no canvas", async () => {
336+
const page = await context.newPage();
337+
await page.goto("data:text/xml,<a/>");
338+
const { screenshot } = await ServerBrowserPage.snapshot({
339+
page,
340+
cdp: await context.newCDPSession(page),
341+
renderScale: 2,
342+
consoleEntries: [],
343+
networkEntries: [],
344+
actionTimeline: [],
345+
});
346+
347+
expect([screenshot.width, screenshot.height]).toEqual([1280, 800]);
348+
});
349+
});
350+
248351
describe("server browser drag", () => {
249352
it("fails the drag, not the server, when it times out while the cursor is still moving", async () => {
250353
const timedOut = new Error("locator.dragTo: Timeout 30000ms exceeded.");

‎apps/server/src/preview/ServerBrowserPage.ts‎

Lines changed: 47 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -410,8 +410,6 @@ const pngSize = (base64: string) => {
410410
return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) };
411411
};
412412

413-
type Viewport = { readonly width: number; readonly height: number };
414-
415413
/**
416414
* A page's display ratio and CSS viewport, read in an isolated world since page script can
417415
* reassign `devicePixelRatio`, `innerWidth`, and `innerHeight` in its own, and its page zoom.
@@ -438,55 +436,64 @@ const readPageMetrics = async (cdp: CDPSession) => {
438436
readonly width: number;
439437
readonly height: number;
440438
};
441-
return {
442-
ratio,
443-
width,
444-
height,
445-
zoom,
446-
offset: { x: cssVisualViewport.pageX * zoom, y: cssVisualViewport.pageY * zoom },
447-
};
439+
return { ratio, width, height, zoom };
448440
}
449441
}
450442
throw new Error(
451443
"The page's zoom kept changing while the snapshot read its size. Try the snapshot again.",
452444
);
453445
};
454446

455-
// Scaled captures repaint live screencasts, so callers pause them. Clips use document offsets.
447+
// Never pass a clip scale: Chromium mirrors the top-left of the page into a `backdrop-filter`
448+
// in any capture it shrinks.
456449
export const captureViewport = async (
457-
page: Page,
458450
cdp: CDPSession,
459-
options: {
460-
readonly format: "png" | "jpeg";
461-
readonly quality?: number;
462-
readonly scale: number;
463-
readonly viewport?: Viewport;
464-
readonly offset?: { readonly x: number; readonly y: number } | undefined;
465-
},
451+
options: { readonly format: "png" | "jpeg"; readonly quality?: number },
466452
) => {
467-
let clip;
468-
if (options.scale < 1) {
469-
const viewport = options.viewport ?? page.viewportSize() ?? { width: 1280, height: 800 };
470-
let offset = options.offset;
471-
if (!offset) {
472-
const { cssVisualViewport } = await cdp.send("Page.getLayoutMetrics");
473-
const zoom = cssVisualViewport.zoom ?? 1;
474-
offset = { x: cssVisualViewport.pageX * zoom, y: cssVisualViewport.pageY * zoom };
475-
}
476-
clip = {
477-
...offset,
478-
...viewport,
479-
scale: options.scale,
480-
};
481-
}
482453
const { data } = await cdp.send("Page.captureScreenshot", {
483454
format: options.format,
484455
...(options.quality === undefined ? {} : { quality: options.quality }),
485-
...(clip ? { clip } : {}),
486456
});
487457
return data;
488458
};
489459

460+
/** Runs in the page's own isolated world, as the server has no image library. */
461+
const SHRINK_PNG = `async function (png, width, height) {
462+
const bytes = Uint8Array.from(atob(png), (char) => char.charCodeAt(0));
463+
const bitmap = await createImageBitmap(new Blob([bytes], { type: "image/png" }));
464+
// Namespaced, as an XML document's createElement makes a plain element with no getContext.
465+
const canvas = document.createElementNS("http://www.w3.org/1999/xhtml", "canvas");
466+
canvas.width = width;
467+
canvas.height = height;
468+
const context = canvas.getContext("2d");
469+
context.imageSmoothingQuality = "high";
470+
context.drawImage(bitmap, 0, 0, width, height);
471+
bitmap.close();
472+
return canvas.toDataURL("image/png").slice("data:image/png;base64,".length);
473+
}`;
474+
475+
const shrinkPng = async (cdp: CDPSession, png: string, scale: number) => {
476+
const size = pngSize(png);
477+
const { frameTree } = await cdp.send("Page.getFrameTree");
478+
const { executionContextId } = await cdp.send("Page.createIsolatedWorld", {
479+
frameId: frameTree.frame.id,
480+
worldName: "t3-preview-screenshot",
481+
});
482+
const { result, exceptionDetails } = await cdp.send("Runtime.callFunctionOn", {
483+
functionDeclaration: SHRINK_PNG,
484+
executionContextId,
485+
arguments: [
486+
{ value: png },
487+
{ value: Math.round(size.width * scale) },
488+
{ value: Math.round(size.height * scale) },
489+
],
490+
awaitPromise: true,
491+
returnByValue: true,
492+
});
493+
if (exceptionDetails) throw new Error("The snapshot could not shrink its screenshot.");
494+
return result.value as string;
495+
};
496+
490497
type SnapshotInput = {
491498
readonly page: Page;
492499
readonly cdp: CDPSession;
@@ -497,17 +504,16 @@ type SnapshotInput = {
497504
readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"];
498505
};
499506

500-
/** The scale a page renders at and its viewport in device-independent pixels, a clip's units. */
507+
/** The scale a page renders at and its viewport in device-independent pixels. */
501508
const renderingOf = async (input: SnapshotInput) => {
502509
if (input.renderScale !== "display") {
503510
const viewport = input.page.viewportSize() ?? { width: 1280, height: 800 };
504-
return { renderScale: input.renderScale, viewport, offset: undefined };
511+
return { renderScale: input.renderScale, viewport };
505512
}
506513
// The display ratio a desktop page reports includes its page zoom, and its sizes are CSS pixels.
507514
const page = await readPageMetrics(input.cdp);
508515
return {
509516
renderScale: page.ratio / page.zoom,
510-
offset: page.offset,
511517
viewport: {
512518
width: Math.round(page.width * page.zoom),
513519
height: Math.round(page.height * page.zoom),
@@ -519,7 +525,7 @@ const capture = async (
519525
input: SnapshotInput,
520526
deadline: number,
521527
): Promise<PreviewAutomationSnapshot> => {
522-
const { renderScale, viewport, offset } = await renderingOf(input);
528+
const { renderScale, viewport } = await renderingOf(input);
523529
// A late metrics reply must not begin a capture after viewers have resumed.
524530
if (Date.now() >= deadline) throw tooBusy();
525531
const scale = Math.min(1, MAX_SCREENSHOT_WIDTH / (viewport.width * renderScale));
@@ -536,7 +542,9 @@ const capture = async (
536542
"url" | "title" | "loading" | "visibleText" | "interactiveElements"
537543
>
538544
>,
539-
captureViewport(input.page, input.cdp, { format: "png", scale, viewport, offset }),
545+
captureViewport(input.cdp, { format: "png" }).then((png) =>
546+
scale < 1 ? shrinkPng(input.cdp, png, scale) : png,
547+
),
540548
]);
541549
void pageAndScreenshot.catch(constVoid);
542550
// Leaves time for the second read; a frame that never loads holds the tree until then.

0 commit comments

Comments
 (0)