Skip to content

Commit 0c6b5af

Browse files
Yuruzuucodexmaria-rcks
authored
fix(server): keep the last page state in agent recordings (pingdotgg#33931)
Co-authored-by: Codex <[email protected]> Co-authored-by: maria-rcks <[email protected]>
1 parent fb4bf07 commit 0c6b5af

4 files changed

Lines changed: 140 additions & 4 deletions

File tree

‎apps/server/src/preview/ServerBrowser.test.ts‎

Lines changed: 68 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -261,31 +261,64 @@ function makeContext(onClose?: (context: BrowserContext) => void, initialUrl = "
261261

262262
// Runs the real encoder script against fake canvas and MediaRecorder globals.
263263
function makeEncoderPage() {
264+
let paintedFrame = "";
264265
const canvas = {
265266
width: 0,
266267
height: 0,
267-
getContext: () => ({ drawImage() {}, fillRect() {} }),
268-
captureStream: (frameRate?: number) => ({ frameRate, getTracks: () => [] }),
268+
getContext: () => ({
269+
drawImage(bitmap: { pixels: string }) {
270+
paintedFrame = bitmap.pixels;
271+
},
272+
fillRect() {},
273+
}),
274+
captureStream: (frameRate?: number) => ({
275+
frameRate,
276+
getTracks: () => [],
277+
getVideoTracks: () => [{ requestFrame() {} }],
278+
}),
279+
};
280+
const video = {
281+
play: async () => {},
282+
pause() {},
283+
requestVideoFrameCallback: (callback: () => void) => queueMicrotask(callback),
269284
};
270285
class FakeMediaRecorder {
271286
static isTypeSupported = (type: string) => type === "video/webm";
287+
mimeType = "video/webm";
288+
state = "inactive";
272289
onstart: (() => void) | null = null;
290+
onstop: (() => void) | null = null;
291+
ondataavailable: ((event: { data: Blob }) => void) | null = null;
292+
readonly stream: { frameRate?: number };
273293
constructor(stream: { frameRate?: number }, options: { videoBitsPerSecond: number }) {
294+
this.stream = stream;
274295
encoderStreams.push({
275296
frameRate: stream.frameRate,
276297
bitsPerSecond: options.videoBitsPerSecond,
277298
});
278299
}
279300
start() {
301+
this.state = "recording";
280302
this.onstart?.();
281303
}
304+
stop() {
305+
this.state = "inactive";
306+
this.ondataavailable?.({ data: new Blob([paintedFrame]) });
307+
this.onstop?.();
308+
}
282309
}
283310
const sandbox: Record<string, unknown> = {
284-
document: { createElement: () => canvas },
311+
document: { createElement: (tag: string) => (tag === "video" ? video : canvas) },
285312
Path2D: function Path2D() {},
286313
MediaRecorder: FakeMediaRecorder,
287-
createImageBitmap: async () => ({ width: 1280, height: 800, close() {} }),
314+
createImageBitmap: async (blob: Blob) => ({
315+
width: 1280,
316+
height: 800,
317+
pixels: await blob.text(),
318+
close() {},
319+
}),
288320
atob,
321+
btoa,
289322
Blob,
290323
performance,
291324
};
@@ -826,6 +859,37 @@ it.live.each([
826859
).pipe(Effect.provide(layer)),
827860
);
828861

862+
it.live.each([
863+
{ refreshFails: false, expected: "final frame" },
864+
{ refreshFails: true, expected: "frame" },
865+
])(
866+
"saves the last page state when stopping a recording ($refreshFails)",
867+
({ refreshFails, expected }) =>
868+
Effect.scoped(
869+
Effect.gen(function* () {
870+
const { broker, tabId } = yield* ready;
871+
const fs = yield* FileSystem.FileSystem;
872+
yield* broker.invoke({ scope, tabId, operation: "recordingStart", input: {} });
873+
874+
const cdp = contexts[0]!.sessions[0]!;
875+
const send = cdp.send.getMockImplementation()!;
876+
cdp.send.mockImplementation((method, input) => {
877+
if (method !== "Page.captureScreenshot") return send(method, input);
878+
return refreshFails
879+
? Promise.reject(new Error("Page cannot capture a frame"))
880+
: Promise.resolve({ data: Buffer.from("final frame").toString("base64") });
881+
});
882+
const artifact = yield* broker.invoke<{ path: string }>({
883+
scope,
884+
tabId,
885+
operation: "recordingStop",
886+
input: {},
887+
});
888+
expect(yield* fs.readFileString(artifact.path)).toBe(expected);
889+
}),
890+
).pipe(Effect.provide(layer)),
891+
);
892+
829893
it.live("enforces provider ownership and explicit targets when a session has multiple tabs", () =>
830894
Effect.scoped(
831895
Effect.gen(function* () {

‎apps/server/src/preview/ServerBrowser.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1882,6 +1882,25 @@ const make = Effect.gen(function* () {
18821882
await recording.session.send("Page.stopScreencast").catch(constVoid);
18831883
await Promise.all(recording.framesInFlight);
18841884
await recording.session.detach().catch(constVoid);
1885+
// A screencast can miss the last paint when the page goes idle.
1886+
if (!tab.closing && !tab.dialog) {
1887+
const finalFrame = await ServerBrowserPage.captureFinalRecordingFrame(
1888+
tab.cdp,
1889+
RECORDING_SCREENCAST.quality,
1890+
);
1891+
if (finalFrame !== undefined) {
1892+
await recording.encoder
1893+
.evaluate(
1894+
([data, width]) =>
1895+
(globalThis as unknown as EncoderWindow).__t3Recorder.frame(data, width),
1896+
[
1897+
finalFrame,
1898+
tab.page.viewportSize()?.width ?? UNATTACHED_FILL_VIEWPORT.width,
1899+
] as const,
1900+
)
1901+
.catch(constVoid);
1902+
}
1903+
}
18851904
const stopped = await recording.encoder.evaluate(() =>
18861905
(globalThis as unknown as EncoderWindow).__t3Recorder.stop(),
18871906
);

‎apps/server/src/preview/ServerBrowserPage.test.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -442,6 +442,41 @@ describe("server browser snapshot screenshot", () => {
442442
}).toEqual({ size: [1280, 800], fullSize: [2560, 1600], colors: ["blue", "red", "red"] });
443443
});
444444

445+
it("records the destination of a finite entrance animation without waiting on infinite ones", async () => {
446+
const page = await context.newPage();
447+
await page.setContent(`
448+
<style>
449+
body { margin: 0; background: #2563eb; }
450+
@keyframes enter { from { opacity: 0; } to { opacity: 1; } }
451+
@keyframes pulse { from { opacity: 0.5; } to { opacity: 1; } }
452+
button { animation: pulse 2s infinite; }
453+
#paused { animation: enter 2s paused; }
454+
#overlay { display: none; position: fixed; inset: 0; background: #ef4444; }
455+
#overlay.open { display: block; animation: enter 200ms both; }
456+
</style>
457+
<button onclick="document.querySelector('#overlay').classList.add('open')">Open</button>
458+
<div id="paused"></div>
459+
<div id="overlay"></div>
460+
`);
461+
await page.getByRole("button", { name: "Open" }).click();
462+
const frame = await ServerBrowserPage.captureFinalRecordingFrame(
463+
await context.newCDPSession(page),
464+
80,
465+
);
466+
expect(frame).toBeDefined();
467+
const pixel = (await page.evaluate(`(async () => {
468+
const bytes = Uint8Array.from(atob(${JSON.stringify(frame)}), char => char.charCodeAt(0));
469+
const bitmap = await createImageBitmap(new Blob([bytes], { type: "image/jpeg" }));
470+
const canvas = document.createElement("canvas");
471+
canvas.width = bitmap.width;
472+
canvas.height = bitmap.height;
473+
const context = canvas.getContext("2d");
474+
context.drawImage(bitmap, 0, 0);
475+
return Array.from(context.getImageData(bitmap.width / 2, bitmap.height / 2, 1, 1).data);
476+
})()`)) as [number, number, number, number];
477+
expect(colorOf(pixel)).toBe("red");
478+
});
479+
445480
it("shrinks the screenshot of an XML document, whose createElement makes no canvas", async () => {
446481
const page = await context.newPage();
447482
await page.goto("data:text/xml,<a/>");

‎apps/server/src/preview/ServerBrowserPage.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -475,6 +475,24 @@ export const captureViewport = async (
475475
return data;
476476
};
477477

478+
export const captureFinalRecordingFrame = (cdp: CDPSession, quality: number) =>
479+
answered(
480+
cdp
481+
.send("Runtime.evaluate", {
482+
// Capture the destination of finite transitions, not their initially transparent frame.
483+
expression: `(async () => {
484+
const paint = () => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)));
485+
await paint();
486+
await Promise.allSettled(document.getAnimations()
487+
.filter(animation => animation.playState === "running" && animation.effect?.getComputedTiming().iterations !== Infinity)
488+
.map(animation => animation.finished));
489+
await paint();
490+
})()`,
491+
awaitPromise: true,
492+
})
493+
.then(() => captureViewport(cdp, { format: "jpeg", quality })),
494+
);
495+
478496
/** Runs in the page's own isolated world, as the server has no image library. */
479497
const SHRINK_PNG = `async function (png, width, height) {
480498
const bytes = Uint8Array.from(atob(png), (char) => char.charCodeAt(0));

0 commit comments

Comments
 (0)