Skip to content

Commit d98b8c5

Browse files
Kiri110Kclaude
andcommitted
fix(desktop): capture fresh preview frames in background windows
Request a redrawn Chromium screenshot instead of returning the guest's cached frame. Keep the host painting and request copies while the redraw is pending so occluded guests present the frame promptly. Stop those requests on completion or abort. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 8d0bec5 commit d98b8c5

2 files changed

Lines changed: 199 additions & 8 deletions

File tree

‎apps/desktop/src/preview/Manager.test.ts‎

Lines changed: 136 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,7 @@ describe("previewWindowOpenAction", () => {
182182
const {
183183
browserWindowConstructor,
184184
clipboardItemConstructor,
185+
createFromBuffer,
185186
createFromPath,
186187
fromId,
187188
getFocusedWebContents,
@@ -193,6 +194,11 @@ const {
193194
} = vi.hoisted(() => ({
194195
browserWindowConstructor: vi.fn(),
195196
clipboardItemConstructor: vi.fn(),
197+
createFromBuffer: vi.fn((data: Buffer) => ({
198+
isEmpty: () => data.length === 0,
199+
getSize: () => ({ width: 800, height: 600 }),
200+
toPNG: () => data,
201+
})),
196202
createFromPath: vi.fn((): { readonly isEmpty: () => boolean; readonly toPNG: () => Buffer } => ({
197203
isEmpty: () => false,
198204
toPNG: () => Buffer.from("png"),
@@ -217,6 +223,7 @@ vi.mock("electron", () => ({
217223
write: writeClipboard,
218224
},
219225
nativeImage: {
226+
createFromBuffer,
220227
createFromPath,
221228
},
222229
shell: {
@@ -2309,6 +2316,102 @@ describe("PreviewManager", () => {
23092316
),
23102317
);
23112318

2319+
effectIt.effect(
2320+
"snapshots a frame redrawn after reading the page, not the guest's last frame",
2321+
() =>
2322+
withManager((manager) =>
2323+
Effect.gen(function* () {
2324+
const setBackgroundThrottling = vi.fn<(enabled: boolean) => void>();
2325+
// capturePage only nudges the window to draw; its stale pixels must not be the snapshot.
2326+
const copies: Array<PromiseWithResolvers<TestCapturedPreviewImage>> = [];
2327+
const capturePage = vi.fn((..._args: ReadonlyArray<unknown>) => {
2328+
const copy = Promise.withResolvers<TestCapturedPreviewImage>();
2329+
copies.push(copy);
2330+
return copy.promise;
2331+
});
2332+
const drawnCopy = {
2333+
isEmpty: () => false,
2334+
toJPEG: () => Buffer.from("stale"),
2335+
getSize: () => ({ width: 1, height: 1 }),
2336+
};
2337+
const settlePromises = Effect.promise(
2338+
() => new Promise<void>((resolve) => setImmediate(resolve)),
2339+
);
2340+
const redrawnPng = Buffer.from("png drawn after the page changed");
2341+
const pendingRedraw = Promise.withResolvers<{ data: string }>();
2342+
const screenshotResults = [Promise.resolve({}), pendingRedraw.promise];
2343+
const sendCommand = vi.fn(async (method: string): Promise<unknown> => {
2344+
if (method === "Page.captureScreenshot") return screenshotResults.shift();
2345+
if (method === "Runtime.evaluate") {
2346+
return {
2347+
result: {
2348+
value: {
2349+
url: "https://example.com",
2350+
title: "Example",
2351+
loading: false,
2352+
visibleText: "QA new marker",
2353+
interactiveElements: [],
2354+
},
2355+
},
2356+
};
2357+
}
2358+
return method === "Accessibility.getFullAXTree" ? { nodes: [] } : undefined;
2359+
});
2360+
const wc = makeTestPreviewWebContents(capturePage);
2361+
Object.assign(wc, { isDevToolsOpened: () => false });
2362+
Object.assign(wc.debugger, { sendCommand });
2363+
fromId.mockReturnValue(wc);
2364+
yield* manager.createTab("tab_1");
2365+
yield* manager.registerWebview("tab_1", 42);
2366+
yield* manager.setMainWindow({
2367+
isDestroyed: () => false,
2368+
once: vi.fn(),
2369+
webContents: { setBackgroundThrottling },
2370+
} as never);
2371+
2372+
const snapshot = yield* manager
2373+
.automationSnapshot("tab_1")
2374+
.pipe(Effect.forkChild({ startImmediately: true }));
2375+
// The first capture returned no image; the retry waits on Chromium's redraw.
2376+
yield* TestClock.adjust(200);
2377+
const screenshotCalls = sendCommand.mock.calls
2378+
.map(([method], index) => ({
2379+
method,
2380+
order: sendCommand.mock.invocationCallOrder[index]!,
2381+
}))
2382+
.filter(({ method }) => method === "Page.captureScreenshot");
2383+
expect(screenshotCalls).toHaveLength(2);
2384+
const pageRead = sendCommand.mock.calls.findIndex(
2385+
([method]) => method === "Runtime.evaluate",
2386+
);
2387+
expect(screenshotCalls[0]!.order).toBeGreaterThan(
2388+
sendCommand.mock.invocationCallOrder[pageRead]!,
2389+
);
2390+
// Presenting the redrawn frame needs the main window to keep drawing.
2391+
expect(setBackgroundThrottling.mock.calls).toEqual([[false]]);
2392+
// Whole-view copies that leave the page hidden make the window draw the guest.
2393+
expect(capturePage).toHaveBeenLastCalledWith(undefined, { stayHidden: true });
2394+
2395+
// Every drawn copy is followed by another until the redraw arrives.
2396+
const copiesBeforeDraw = capturePage.mock.calls.length;
2397+
copies.at(-1)!.resolve(drawnCopy);
2398+
yield* settlePromises;
2399+
expect(capturePage).toHaveBeenCalledTimes(copiesBeforeDraw + 1);
2400+
2401+
pendingRedraw.resolve({ data: redrawnPng.toString("base64") });
2402+
const result = yield* Fiber.join(snapshot);
2403+
expect(result.visibleText).toBe("QA new marker");
2404+
expect(result.screenshot.data).toBe(redrawnPng.toString("base64"));
2405+
expect(setBackgroundThrottling.mock.calls).toEqual([[false], [true]]);
2406+
2407+
// Once the screenshot settles, the copy in flight is the last one.
2408+
for (const copy of copies) copy.resolve(drawnCopy);
2409+
yield* settlePromises;
2410+
expect(capturePage).toHaveBeenCalledTimes(copiesBeforeDraw + 1);
2411+
}),
2412+
),
2413+
);
2414+
23122415
effectIt.effect("keeps every recorded guest unthrottled until its frame capture stops", () =>
23132416
withManager((manager) =>
23142417
Effect.gen(function* () {
@@ -2694,11 +2797,26 @@ describe("PreviewManager", () => {
26942797
effectIt.effect("releases snapshot control when every capture attempt stalls", () =>
26952798
withManager((manager) =>
26962799
Effect.gen(function* () {
2697-
const capturePage = vi.fn(() => new Promise<TestCapturedPreviewImage>(() => {}));
2800+
const copies: Array<PromiseWithResolvers<TestCapturedPreviewImage>> = [];
2801+
const capturePage = vi.fn(() => {
2802+
const copy = Promise.withResolvers<TestCapturedPreviewImage>();
2803+
copies.push(copy);
2804+
return copy.promise;
2805+
});
2806+
const copyImage = (empty: boolean) => ({
2807+
isEmpty: () => empty,
2808+
toJPEG: () => Buffer.from("copy"),
2809+
getSize: () => ({ width: empty ? 0 : 1, height: empty ? 0 : 1 }),
2810+
});
2811+
const settlePromises = Effect.promise(
2812+
() => new Promise<void>((resolve) => setImmediate(resolve)),
2813+
);
2814+
const captureScreenshot = vi.fn(() => new Promise<never>(() => {}));
26982815
const wc = makeTestPreviewWebContents(capturePage);
26992816
Object.assign(wc, { isDevToolsOpened: () => false });
27002817
Object.assign(wc.debugger, {
27012818
sendCommand: vi.fn(async (method: string, params?: Record<string, unknown>) => {
2819+
if (method === "Page.captureScreenshot") return captureScreenshot();
27022820
if (method === "Runtime.evaluate") {
27032821
return {
27042822
result: {
@@ -2726,21 +2844,37 @@ describe("PreviewManager", () => {
27262844
Effect.forkChild({ startImmediately: true }),
27272845
);
27282846
yield* TestClock.adjust(100);
2847+
expect(captureScreenshot).toHaveBeenCalledOnce();
27292848
expect(capturePage).toHaveBeenCalledOnce();
27302849
const evaluate = yield* manager
27312850
.automationEvaluate("tab_1", { expression: "42" })
27322851
.pipe(Effect.forkChild({ startImmediately: true }));
27332852
expect(evaluate.pollUnsafe()).toBeUndefined();
27342853

2854+
// The first attempt timed out; its copy landing afterwards must not request another.
2855+
yield* TestClock.adjust(950);
2856+
copies[0]!.resolve(copyImage(false));
2857+
yield* settlePromises;
2858+
expect(capturePage).toHaveBeenCalledOnce();
2859+
2860+
// An empty copy did not make the window draw, so the retry stops requesting copies.
2861+
yield* TestClock.adjust(100);
2862+
expect(captureScreenshot).toHaveBeenCalledTimes(2);
2863+
expect(capturePage).toHaveBeenCalledTimes(2);
2864+
copies[1]!.resolve(copyImage(true));
2865+
yield* settlePromises;
2866+
expect(capturePage).toHaveBeenCalledTimes(2);
2867+
27352868
yield* TestClock.adjust(4_000);
27362869
const exit = yield* Fiber.join(snapshot);
27372870
expect(Exit.isFailure(exit)).toBe(true);
2871+
expect(captureScreenshot).toHaveBeenCalledTimes(3);
27382872
expect(capturePage).toHaveBeenCalledTimes(3);
27392873
if (Exit.isSuccess(exit)) return;
27402874
const error = Option.getOrThrow(Cause.findErrorOption(exit.cause));
27412875
expect(error).toMatchObject({
27422876
_tag: "PreviewOperationError",
2743-
operation: "automationSnapshot.capturePage",
2877+
operation: "automationSnapshot.captureScreenshot",
27442878
tabId: "tab_1",
27452879
webContentsId: 42,
27462880
cause: { _tag: "TimeoutError" },

‎apps/desktop/src/preview/Manager.ts‎

Lines changed: 63 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,60 @@ const normalizeCaptureRect = (value: unknown): PreviewAnnotationRect | null => {
343343
};
344344
};
345345

346+
/**
347+
* Captures a guest frame drawn after this call. `capturePage` copies whatever frame the guest
348+
* last drew, and a guest hidden behind a minimized or hidden window has not drawn since its DOM
349+
* changed, so its copy shows the previous state. `Page.captureScreenshot` keeps the guest
350+
* painting while hidden, forces a redraw, and copies only after that frame is presented.
351+
* Needs an attached debugger, so it serves agent snapshots. The screenshot button and
352+
* annotations keep `capturePage`: a person triggers them from a visible preview whose guest
353+
* is already drawing what they see.
354+
*/
355+
const captureRedrawnFrame = async (
356+
wc: Electron.WebContents,
357+
signal: AbortSignal,
358+
): Promise<Electron.NativeImage> => {
359+
const screenshot = wc.debugger.sendCommand("Page.captureScreenshot", {
360+
format: "png",
361+
// Decoded and re-encoded below, so trade compression for main-thread time.
362+
optimizeForSpeed: true,
363+
});
364+
let settled = false;
365+
const settle = () => {
366+
settled = true;
367+
};
368+
screenshot.then(settle, settle);
369+
// The window does not draw a guest it does not show (behind the app, or in a minimized
370+
// window) except to serve a copy request, and Chromium throttles an undrawn guest to one
371+
// frame a second, so the forced redraw is presented seconds late. Each copy makes the window
372+
// draw the guest's latest frame, so keep requesting copies, one at a time, until the
373+
// screenshot settles or this attempt is abandoned. A copy that fails or comes back empty ends
374+
// the requests rather than spinning; the attempt then falls back to its timeout and retry.
375+
// stayHidden leaves the page's visibility state alone, and Chromium only allows it for a
376+
// whole-view copy (an empty capture size).
377+
const requestDraw = (): void => {
378+
if (settled || signal.aborted || wc.isDestroyed()) return;
379+
wc.capturePage(undefined, { stayHidden: true }).then(
380+
(copy) => {
381+
if (!copy.isEmpty()) requestDraw();
382+
},
383+
() => undefined,
384+
);
385+
};
386+
requestDraw();
387+
const result: unknown = await screenshot;
388+
const data =
389+
typeof result === "object" && result !== null
390+
? (result as Record<string, unknown>)["data"]
391+
: undefined;
392+
const image =
393+
typeof data === "string" ? nativeImage.createFromBuffer(Buffer.from(data, "base64")) : null;
394+
if (!image || image.isEmpty()) {
395+
throw new Error("Page.captureScreenshot returned no image.");
396+
}
397+
return image;
398+
};
399+
346400
/** `capturePage` never settles when the guest's compositor is wedged. */
347401
const ANNOTATION_SCREENSHOT_TIMEOUT = "5 seconds";
348402

@@ -709,10 +763,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
709763
try: evaluate,
710764
catch: (cause) => new PreviewOperationError({ ...errorContext, cause }),
711765
});
712-
const capturePageWithRetry = Effect.fn("PreviewManager.capturePageWithRetry")(function* (
766+
const captureGuestWithRetry = Effect.fn("PreviewManager.captureGuestWithRetry")(function* (
713767
errorContext: PreviewOperationContext,
714768
tabId: string,
715769
wc: Electron.WebContents,
770+
captureImage: (wc: Electron.WebContents, signal: AbortSignal) => Promise<Electron.NativeImage>,
716771
) {
717772
const requireCurrentGuest = Effect.gen(function* () {
718773
const tabs = yield* SynchronizedRef.get(tabsRef);
@@ -725,7 +780,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
725780
yield* requireCurrentGuest;
726781
const image = yield* Effect.tryPromise({
727782
// An abort-signal parameter makes a stalled promise interruptible.
728-
try: (_signal) => wc.capturePage(),
783+
try: (signal) => captureImage(wc, signal),
729784
catch: (cause) => new PreviewOperationError({ ...errorContext, cause }),
730785
}).pipe(
731786
Effect.timeout(CAPTURE_PAGE_ATTEMPT_TIMEOUT_MS),
@@ -737,7 +792,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
737792
yield* requireCurrentGuest;
738793
return image;
739794
});
740-
// capturePage waits for a window frame that contains the guest. A throttled main window
795+
// Both captures wait for a window frame that contains the guest. A throttled main window
741796
// that is covered, minimized, or on another Space draws none, so the capture never settles.
742797
return yield* Effect.acquireUseRelease(
743798
holdMainWindowPainting,
@@ -2883,14 +2938,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
28832938
const [createdAt, millis, image] = yield* Effect.all([
28842939
currentIso,
28852940
currentMillis,
2886-
capturePageWithRetry(
2941+
captureGuestWithRetry(
28872942
{
28882943
operation: "captureScreenshot.capturePage",
28892944
tabId,
28902945
webContentsId: wc.id,
28912946
},
28922947
tabId,
28932948
wc,
2949+
(guest) => guest.capturePage(),
28942950
),
28952951
]);
28962952
const id = `browser-screenshot-${artifactSiteSlug(wc.getURL())}-${millis.toString(36)}`;
@@ -3758,14 +3814,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
37583814
);
37593815
const [accessibility, sourceImage, diagnostics, timelines] = yield* Effect.all([
37603816
send("Accessibility.getFullAXTree"),
3761-
capturePageWithRetry(
3817+
captureGuestWithRetry(
37623818
{
3763-
operation: "automationSnapshot.capturePage",
3819+
operation: "automationSnapshot.captureScreenshot",
37643820
tabId,
37653821
webContentsId: wc.id,
37663822
},
37673823
tabId,
37683824
wc,
3825+
captureRedrawnFrame,
37693826
),
37703827
Ref.get(diagnosticsRef),
37713828
Ref.get(actionTimelineRef),

0 commit comments

Comments
 (0)