Skip to content

Commit 3d21618

Browse files
committed
fix(desktop): capture background preview screenshots
1 parent 5d27ecd commit 3d21618

9 files changed

Lines changed: 379 additions & 7 deletions

File tree

‎apps/desktop/src/preview/DesktopBrowserHost.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ const makeDebuggee = () => {
3838
tab: {
3939
webContents: webContents as unknown as Electron.WebContents,
4040
debugger: debuggee as unknown as Electron.Debugger,
41+
captureScreenshot: (_params: Record<string, unknown>, _signal: AbortSignal) =>
42+
debuggee.sendCommand("Page.captureScreenshot"),
4143
},
4244
emit: (method: string, params: unknown) => emitter.emit("message", {}, method, params, ""),
4345
release: () => pending.splice(0).forEach((resolve) => resolve()),
@@ -52,6 +54,43 @@ const takeEvents = (host: DesktopBrowserHost.DesktopBrowserHost["Service"], coun
5254
);
5355

5456
describe("DesktopBrowserHost", () => {
57+
it.effect.each(["release", "detach", "reconnect"] as const)(
58+
"interrupts pending screenshots on %s",
59+
(reset) =>
60+
Effect.gen(function* () {
61+
const host = yield* DesktopBrowserHost.make;
62+
const debuggee = makeDebuggee();
63+
const signals: AbortSignal[] = [];
64+
debuggee.tab.captureScreenshot = (_params, signal) => {
65+
signals.push(signal);
66+
return new Promise((_resolve, reject) => {
67+
signal.addEventListener("abort", () => reject(new Error("capture aborted")), {
68+
once: true,
69+
});
70+
});
71+
};
72+
host.attach(key, debuggee.tab);
73+
yield* host.handleCommandLine(
74+
encodeJson({
75+
type: "cdp",
76+
...key,
77+
message: encodeJson({
78+
id: 1,
79+
method: "Page.captureScreenshot",
80+
sessionId: "t3-preview-page",
81+
}),
82+
}),
83+
);
84+
expect(signals).toHaveLength(1);
85+
expect(signals[0]?.aborted).toBe(false);
86+
if (reset === "release")
87+
yield* host.handleCommandLine(encodeJson({ type: "release", ...key }));
88+
else if (reset === "detach") host.detach(key);
89+
else yield* takeEvents(host, 1);
90+
expect(signals[0]?.aborted).toBe(true);
91+
}),
92+
);
93+
5594
it.effect("announces tabs already attached to a backend that starts later", () =>
5695
Effect.gen(function* () {
5796
const host = yield* DesktopBrowserHost.make;

‎apps/desktop/src/preview/DesktopBrowserHost.ts‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ export interface DesktopBrowserTabKey {
3535
export interface DesktopBrowserTabDebugger {
3636
readonly webContents: Electron.WebContents;
3737
readonly debugger: Electron.Debugger;
38+
readonly captureScreenshot: (
39+
params: Record<string, unknown>,
40+
signal: AbortSignal,
41+
) => Promise<unknown>;
3842
}
3943

4044
const keyOf = ({ threadId, tabId }: DesktopBrowserTabKey) => `${threadId}\u0000${tabId}`;
@@ -43,6 +47,7 @@ interface AttachedTab {
4347
readonly key: DesktopBrowserTabKey;
4448
readonly debuggee: DesktopBrowserTabDebugger;
4549
relay: CdpRelayConnection | null;
50+
readonly captures: Set<AbortController>;
4651
downloadDirectory: string | null;
4752
pendingDownloadGuid: string | null;
4853
agentInputAt: number;
@@ -84,15 +89,29 @@ export const make = Effect.gen(function* () {
8489
const tabs = new Map<string, AttachedTab>();
8590
const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event));
8691

92+
const resetRelay = (tab: AttachedTab) => {
93+
tab.relay = null;
94+
for (const controller of tab.captures) controller.abort();
95+
tab.captures.clear();
96+
};
97+
8798
const relayFor = (tab: AttachedTab) => {
8899
if (tab.relay) return tab.relay;
89100
const { webContents, debugger: debuggee } = tab.debuggee;
90101
const relay: CdpRelayConnection = createCdpRelayConnection(
91102
{
92-
send: (method, params, sessionId) =>
93-
sessionId === undefined
103+
send: (method, params, sessionId) => {
104+
if (method === "Page.captureScreenshot" && sessionId === undefined) {
105+
const controller = new AbortController();
106+
tab.captures.add(controller);
107+
return tab.debuggee.captureScreenshot(params, controller.signal).finally(() => {
108+
tab.captures.delete(controller);
109+
});
110+
}
111+
return sessionId === undefined
94112
? sendPageCommand(webContents, debuggee, method, params)
95-
: debuggee.sendCommand(method, params, sessionId),
113+
: debuggee.sendCommand(method, params, sessionId);
114+
},
96115
targetId: () =>
97116
debuggee
98117
.sendCommand("Target.getTargetInfo")
@@ -133,6 +152,7 @@ export const make = Effect.gen(function* () {
133152
const tab = tabs.get(id);
134153
if (!tab) return;
135154
tabs.delete(id);
155+
resetRelay(tab);
136156
tab.debuggee.debugger.off("message", tab.onMessage);
137157
emit({ type: "detached", ...key });
138158
};
@@ -145,6 +165,7 @@ export const make = Effect.gen(function* () {
145165
key,
146166
debuggee,
147167
relay: null,
168+
captures: new Set(),
148169
downloadDirectory: null,
149170
pendingDownloadGuid: null,
150171
agentInputAt: Number.NEGATIVE_INFINITY,
@@ -173,7 +194,7 @@ export const make = Effect.gen(function* () {
173194
return;
174195
}
175196
if (command.value.type === "release") {
176-
tab.relay = null;
197+
resetRelay(tab);
177198
return;
178199
}
179200
if (AGENT_INPUT_COMMAND.test(command.value.message)) tab.agentInputAt = performance.now();
@@ -189,7 +210,7 @@ export const make = Effect.gen(function* () {
189210
Effect.forEach(
190211
[...tabs.values()],
191212
(tab) => {
192-
tab.relay = null;
213+
resetRelay(tab);
193214
return PubSub.publish(outbox, { type: "attached", ...tab.key });
194215
},
195216
{ discard: true },

‎apps/desktop/src/preview/Manager.ts‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ import * as DesktopEnvironment from "../app/DesktopEnvironment.ts";
6565
import * as DesktopRendererHistory from "../telemetry/DesktopRendererHistory.ts";
6666
import { MENU_ACTION_CHANNEL, PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL } from "../ipc/channels.ts";
6767
import * as DesktopBrowserHost from "./DesktopBrowserHost.ts";
68+
import { captureFreshScreenshot } from "./ScreenshotCapture.ts";
6869
import * as PreviewPasskeys from "./Passkeys.ts";
6970
import * as BrowserSession from "./BrowserSession.ts";
7071
import {
@@ -391,7 +392,7 @@ interface ManagedListeners {
391392
readonly webContents: Electron.WebContents;
392393
}
393394

394-
type FrameCaptureConsumer = "picture-in-picture" | "recording";
395+
type FrameCaptureConsumer = "picture-in-picture" | "recording" | "screenshot";
395396

396397
interface FrameCaptureSession {
397398
readonly recordingInputOptions?: RecordingInputOptions;
@@ -556,6 +557,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
556557
const path = yield* Path.Path;
557558
const crypto = yield* Crypto.Crypto;
558559
const parentScope = yield* Scope.Scope;
560+
const runCapture = Effect.runPromiseWith(yield* Effect.context<never>());
559561
const browserHost = yield* DesktopBrowserHost.DesktopBrowserHost;
560562
const passkeys = yield* PreviewPasskeys.PreviewPasskeys;
561563
const context = yield* Effect.context<never>();
@@ -2387,7 +2389,38 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
23872389
}
23882390
if (afterAttach.serverTab) {
23892391
yield* listenForAgentPointers;
2390-
browserHost.attach(afterAttach.serverTab, { webContents: wc, debugger: control.debugger });
2392+
browserHost.attach(afterAttach.serverTab, {
2393+
webContents: wc,
2394+
debugger: control.debugger,
2395+
captureScreenshot: (params, signal) =>
2396+
runCapture(
2397+
Effect.acquireUseRelease(
2398+
Effect.forkIn(
2399+
control.semaphore.withPermit(
2400+
Effect.scoped(
2401+
Effect.gen(function* () {
2402+
yield* Effect.acquireRelease(startFrameCapture(tabId, "screenshot"), () =>
2403+
stopFrameCapture(tabId, "screenshot").pipe(Effect.ignore),
2404+
);
2405+
const token = yield* crypto.randomUUIDv4();
2406+
return yield* captureFreshScreenshot(
2407+
wc,
2408+
control.debugger,
2409+
params,
2410+
tabId,
2411+
token,
2412+
);
2413+
}),
2414+
),
2415+
),
2416+
control.scope,
2417+
),
2418+
Fiber.join,
2419+
Fiber.interrupt,
2420+
),
2421+
{ signal },
2422+
),
2423+
});
23912424
}
23922425
if (afterAttach.colorScheme !== "system") {
23932426
yield* attemptPromise({ operation: "applyColorScheme", tabId, webContentsId: wc.id }, () =>
Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
import { describe, expect, it } from "@effect/vitest";
2+
import { DESKTOP_PREVIEW_SCREENSHOT_ACTIVITY } from "@t3tools/contracts";
3+
import * as Cause from "effect/Cause";
4+
import * as Effect from "effect/Effect";
5+
import * as Exit from "effect/Exit";
6+
import * as Fiber from "effect/Fiber";
7+
import { TestClock } from "effect/testing";
8+
import { vi } from "vite-plus/test";
9+
10+
import { captureFreshScreenshot } from "./ScreenshotCapture.ts";
11+
12+
const makeHarness = () => {
13+
const reply = Promise.withResolvers<unknown>();
14+
const copying = Promise.withResolvers<void>();
15+
const copy = Promise.withResolvers<Electron.NativeImage>();
16+
const activity: unknown[][] = [];
17+
const image = { isEmpty: () => false } as Electron.NativeImage;
18+
const globals = {
19+
[DESKTOP_PREVIEW_SCREENSHOT_ACTIVITY]: (...args: unknown[]) => {
20+
activity.push(args);
21+
return true;
22+
},
23+
};
24+
const sendCommand = vi.fn(() => reply.promise);
25+
const capturePage = vi.fn(() => {
26+
copying.resolve();
27+
return copy.promise;
28+
});
29+
const contents = {
30+
hostWebContents: {
31+
isDestroyed: () => false,
32+
executeJavaScript: async (script: string) =>
33+
new Function("globalThis", `return ${script}`)(globals),
34+
},
35+
capturePage,
36+
} as unknown as Electron.WebContents;
37+
const debuggerSession = { sendCommand } as unknown as Electron.Debugger;
38+
const capture = (params: Record<string, unknown> = {}) =>
39+
captureFreshScreenshot(contents, debuggerSession, params, "runtime-tab", "capture-token");
40+
return { reply, copying, copy, image, activity, sendCommand, capturePage, capture };
41+
};
42+
43+
describe("captureFreshScreenshot", () => {
44+
it.effect("drives an occluded capture without replacing the CDP image or its parameters", () =>
45+
Effect.gen(function* () {
46+
const h = makeHarness();
47+
const params = {
48+
format: "jpeg",
49+
quality: 65,
50+
clip: { x: 2, y: 3, width: 40, height: 50, scale: 0.5 },
51+
};
52+
const fiber = yield* h.capture(params).pipe(Effect.forkChild);
53+
yield* Effect.promise(() => h.copying.promise);
54+
h.reply.resolve({ data: "fresh-cdp-image" });
55+
expect(yield* Fiber.join(fiber)).toEqual({ data: "fresh-cdp-image" });
56+
h.copy.resolve(h.image);
57+
expect(h.sendCommand).toHaveBeenCalledExactlyOnceWith("Page.captureScreenshot", params);
58+
expect(h.activity.map((args) => args[2])).toEqual([true, false]);
59+
expect(h.activity[0]?.slice(0, 2)).toEqual(h.activity[1]?.slice(0, 2));
60+
expect(h.capturePage).toHaveBeenCalledTimes(1);
61+
}),
62+
);
63+
64+
it.effect("retries a cold guest copy while preserving the pending CDP capture", () =>
65+
Effect.gen(function* () {
66+
const h = makeHarness();
67+
const firstCopy = Promise.withResolvers<void>();
68+
h.capturePage.mockImplementationOnce(() => {
69+
firstCopy.resolve();
70+
return Promise.reject(new Error("UnknownVizError"));
71+
});
72+
const fiber = yield* h.capture().pipe(Effect.forkChild);
73+
yield* Effect.promise(() => firstCopy.promise);
74+
yield* TestClock.adjust("120 millis");
75+
yield* Effect.promise(() => h.copying.promise);
76+
h.reply.resolve({ data: "fresh-after-cold-start" });
77+
expect(yield* Fiber.join(fiber)).toEqual({ data: "fresh-after-cold-start" });
78+
h.copy.resolve(h.image);
79+
expect(h.capturePage).toHaveBeenCalledTimes(2);
80+
expect(h.sendCommand).toHaveBeenCalledTimes(1);
81+
}),
82+
);
83+
84+
it.effect("bounds a stalled native copy and releases renderer activity", () =>
85+
Effect.gen(function* () {
86+
const h = makeHarness();
87+
const fiber = yield* h.capture().pipe(Effect.forkChild);
88+
yield* Effect.promise(() => h.copying.promise);
89+
yield* TestClock.adjust("3 seconds");
90+
expect(Exit.isFailure(yield* Fiber.await(fiber))).toBe(true);
91+
expect(h.activity.map((args) => args[2])).toEqual([true, false]);
92+
h.copy.resolve(h.image);
93+
expect(h.capturePage).toHaveBeenCalledTimes(1);
94+
}),
95+
);
96+
97+
it.effect("releases activity when its relay is interrupted while a copy is pending", () =>
98+
Effect.gen(function* () {
99+
const h = makeHarness();
100+
const fiber = yield* h.capture().pipe(Effect.forkChild);
101+
yield* Effect.promise(() => h.copying.promise);
102+
yield* Fiber.interrupt(fiber);
103+
h.copy.resolve(h.image);
104+
expect(h.activity.map((args) => args[2])).toEqual([true, false]);
105+
expect(h.capturePage).toHaveBeenCalledTimes(1);
106+
}),
107+
);
108+
109+
it.effect("stops on an unavailable native surface instead of repeatedly copying it", () =>
110+
Effect.gen(function* () {
111+
const h = makeHarness();
112+
h.copy.resolve({ isEmpty: () => true } as Electron.NativeImage);
113+
const result = yield* h.capture().pipe(Effect.exit);
114+
expect(Exit.isFailure(result)).toBe(true);
115+
if (Exit.isFailure(result))
116+
expect(Cause.pretty(result.cause)).toContain("surface is unavailable");
117+
expect(h.activity.map((args) => args[2])).toEqual([true, false]);
118+
expect(h.capturePage).toHaveBeenCalledTimes(1);
119+
}),
120+
);
121+
});
Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import { DESKTOP_PREVIEW_SCREENSHOT_ACTIVITY } from "@t3tools/contracts";
2+
import * as Effect from "effect/Effect";
3+
import * as Schema from "effect/Schema";
4+
import * as Schedule from "effect/Schedule";
5+
6+
export class ScreenshotCaptureError extends Schema.TaggedError<ScreenshotCaptureError>()(
7+
"ScreenshotCaptureError",
8+
{ message: Schema.String },
9+
) {}
10+
11+
export const captureFreshScreenshot = Effect.fn("Preview.captureFreshScreenshot")(function* (
12+
contents: Electron.WebContents,
13+
debuggee: Electron.Debugger,
14+
params: Record<string, unknown>,
15+
runtimeTabId: string,
16+
token: string,
17+
) {
18+
const host = contents.hostWebContents;
19+
if (!host || host.isDestroyed()) {
20+
return yield* Effect.fail(
21+
new ScreenshotCaptureError({ message: "The desktop preview host window is unavailable." }),
22+
);
23+
}
24+
const expression = (active: boolean) =>
25+
`globalThis[${JSON.stringify(DESKTOP_PREVIEW_SCREENSHOT_ACTIVITY)}]?.(${JSON.stringify(runtimeTabId)},${JSON.stringify(token)},${active}) === true`;
26+
const screenshot = Effect.tryPromise(() =>
27+
debuggee.sendCommand("Page.captureScreenshot", params),
28+
);
29+
// CDP requests a redraw, but an occluded guest needs page copies to present it.
30+
// Discard those copies: only the CDP reply carries the requested clip and scale.
31+
const copies = Effect.forever(
32+
Effect.tryPromise(() => contents.capturePage()).pipe(
33+
// A cold guest can reject its first copy with Chromium's UnknownVizError.
34+
Effect.retry({ times: 2, schedule: Schedule.spaced("120 millis") }),
35+
Effect.filterOrFail(
36+
(image) => !image.isEmpty(),
37+
() =>
38+
new ScreenshotCaptureError({
39+
message: "The desktop preview capture surface is unavailable.",
40+
}),
41+
),
42+
Effect.andThen(Effect.sleep("16 millis")),
43+
),
44+
);
45+
return yield* Effect.scoped(
46+
Effect.gen(function* () {
47+
// Register cleanup before preparation: a late renderer reply must not leave
48+
// an activity lease behind after the caller's deadline or disconnect.
49+
yield* Effect.addFinalizer(() =>
50+
Effect.tryPromise(() => host.executeJavaScript(expression(false))).pipe(
51+
Effect.timeout("1 second"),
52+
Effect.ignore,
53+
),
54+
);
55+
yield* Effect.tryPromise(() => host.executeJavaScript(expression(true))).pipe(
56+
Effect.filterOrFail(
57+
(prepared) => prepared === true,
58+
() =>
59+
new ScreenshotCaptureError({
60+
message: "The desktop preview rendering surface is unavailable.",
61+
}),
62+
),
63+
);
64+
return yield* screenshot.pipe(Effect.raceFirst(copies));
65+
}),
66+
).pipe(Effect.timeout("3 seconds"));
67+
});

0 commit comments

Comments
 (0)