Skip to content

Commit 57095e6

Browse files
fix(desktop): prepare inactive guests before agent capture
1 parent a9e29f5 commit 57095e6

8 files changed

Lines changed: 325 additions & 21 deletions

File tree

‎apps/desktop/src/ipc/channels.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ export const PREVIEW_RECORDING_START_CHANNEL = "desktop:preview-recording-start"
109109
export const PREVIEW_RECORDING_STOP_CHANNEL = "desktop:preview-recording-stop";
110110
export const PREVIEW_RECORDING_SAVE_CHANNEL = "desktop:preview-recording-save";
111111
export const PREVIEW_RECORDING_FRAME_CHANNEL = "desktop:preview-recording-frame";
112+
export const PREVIEW_CAPTURE_READY_CHANNEL = "desktop:preview-capture-ready";
112113
export const PREVIEW_STATE_CHANGE_CHANNEL = "desktop:preview-state-change";
113114
export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event";
114115
export const PREVIEW_OPEN_LINK_CHANNEL = "desktop:preview-open-link";

‎apps/desktop/src/ipc/methods/preview.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
DesktopPreviewRecordingArtifactSchema,
88
DesktopPreviewRecordingSaveInputSchema,
99
DesktopPreviewRegisterWebviewInputSchema,
10+
DesktopPreviewCaptureReadyInputSchema,
1011
DesktopPreviewScreenshotArtifactSchema,
1112
DesktopPreviewSetAudioMutedInputSchema,
1213
DesktopPreviewSetColorSchemeInputSchema,
@@ -104,6 +105,16 @@ export const registerWebview = DesktopIpc.makeIpcMethod({
104105
}),
105106
});
106107

108+
export const captureReady = DesktopIpc.makeIpcMethod({
109+
channel: IpcChannels.PREVIEW_CAPTURE_READY_CHANNEL,
110+
payload: DesktopPreviewCaptureReadyInputSchema,
111+
result: Schema.Void,
112+
handler: Effect.fn("desktop.ipc.preview.captureReady")(function* (input) {
113+
const manager = yield* PreviewManager.PreviewManager;
114+
yield* manager.captureReady(input.tabId, input.webContentsId, input.requestId);
115+
}),
116+
});
117+
107118
export const navigate = DesktopIpc.makeIpcMethod({
108119
channel: IpcChannels.PREVIEW_NAVIGATE_CHANNEL,
109120
payload: DesktopPreviewNavigateInputSchema,
@@ -432,6 +443,7 @@ export const methods = [
432443
createTab,
433444
closeTab,
434445
registerWebview,
446+
captureReady,
435447
navigate,
436448
goBack,
437449
goForward,

‎apps/desktop/src/preload.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,12 @@ contextBridge.exposeInMainWorld("desktopBridge", {
327327
closeTab: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_CLOSE_TAB_CHANNEL, { tabId }),
328328
registerWebview: (tabId, webContentsId) =>
329329
ipcRenderer.invoke(IpcChannels.PREVIEW_REGISTER_WEBVIEW_CHANNEL, { tabId, webContentsId }),
330+
captureReady: (tabId, webContentsId, requestId) =>
331+
ipcRenderer.invoke(IpcChannels.PREVIEW_CAPTURE_READY_CHANNEL, {
332+
tabId,
333+
webContentsId,
334+
requestId,
335+
}),
330336
navigate: (tabId, url) =>
331337
ipcRenderer.invoke(IpcChannels.PREVIEW_NAVIGATE_CHANNEL, { tabId, url }),
332338
goBack: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_GO_BACK_CHANNEL, { tabId }),

‎apps/desktop/src/preview/Manager.test.ts‎

Lines changed: 70 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -421,7 +421,7 @@ const decodeReply = Schema.decodeUnknownSync(
421421
),
422422
);
423423

424-
const setupAgentCapture = Effect.gen(function* () {
424+
const setupAgentCapture = Effect.fnUntraced(function* (autoReady = true) {
425425
const manager = yield* PreviewManager.PreviewManager;
426426
const host = yield* DesktopBrowserHost.DesktopBrowserHost;
427427
const serverTab = { threadId: "thread-capture", tabId: "server-capture" };
@@ -440,6 +440,18 @@ const setupAgentCapture = Effect.gen(function* () {
440440
}),
441441
Effect.forkScoped,
442442
);
443+
const rendererStates: Array<PreviewManager.PreviewTabState> = [];
444+
yield* manager.subscribeStateChanges((tabId, state) =>
445+
Effect.gen(function* () {
446+
if (tabId !== "tab_agent_capture") return;
447+
rendererStates.push(state);
448+
if (autoReady && state.webContentsId !== null) {
449+
yield* Effect.forEach(state.captureRequestIds ?? [], (id) =>
450+
manager.captureReady(tabId, state.webContentsId!, id),
451+
);
452+
}
453+
}),
454+
);
443455
const copies: Array<PromiseWithResolvers<TestCapturedPreviewImage>> = [];
444456
const captures: Array<PromiseWithResolvers<{ data: string }>> = [];
445457
const capturePage = vi.fn(() => {
@@ -473,6 +485,7 @@ const setupAgentCapture = Effect.gen(function* () {
473485
yield* Deferred.await(attached);
474486
return {
475487
hostWebContents,
488+
rendererStates,
476489
manager,
477490
wc,
478491
copies,
@@ -3041,7 +3054,7 @@ describe("PreviewManager", () => {
30413054
replies,
30423055
send,
30433056
setBackgroundThrottling,
3044-
} = yield* setupAgentCapture;
3057+
} = yield* setupAgentCapture();
30453058
const staleCopy = {
30463059
isEmpty: () => false,
30473060
toPNG: () => Buffer.from("previous frame"),
@@ -3091,6 +3104,60 @@ describe("PreviewManager", () => {
30913104
}).pipe(Effect.provide(managerLayer()), Effect.scoped),
30923105
);
30933106

3107+
effectIt.effect(
3108+
"waits for the renderer before copying an inactive guest and bounds preparation",
3109+
() =>
3110+
Effect.gen(function* () {
3111+
const {
3112+
manager,
3113+
wc,
3114+
copies,
3115+
captures,
3116+
replies,
3117+
send,
3118+
rendererStates,
3119+
setBackgroundThrottling,
3120+
} = yield* setupAgentCapture(false);
3121+
const requests = () => rendererStates.at(-1)?.captureRequestIds ?? [];
3122+
yield* send(1);
3123+
yield* TestClock.adjust(0);
3124+
expect(requests()).toHaveLength(1);
3125+
expect(captures).toHaveLength(0);
3126+
expect(copies).toHaveLength(0);
3127+
const expired = requests()[0]!;
3128+
yield* TestClock.adjust(9_000);
3129+
expect(yield* Queue.take(replies)).toMatchObject({ id: 1, error: expect.anything() });
3130+
expect(requests()).toEqual([]);
3131+
expect(setBackgroundThrottling).toHaveBeenLastCalledWith(true);
3132+
3133+
yield* send(2);
3134+
yield* TestClock.adjust(0);
3135+
const first = requests()[0]!;
3136+
yield* send(3);
3137+
yield* TestClock.adjust(0);
3138+
const second = requests().find((id) => id !== first)!;
3139+
expect(requests()).toHaveLength(2);
3140+
yield* manager.captureReady("tab_agent_capture", wc.id, expired);
3141+
yield* manager.captureReady("tab_agent_capture", wc.id + 1, first);
3142+
expect(captures).toHaveLength(0);
3143+
yield* manager.captureReady("tab_agent_capture", wc.id, first);
3144+
yield* TestClock.adjust(0);
3145+
expect(captures).toHaveLength(1);
3146+
expect(copies).toHaveLength(1);
3147+
captures[0]!.resolve({ data: "first" });
3148+
expect(yield* Queue.take(replies)).toMatchObject({ id: 2, result: { data: "first" } });
3149+
expect(requests()).toEqual([second]);
3150+
expect(setBackgroundThrottling).toHaveBeenLastCalledWith(false);
3151+
yield* manager.captureReady("tab_agent_capture", wc.id, second);
3152+
yield* TestClock.adjust(0);
3153+
captures[1]!.resolve({ data: "second" });
3154+
expect(yield* Queue.take(replies)).toMatchObject({ id: 3, result: { data: "second" } });
3155+
expect(requests()).toEqual([]);
3156+
expect(setBackgroundThrottling).toHaveBeenLastCalledWith(true);
3157+
expect(wc.setBackgroundThrottling).not.toHaveBeenCalled();
3158+
}).pipe(Effect.provide(managerLayer("linux")), Effect.scoped),
3159+
);
3160+
30943161
effectIt.effect("releases abandoned agent captures and lets the next capture finish", () =>
30953162
Effect.forEach(["failure", "timeout", "replacement"] as const, (outcome) =>
30963163
Effect.gen(function* () {
@@ -3104,7 +3171,7 @@ describe("PreviewManager", () => {
31043171
replies,
31053172
send,
31063173
setBackgroundThrottling,
3107-
} = yield* setupAgentCapture;
3174+
} = yield* setupAgentCapture();
31083175
const image = {
31093176
isEmpty: () => false,
31103177
toJPEG: () => Buffer.from("old copy"),

‎apps/desktop/src/preview/Manager.ts‎

Lines changed: 88 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ export interface PreviewTabState {
111111
audible: boolean;
112112
controller: "human" | "agent" | "none";
113113
favicon?: DesktopPreviewFavicon;
114+
captureRequestIds?: ReadonlyArray<string>;
114115
serverTab?: { readonly threadId: string; readonly tabId: string };
115116
updatedAt: string;
116117
}
@@ -562,6 +563,14 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
562563
const recordingFrameListenersRef = yield* Ref.make<ReadonlySet<RecordingFrameListener>>(
563564
new Set(),
564565
);
566+
const captureRequests = new Map<
567+
string,
568+
{
569+
readonly tabId: string;
570+
readonly webContentsId: number;
571+
readonly ready: Deferred.Deferred<void>;
572+
}
573+
>();
565574
const pickSessionsRef = yield* Ref.make<ReadonlyMap<string, PickSession>>(new Map());
566575
const controlSessionsRef = yield* SynchronizedRef.make<
567576
ReadonlyMap<number, BrowserControlSession>
@@ -722,7 +731,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
722731
) {
723732
yield* SynchronizedRef.modifyEffect(frameCaptureSessionsRef, (sessions) => {
724733
const current = sessions.get(tabId);
725-
if (!current || current.unthrottledWebContentsIds.has(wc.id)) {
734+
if (
735+
!current ||
736+
![...current.consumers].some((consumer) => typeof consumer === "string") ||
737+
current.unthrottledWebContentsIds.has(wc.id)
738+
) {
726739
return Effect.succeed([undefined, sessions] as const);
727740
}
728741
return setFrameCaptureWebContentsBackgroundThrottling(wc, false).pipe(
@@ -2349,21 +2362,67 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
23492362
);
23502363
});
23512364

2365+
const publishCaptureRequests = Effect.fnUntraced(function* (tabId: string) {
2366+
const next = yield* SynchronizedRef.modify(tabsRef, (tabs) => {
2367+
const current = tabs.get(tabId);
2368+
if (!current) return [undefined, tabs] as const;
2369+
const state = {
2370+
...current,
2371+
captureRequestIds: [...captureRequests].flatMap(([id, request]) =>
2372+
request.tabId === tabId ? [id] : [],
2373+
),
2374+
};
2375+
return [state, replaceMap(tabs, (copy) => copy.set(tabId, state))] as const;
2376+
});
2377+
if (next) yield* emitIfCurrent(tabId, next);
2378+
});
2379+
2380+
const captureReady = Effect.fn("PreviewManager.captureReady")(function* (
2381+
tabId: string,
2382+
webContentsId: number,
2383+
requestId: string,
2384+
) {
2385+
const request = captureRequests.get(requestId);
2386+
if (request?.tabId !== tabId || request.webContentsId !== webContentsId) return;
2387+
const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId);
2388+
if (tab?.webContentsId === webContentsId)
2389+
yield* Deferred.succeed(request.ready, undefined).pipe(Effect.asVoid);
2390+
});
2391+
23522392
const captureAgentScreenshot = Effect.fn("PreviewManager.captureAgentScreenshot")(function* (
23532393
tabId: string,
23542394
wc: Electron.WebContents,
23552395
debuggee: Electron.Debugger,
23562396
params: Record<string, unknown>,
23572397
) {
23582398
const consumer = Symbol("agent-screenshot");
2399+
const requestId = yield* crypto.randomUUIDv4().pipe(Effect.orDie);
2400+
const ready = yield* Deferred.make<void>();
23592401
return yield* Effect.acquireUseRelease(
23602402
startFrameCapture(tabId, consumer),
23612403
() =>
2362-
Effect.tryPromise((signal) =>
2363-
sendPageCommand(wc, debuggee, "Page.captureScreenshot", params, signal),
2364-
).pipe(
2365-
// ServerBrowserPage gives the whole snapshot ten seconds.
2366-
Effect.timeout("8 seconds"),
2404+
Effect.acquireUseRelease(
2405+
Effect.sync(() => {
2406+
captureRequests.set(requestId, { tabId, webContentsId: wc.id, ready });
2407+
}),
2408+
() =>
2409+
publishCaptureRequests(tabId).pipe(
2410+
Effect.andThen(Deferred.await(ready)),
2411+
// The renderer must commit the guest's in-viewport position before the first
2412+
// native copy: an offscreen copy can stall until the capture deadline.
2413+
Effect.andThen(
2414+
Effect.tryPromise((signal) =>
2415+
sendPageCommand(wc, debuggee, "Page.captureScreenshot", params, signal),
2416+
),
2417+
),
2418+
// ServerBrowserPage gives the whole snapshot ten seconds, including preparation.
2419+
Effect.timeout("8 seconds"),
2420+
),
2421+
() =>
2422+
Effect.gen(function* () {
2423+
captureRequests.delete(requestId);
2424+
yield* publishCaptureRequests(tabId);
2425+
}),
23672426
),
23682427
() => stopFrameCapture(tabId, consumer),
23692428
);
@@ -2709,7 +2768,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
27092768
if (current.consumers.has(consumer)) {
27102769
return [false, sessions] as const;
27112770
}
2712-
if (!current.unthrottledWebContentsIds.has(wc.id)) {
2771+
if (typeof consumer === "string" && !current.unthrottledWebContentsIds.has(wc.id)) {
27132772
yield* setFrameCaptureWebContentsBackgroundThrottling(wc, false);
27142773
}
27152774
let scope = current.scope;
@@ -2724,21 +2783,27 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
27242783
...current,
27252784
scope,
27262785
consumers: new Set([...current.consumers, consumer]),
2727-
unthrottledWebContentsIds: new Set([...current.unthrottledWebContentsIds, wc.id]),
2786+
unthrottledWebContentsIds: new Set([
2787+
...current.unthrottledWebContentsIds,
2788+
...(typeof consumer === "string" ? [wc.id] : []),
2789+
]),
27282790
});
27292791
}),
27302792
] as const;
27312793
}
27322794
if (sessions.size === 0) {
27332795
yield* setFrameCaptureBackgroundThrottling(false);
27342796
}
2735-
yield* setFrameCaptureWebContentsBackgroundThrottling(wc, false).pipe(
2736-
Effect.onError(() =>
2737-
sessions.size === 0
2738-
? setFrameCaptureBackgroundThrottling(true).pipe(Effect.ignore)
2739-
: Effect.void,
2740-
),
2741-
);
2797+
// Agent screenshots need presentation, not a page visibility override.
2798+
if (typeof consumer === "string") {
2799+
yield* setFrameCaptureWebContentsBackgroundThrottling(wc, false).pipe(
2800+
Effect.onError(() =>
2801+
sessions.size === 0
2802+
? setFrameCaptureBackgroundThrottling(true).pipe(Effect.ignore)
2803+
: Effect.void,
2804+
),
2805+
);
2806+
}
27422807
const scope =
27432808
consumer === "picture-in-picture" ? yield* Scope.fork(parentScope, "sequential") : null;
27442809
if (scope !== null) {
@@ -2750,7 +2815,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
27502815
copy.set(tabId, {
27512816
scope,
27522817
consumers: new Set([consumer]),
2753-
unthrottledWebContentsIds: new Set([wc.id]),
2818+
unthrottledWebContentsIds: new Set(typeof consumer === "string" ? [wc.id] : []),
27542819
lastPictureInPictureFingerprint: null,
27552820
});
27562821
}),
@@ -3392,6 +3457,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
33923457

33933458
return {
33943459
cancelPickElement,
3460+
captureReady,
33953461
captureScreenshot,
33963462
closeTab,
33973463
copyArtifactToClipboard,
@@ -3600,6 +3666,11 @@ export class PreviewManager extends Context.Service<
36003666
tabId: string,
36013667
webContentsId: number,
36023668
) => Effect.Effect<void, PreviewManagerError>;
3669+
readonly captureReady: (
3670+
tabId: string,
3671+
webContentsId: number,
3672+
requestId: string,
3673+
) => Effect.Effect<void>;
36033674
readonly prepareWebview: (webContents: Electron.WebContents) => Effect.Effect<void>;
36043675
readonly navigate: (tabId: string, url: string) => Effect.Effect<void, PreviewManagerError>;
36053676
readonly goBack: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
@@ -3733,6 +3804,7 @@ export const make = Effect.gen(function* PreviewManagerMake() {
37333804

37343805
return PreviewManager.of({
37353806
setMainWindow: operations.setMainWindow,
3807+
captureReady: operations.captureReady,
37363808
setForwardedShortcuts: operations.setForwardedShortcuts,
37373809
getBrowserSession: Effect.fn("PreviewManager.getBrowserSession")(
37383810
function* (scope, persistent, namespace) {

0 commit comments

Comments
 (0)