Skip to content

Commit 07dbb78

Browse files
t3dotggclaude
andcommitted
fix(preview): agent browser tabs show one page and work while hidden
A server tab that falls back to headless Chromium is now marked headless, so the desktop streams it instead of opening a second page later. When the server captures a desktop tab, the desktop asks the renderer to keep that tab drawing until the thread's turn ends. Hidden tabs no longer make agent snapshots time out. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 448b8a9 commit 07dbb78

14 files changed

Lines changed: 238 additions & 3 deletions

File tree

‎apps/desktop/src/ipc/channels.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ export const PREVIEW_RECORDING_SAVE_CHANNEL = "desktop:preview-recording-save";
113113
export const PREVIEW_RECORDING_FRAME_CHANNEL = "desktop:preview-recording-frame";
114114
export const PREVIEW_STATE_CHANGE_CHANNEL = "desktop:preview-state-change";
115115
export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event";
116+
export const PREVIEW_CAPTURE_REQUEST_CHANNEL = "desktop:preview-capture-request";
116117
export const PREVIEW_OPEN_LINK_CHANNEL = "desktop:preview-open-link";
117118

118119
export const MAC_PERMISSION_HELPER_CHANNEL = "desktop:mac-permission-helper";

‎apps/desktop/src/ipc/methods/preview.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,12 @@ import {
2828
import * as Effect from "effect/Effect";
2929
import * as Schema from "effect/Schema";
3030
import * as Option from "effect/Option";
31+
import * as Stream from "effect/Stream";
3132
import * as NodeURL from "node:url";
3233

3334
import * as ElectronWindow from "../../electron/ElectronWindow.ts";
3435
import * as BrowserImport from "../../preview/BrowserImport/BrowserImport.ts";
36+
import * as DesktopBrowserHost from "../../preview/DesktopBrowserHost.ts";
3537
import * as PreviewManager from "../../preview/Manager.ts";
3638
import { normalizeCaptureFailure } from "../../preview/captureErrors.ts";
3739
import * as DesktopClientSettings from "../../settings/DesktopClientSettings.ts";
@@ -59,6 +61,13 @@ export const installPreviewEventForwarding = Effect.fn(
5961
yield* manager.subscribeOpenLinks((event) =>
6062
electronWindow.sendAll(IpcChannels.PREVIEW_OPEN_LINK_CHANNEL, event),
6163
);
64+
const browserHost = yield* DesktopBrowserHost.DesktopBrowserHost;
65+
yield* browserHost.captures.pipe(
66+
Stream.runForEach((key) =>
67+
electronWindow.sendAll(IpcChannels.PREVIEW_CAPTURE_REQUEST_CHANNEL, key),
68+
),
69+
Effect.forkScoped,
70+
);
6271
});
6372

6473
const setForwardedShortcuts = DesktopIpc.makeIpcMethod({

‎apps/desktop/src/preload.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import type {
22
DesktopBridge,
33
DesktopPreviewOpenLinkEvent,
44
DesktopPreviewPointerEvent,
5+
DesktopPreviewServerTabKey,
56
DesktopPreviewRecordingInputEvent,
67
DesktopPreviewRecordingFrame,
78
DesktopPreviewTabState,
@@ -431,6 +432,15 @@ contextBridge.exposeInMainWorld("desktopBridge", {
431432
return () =>
432433
ipcRenderer.removeListener(IpcChannels.PREVIEW_POINTER_EVENT_CHANNEL, wrappedListener);
433434
},
435+
onCaptureRequest: (listener) => {
436+
const wrappedListener = (_event: Electron.IpcRendererEvent, key: unknown) => {
437+
if (typeof key !== "object" || key === null) return;
438+
listener(key as DesktopPreviewServerTabKey);
439+
};
440+
ipcRenderer.on(IpcChannels.PREVIEW_CAPTURE_REQUEST_CHANNEL, wrappedListener);
441+
return () =>
442+
ipcRenderer.removeListener(IpcChannels.PREVIEW_CAPTURE_REQUEST_CHANNEL, wrappedListener);
443+
},
434444
onOpenLink: (listener) => {
435445
const wrappedListener = (_event: Electron.IpcRendererEvent, linkEvent: unknown) => {
436446
if (typeof linkEvent !== "object" || linkEvent === null) return;

‎apps/desktop/src/preview/DesktopBrowserHost.test.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,32 @@ describe("DesktopBrowserHost", () => {
8989
}).pipe(Effect.scoped),
9090
);
9191

92+
it.effect("reports a tab when the server captures it, so a hidden tab can draw", () =>
93+
Effect.gen(function* () {
94+
const host = yield* DesktopBrowserHost.make;
95+
host.attach(key, makeDebuggee().tab);
96+
const captures = yield* host.captures.pipe(
97+
Stream.take(1),
98+
Stream.runCollect,
99+
Effect.forkScoped,
100+
);
101+
102+
yield* Effect.yieldNow;
103+
const command = (id: number, method: string) =>
104+
host.handleCommandLine(
105+
encodeJson({
106+
type: "cdp",
107+
...key,
108+
message: encodeJson({ id, method, sessionId: "t3-preview-page" }),
109+
}),
110+
);
111+
yield* command(1, "DOM.enable");
112+
yield* command(2, "Page.captureScreenshot");
113+
114+
expect(yield* Fiber.join(captures)).toEqual([key]);
115+
}).pipe(Effect.scoped),
116+
);
117+
92118
it.effect("saves a server tab's download under its CDP guid where the server asked", () =>
93119
Effect.gen(function* () {
94120
const host = yield* DesktopBrowserHost.make;

‎apps/desktop/src/preview/DesktopBrowserHost.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ const lineEncoder = new TextEncoder();
2626
const AGENT_INPUT_COMMAND =
2727
/"method":"(?:Input\.|Page\.navigate"|Runtime\.(?:evaluate|callFunctionOn)")/;
2828
const AGENT_DOWNLOAD_WINDOW_MS = 5_000;
29+
const CAPTURE_COMMAND = /"method":"Page\.captureScreenshot"/;
2930

3031
export interface DesktopBrowserTabKey {
3132
readonly threadId: string;
@@ -69,6 +70,8 @@ export class DesktopBrowserHost extends Context.Service<
6970
readonly x: number;
7071
readonly y: number;
7172
}>;
73+
/** Tabs the server is capturing. A hidden tab draws no frames, so the capture waits for one. */
74+
readonly captures: Stream.Stream<DesktopBrowserTabKey>;
7275
}
7376
>()("@t3tools/desktop/preview/DesktopBrowserHost") {}
7477

@@ -80,6 +83,7 @@ export const make = Effect.gen(function* () {
8083
readonly x: number;
8184
readonly y: number;
8285
}>(16);
86+
const captures = yield* PubSub.sliding<DesktopBrowserTabKey>(16);
8387
const runFork = Effect.runForkWith(yield* Effect.context<never>());
8488
const tabs = new Map<string, AttachedTab>();
8589
const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event));
@@ -177,6 +181,7 @@ export const make = Effect.gen(function* () {
177181
return;
178182
}
179183
if (AGENT_INPUT_COMMAND.test(command.value.message)) tab.agentInputAt = performance.now();
184+
if (CAPTURE_COMMAND.test(command.value.message)) runFork(PubSub.publish(captures, tab.key));
180185
relayFor(tab).receive(command.value.message);
181186
});
182187

@@ -198,6 +203,7 @@ export const make = Effect.gen(function* () {
198203

199204
return DesktopBrowserHost.of({
200205
pointers: Stream.fromPubSub(pointers),
206+
captures: Stream.fromPubSub(captures),
201207
// Subscribes before announcing, so no attach falls between the two.
202208
events: Stream.unwrap(
203209
Effect.gen(function* () {

‎apps/server/src/preview/Manager.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@ export class PreviewManager extends Context.Service<
4747
readonly reportStatus: (
4848
input: PreviewReportStatusInput & { readonly serverControlled?: boolean },
4949
) => Effect.Effect<void, PreviewError>;
50+
/** Records that a server tab runs headless. It stays headless until it closes. */
51+
readonly markHeadless: (
52+
input: PreviewCloseInput & { readonly tabId: string },
53+
) => Effect.Effect<void, PreviewError>;
5054
readonly requestReveal: (
5155
input: PreviewCloseInput & { readonly tabId: string; readonly force: boolean },
5256
) => Effect.Effect<void, PreviewError>;
@@ -310,6 +314,34 @@ export const make = Effect.gen(function* PreviewManagerMake() {
310314
);
311315
});
312316

317+
const markHeadless: PreviewManager["Service"]["markHeadless"] = Effect.fn(
318+
"PreviewManager.markHeadless",
319+
)(function* (input) {
320+
return yield* mutateExistingSession(
321+
input.threadId,
322+
input.tabId,
323+
Effect.fn("PreviewManager.markSessionHeadless")(function* (session) {
324+
if (session.snapshot.headless) return { next: session, emit: null, result: undefined };
325+
const snapshot: PreviewSessionSnapshot = {
326+
...session.snapshot,
327+
headless: true,
328+
updatedAt: yield* currentIsoTimestamp,
329+
};
330+
return {
331+
next: { ...session, snapshot },
332+
emit: {
333+
type: "resized",
334+
threadId: session.threadId,
335+
tabId: session.tabId,
336+
createdAt: snapshot.updatedAt,
337+
snapshot,
338+
},
339+
result: undefined,
340+
};
341+
}),
342+
);
343+
});
344+
313345
const resize: PreviewManager["Service"]["resize"] = Effect.fn("PreviewManager.resize")(
314346
function* (input) {
315347
return yield* mutateExistingSession(
@@ -472,6 +504,7 @@ export const make = Effect.gen(function* PreviewManagerMake() {
472504
requestReveal,
473505
navigate,
474506
reportStatus,
507+
markHeadless,
475508
resize,
476509
adjust,
477510
refresh,

‎apps/server/src/preview/ServerBrowser.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,27 @@ it.live.each(["desktop" as const, undefined])(
370370
).pipe(Effect.provide(layer)),
371371
);
372372

373+
it.live("a tab the desktop does not draw is marked headless, so the desktop streams it", () =>
374+
Effect.scoped(
375+
Effect.gen(function* () {
376+
const { broker, tabId } = yield* ready;
377+
const manager = yield* Manager.PreviewManager;
378+
desktopRendersNext = true;
379+
const desktopTab = yield* broker.invoke<PreviewAutomationStatus>({
380+
scope,
381+
operation: "open",
382+
input: { reuseExistingTab: false, show: false },
383+
});
384+
const { sessions } = yield* manager.list({ threadId: scope.thread.threadId });
385+
const headlessOf = (id: string | null) =>
386+
sessions.find((session) => session.tabId === id)?.headless;
387+
388+
expect(headlessOf(tabId)).toBe(true);
389+
expect(headlessOf(desktopTab.tabId)).toBeUndefined();
390+
}),
391+
).pipe(Effect.provide(layer)),
392+
);
393+
373394
it.live.each([
374395
{ method: "goto" as const, message: { type: "navigate", url: "http://10.255.255.1/" } },
375396
{ method: "goBack" as const, message: { type: "history", delta: -1 } },

‎apps/server/src/preview/ServerBrowser.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -683,7 +683,7 @@ const make = Effect.gen(function* () {
683683
};
684684

685685
const desktopRenders = (snapshot: PreviewSessionSnapshot) =>
686-
desktopChannel.available
686+
desktopChannel.available && !snapshot.headless
687687
? Effect.runPromise(
688688
desktopChannel.awaitAttached(
689689
{ threadId: snapshot.threadId, tabId: snapshot.tabId },
@@ -721,6 +721,14 @@ const make = Effect.gen(function* () {
721721
adopted === undefined && (await desktopRenders(snapshot))
722722
? await connectDesktop(snapshot)
723723
: null;
724+
// Otherwise a desktop that attaches late shows its own page, and the agent drives another.
725+
if (!desktop && desktopChannel.available) {
726+
await Effect.runPromise(
727+
manager
728+
.markHeadless({ threadId: ThreadId.make(snapshot.threadId), tabId: snapshot.tabId })
729+
.pipe(Effect.ignore),
730+
);
731+
}
724732
const isolatedContext =
725733
adopted === undefined &&
726734
desktop === null &&

‎apps/web/src/browser/ElectronBrowserHost.tsx‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { parseScopedThreadKey } from "@t3tools/client-runtime/environment";
22
import {
33
AuthPreviewOperateScope,
44
type DesktopPreviewOpenLinkEvent,
5+
type DesktopPreviewServerTabKey,
56
FILL_PREVIEW_VIEWPORT,
67
} from "@t3tools/contracts";
78
import { useAtomValue } from "@effect/atom-react";
@@ -20,6 +21,7 @@ import { useAtomCommand } from "~/state/use-atom-command";
2021
import { readPreviewAnnotationTheme } from "./annotationTheme";
2122
import { useBrowserDefaults } from "./browserDefaults";
2223
import { useBrowserPointerStore } from "./browserPointerStore";
24+
import { holdFramesForCapture } from "./captureFrames";
2325
import { HostedBrowserWebview } from "./HostedBrowserWebview";
2426
import { openUrlInPreview } from "./openFileInPreview";
2527
import { rendersServerTabNatively } from "./previewRuntime";
@@ -113,6 +115,21 @@ export function ElectronBrowserHost() {
113115
return preview.onOpenLink(onOpenLink);
114116
}, []);
115117

118+
const onCaptureRequest = useEffectEvent(({ threadId, tabId }: DesktopPreviewServerTabKey) => {
119+
const session = sessions.find(
120+
(candidate) =>
121+
candidate.threadRef.environmentId === primaryEnvironmentId &&
122+
candidate.threadRef.threadId === threadId &&
123+
candidate.snapshot.tabId === tabId,
124+
);
125+
if (session) holdFramesForCapture(session.runtimeTabId, session.threadRef);
126+
});
127+
useEffect(() => {
128+
const preview = window.desktopBridge?.preview;
129+
if (!preview) return;
130+
return preview.onCaptureRequest(onCaptureRequest);
131+
}, []);
132+
116133
if (!isElectron) return null;
117134
return (
118135
<div className="contents" data-electron-browser-host>
Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import { EnvironmentId, ThreadId } from "@t3tools/contracts";
2+
import { Atom } from "effect/reactivity";
3+
import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";
4+
5+
import { appAtomRegistry } from "~/rpc/atomRegistry";
6+
import { environmentThreadShells } from "~/state/threads";
7+
8+
import { useBrowserSurfaceStore } from "./browserSurfaceStore";
9+
import { holdFramesForCapture } from "./captureFrames";
10+
11+
vi.mock("~/state/threads", async () => {
12+
const { Atom } = await import("effect/reactivity");
13+
const shell = Atom.make<{ runtime: { status: string } } | null>(null).pipe(Atom.keepAlive);
14+
return { environmentThreadShells: { threadShellAtom: () => shell } };
15+
});
16+
17+
type TestShell = { runtime: { status: string } } | null;
18+
const threadRef = {
19+
environmentId: EnvironmentId.make("environment-1"),
20+
threadId: ThreadId.make("thread-1"),
21+
};
22+
const shellAtom = environmentThreadShells.threadShellAtom(
23+
threadRef,
24+
) as unknown as Atom.Writable<TestShell>;
25+
const activity = () => useBrowserSurfaceStore.getState().activityByTabId["tab-1"];
26+
27+
describe("holdFramesForCapture", () => {
28+
beforeEach(() => {
29+
useBrowserSurfaceStore.setState({ activityByTabId: {} });
30+
});
31+
afterEach(() => {
32+
appAtomRegistry.set(shellAtom, null);
33+
vi.useRealTimers();
34+
});
35+
36+
it("keeps a captured tab drawing until its thread's turn ends", () => {
37+
appAtomRegistry.set(shellAtom, { runtime: { status: "running" } });
38+
39+
holdFramesForCapture("tab-1", threadRef);
40+
holdFramesForCapture("tab-1", threadRef);
41+
expect(activity()).toBe(1);
42+
43+
appAtomRegistry.set(shellAtom, { runtime: { status: "idle" } });
44+
expect(activity()).toBeUndefined();
45+
});
46+
47+
it("draws for one capture when the thread is not running a turn", () => {
48+
vi.useFakeTimers();
49+
50+
holdFramesForCapture("tab-1", threadRef);
51+
vi.advanceTimersByTime(9_999);
52+
expect(activity()).toBe(1);
53+
54+
vi.advanceTimersByTime(1);
55+
expect(activity()).toBeUndefined();
56+
});
57+
});

0 commit comments

Comments
 (0)