Skip to content

Commit 00852f7

Browse files
PR Batch Testerclaude
andcommitted
fix(desktop): agents see visible:true while the desktop shows their tab
The server counted a tab as visible only when a client streamed its frames. The desktop draws server tabs in its own webview and never streams them, so preview_status and tabs[].visible said false while the tab was on screen, and tabs picked without a tabId fell back to recency. The desktop renderer now reports when a server-driven webview is on screen (Browser panel or picture-in-picture). Electron's main process sends that over the existing desktop browser channel as a "shown" event, re-sends it when the backend reconnects or the tab attaches again, and clears it when the tab's webview is destroyed. The server counts a shown, attached desktop tab as visible. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent f054d1b commit 00852f7

12 files changed

Lines changed: 161 additions & 7 deletions

File tree

‎apps/desktop/src/ipc/channels.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ export const PREVIEW_SET_ZOOM_FACTOR_CHANNEL = "desktop:preview-set-zoom-factor"
9090
export const PREVIEW_HARD_RELOAD_CHANNEL = "desktop:preview-hard-reload";
9191
export const PREVIEW_SET_COLOR_SCHEME_CHANNEL = "desktop:preview-set-color-scheme";
9292
export const PREVIEW_SET_AUDIO_MUTED_CHANNEL = "desktop:preview-set-audio-muted";
93+
export const PREVIEW_SET_SHOWN_CHANNEL = "desktop:preview-set-shown";
9394
export const PREVIEW_OPEN_DEVTOOLS_CHANNEL = "desktop:preview-open-devtools";
9495
export const PREVIEW_CLEAR_COOKIES_CHANNEL = "desktop:preview-clear-cookies";
9596
export const PREVIEW_CLEAR_CACHE_CHANNEL = "desktop:preview-clear-cache";

‎apps/desktop/src/ipc/methods/preview.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
DesktopPreviewRegisterWebviewInputSchema,
1010
DesktopPreviewScreenshotArtifactSchema,
1111
DesktopPreviewSetAudioMutedInputSchema,
12+
DesktopPreviewSetShownInputSchema,
1213
DesktopPreviewSetColorSchemeInputSchema,
1314
DesktopPreviewSetZoomFactorInputSchema,
1415
BrowserImportResult,
@@ -196,6 +197,15 @@ const setAudioMuted = DesktopIpc.makeIpcMethod({
196197
yield* manager.setAudioMuted(tabId, audioMuted);
197198
}),
198199
});
200+
const setShown = DesktopIpc.makeIpcMethod({
201+
channel: IpcChannels.PREVIEW_SET_SHOWN_CHANNEL,
202+
payload: DesktopPreviewSetShownInputSchema,
203+
result: Schema.Void,
204+
handler: Effect.fn("desktop.ipc.preview.setShown")(function* ({ tabId, shown }) {
205+
const manager = yield* PreviewManager.PreviewManager;
206+
yield* manager.setShown(tabId, shown);
207+
}),
208+
});
199209
const openDevTools = tabMethod(
200210
IpcChannels.PREVIEW_OPEN_DEVTOOLS_CHANNEL,
201211
"desktop.ipc.preview.openDevTools",
@@ -452,6 +462,7 @@ export const methods = [
452462
setColorScheme,
453463
setZoomFactor,
454464
setAudioMuted,
465+
setShown,
455466
openDevTools,
456467
clearCookies,
457468
clearCache,

‎apps/desktop/src/preload.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,8 @@ contextBridge.exposeInMainWorld("desktopBridge", {
346346
ipcRenderer.invoke(IpcChannels.PREVIEW_SET_COLOR_SCHEME_CHANNEL, { tabId, colorScheme }),
347347
setAudioMuted: (tabId, audioMuted) =>
348348
ipcRenderer.invoke(IpcChannels.PREVIEW_SET_AUDIO_MUTED_CHANNEL, { tabId, audioMuted }),
349+
setShown: (tabId, shown) =>
350+
ipcRenderer.invoke(IpcChannels.PREVIEW_SET_SHOWN_CHANNEL, { tabId, shown }),
349351
openDevTools: (tabId) =>
350352
ipcRenderer.invoke(IpcChannels.PREVIEW_OPEN_DEVTOOLS_CHANNEL, { tabId }),
351353
listBrowserImportSources: () => ipcRenderer.invoke(IpcChannels.PREVIEW_IMPORT_SOURCES_CHANNEL),

‎apps/desktop/src/preview/DesktopBrowserHost.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,27 @@ describe("DesktopBrowserHost", () => {
6262
}),
6363
);
6464

65+
it.effect("re-sends shown to a backend that starts later and when a tab attaches again", () =>
66+
Effect.gen(function* () {
67+
const host = yield* DesktopBrowserHost.make;
68+
host.attach(key, makeDebuggee().tab);
69+
host.setShown(key, true);
70+
const reader = yield* takeEvents(host, 5).pipe(Effect.forkScoped);
71+
72+
yield* Effect.yieldNow;
73+
host.detach(key);
74+
host.attach(key, makeDebuggee().tab);
75+
76+
expect(yield* Fiber.join(reader)).toEqual([
77+
{ type: "attached", ...key },
78+
{ type: "shown", ...key, shown: true },
79+
{ type: "detached", ...key },
80+
{ type: "attached", ...key },
81+
{ type: "shown", ...key, shown: true },
82+
]);
83+
}),
84+
);
85+
6586
it.effect("drops replies from a relay the server released", () =>
6687
Effect.gen(function* () {
6788
const host = yield* DesktopBrowserHost.make;

‎apps/desktop/src/preview/DesktopBrowserHost.ts‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ export class DesktopBrowserHost extends Context.Service<
6161
readonly handleCommandLine: (line: string) => Effect.Effect<void>;
6262
readonly attach: (key: DesktopBrowserTabKey, debuggee: DesktopBrowserTabDebugger) => void;
6363
readonly detach: (key: DesktopBrowserTabKey) => void;
64+
readonly setShown: (key: DesktopBrowserTabKey, shown: boolean) => void;
6465
readonly humanStartedDownload: (source: Electron.WebContents) => boolean;
6566
readonly placeDownload: (source: Electron.WebContents, item: Electron.DownloadItem) => boolean;
6667
readonly pointers: Stream.Stream<{
@@ -82,6 +83,9 @@ export const make = Effect.gen(function* () {
8283
}>(16);
8384
const runFork = Effect.runForkWith(yield* Effect.context<never>());
8485
const tabs = new Map<string, AttachedTab>();
86+
// Kept apart from attachment: the renderer can show a webview before its debugger attaches,
87+
// and the server forgets `shown` on detach, for example while DevTools holds the debugger.
88+
const shownTabs = new Set<string>();
8589
const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event));
8690

8791
const relayFor = (tab: AttachedTab) => {
@@ -159,6 +163,15 @@ export const make = Effect.gen(function* () {
159163
tabs.set(id, tab);
160164
debuggee.debugger.on("message", tab.onMessage);
161165
emit({ type: "attached", ...key });
166+
if (shownTabs.has(id)) emit({ type: "shown", ...key, shown: true });
167+
};
168+
169+
const setShown = (key: DesktopBrowserTabKey, shown: boolean) => {
170+
const id = keyOf(key);
171+
if (shownTabs.has(id) === shown) return;
172+
if (shown) shownTabs.add(id);
173+
else shownTabs.delete(id);
174+
if (tabs.has(id)) emit({ type: "shown", ...key, shown });
162175
};
163176

164177
const handleCommandLine = (line: string) =>
@@ -190,7 +203,13 @@ export const make = Effect.gen(function* () {
190203
[...tabs.values()],
191204
(tab) => {
192205
tab.relay = null;
193-
return PubSub.publish(outbox, { type: "attached", ...tab.key });
206+
return PubSub.publish(outbox, { type: "attached", ...tab.key }).pipe(
207+
Effect.andThen(
208+
shownTabs.has(keyOf(tab.key))
209+
? PubSub.publish(outbox, { type: "shown", ...tab.key, shown: true })
210+
: Effect.void,
211+
),
212+
);
194213
},
195214
{ discard: true },
196215
),
@@ -209,6 +228,7 @@ export const make = Effect.gen(function* () {
209228
handleCommandLine,
210229
attach,
211230
detach,
231+
setShown,
212232
placeDownload,
213233
humanStartedDownload,
214234
});

‎apps/desktop/src/preview/Manager.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1767,6 +1767,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
17671767
});
17681768
if (Option.isNone(tab)) return;
17691769
const closedTab = tab.value;
1770+
if (closedTab.serverTab) browserHost.setShown(closedTab.serverTab, false);
17701771
if (closedTab.webContentsId != null) {
17711772
yield* Effect.all(
17721773
[
@@ -1984,6 +1985,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
19841985
yield* rendererHistory.register(wc, { surface: "preview" });
19851986
const webContentsId = wc.id;
19861987
wc.once("destroyed", () => {
1988+
runFork(hideDestroyedGuest(webContentsId));
19871989
runFork(detachControlSession(webContentsId));
19881990
});
19891991
yield* ensureControlSession(wc).pipe(
@@ -2449,6 +2451,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
24492451
yield* applyColorScheme(tabId, wc, colorScheme);
24502452
});
24512453

2454+
const setShown = Effect.fn("PreviewManager.setShown")(function* (tabId: string, shown: boolean) {
2455+
const serverTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId)?.serverTab;
2456+
if (serverTab) browserHost.setShown(serverTab, shown);
2457+
});
2458+
2459+
// A window reload or guest crash destroys the guest without the renderer reporting it hidden.
2460+
// The renderer reports again once a new guest registers.
2461+
const hideDestroyedGuest = Effect.fnUntraced(function* (webContentsId: number) {
2462+
for (const tab of (yield* SynchronizedRef.get(tabsRef)).values()) {
2463+
if (tab.webContentsId === webContentsId && tab.serverTab) {
2464+
browserHost.setShown(tab.serverTab, false);
2465+
}
2466+
}
2467+
});
2468+
24522469
const setAudioMuted = Effect.fn("PreviewManager.setAudioMuted")(function* (
24532470
tabId: string,
24542471
audioMuted: boolean,
@@ -3416,6 +3433,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
34163433
setAnnotationTheme,
34173434
setAnnotationSendEnabled,
34183435
setAudioMuted,
3436+
setShown,
34193437
setColorScheme,
34203438
setMainWindow,
34213439
setForwardedShortcuts: (shortcuts: ReadonlyArray<PreviewForwardedShortcut>) =>
@@ -3623,6 +3641,7 @@ export class PreviewManager extends Context.Service<
36233641
tabId: string,
36243642
audioMuted: boolean,
36253643
) => Effect.Effect<void, PreviewManagerError>;
3644+
readonly setShown: (tabId: string, shown: boolean) => Effect.Effect<void>;
36263645
readonly openDevTools: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
36273646
readonly clearCookies: (
36283647
partitions?: ReadonlyArray<string>,
@@ -3766,6 +3785,7 @@ export const make = Effect.gen(function* PreviewManagerMake() {
37663785
setColorScheme: operations.setColorScheme,
37673786
setZoomFactor: operations.setZoomFactor,
37683787
setAudioMuted: operations.setAudioMuted,
3788+
setShown: operations.setShown,
37693789
openDevTools: operations.openDevTools,
37703790
clearCookies: Effect.fn("PreviewManager.clearCookies")(function* (partitions) {
37713791
yield* browserSession

‎apps/server/src/preview/DesktopBrowserChannel.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ export class DesktopBrowserChannel extends Context.Service<
5252
readonly detached: Stream.Stream<DesktopTabKey>;
5353
readonly attached: Stream.Stream<DesktopTabKey>;
5454
readonly isAttached: (key: DesktopTabKey) => Effect.Effect<boolean>;
55+
/** Whether a desktop window draws the tab on screen, which no streamed viewer counts. */
56+
readonly isShown: (key: DesktopTabKey) => boolean;
5557
readonly endpoint: (key: DesktopTabKey) => Effect.Effect<string, never, Scope.Scope>;
5658
readonly pointer: (
5759
key: DesktopTabKey,
@@ -66,6 +68,8 @@ const make = Effect.gen(function* () {
6668
const controlFd = config.desktopBrowserControlFd;
6769
const changes = yield* PubSub.unbounded<{ key: DesktopTabKey; attached: boolean }>();
6870
const attachedTabs = new Set<string>();
71+
// Only attached tabs count: the desktop re-sends `shown` when a tab attaches again.
72+
const shownTabs = new Set<string>();
6973
const inbound = new Map<string, Queue.Queue<string>>();
7074
const writeLock = yield* Semaphore.make(1);
7175

@@ -76,6 +80,7 @@ const make = Effect.gen(function* () {
7680
detached: Stream.empty,
7781
attached: Stream.empty,
7882
isAttached: () => Effect.succeed(false),
83+
isShown: () => false,
7984
endpoint: () => Effect.die("No desktop app is attached to this server."),
8085
pointer: () => Effect.void,
8186
});
@@ -119,14 +124,20 @@ const make = Effect.gen(function* () {
119124
return PubSub.publish(changes, { key, attached: true });
120125
case "detached": {
121126
attachedTabs.delete(id);
127+
shownTabs.delete(id);
122128
const queue = inbound.get(id);
123129
return (queue ? Queue.shutdown(queue) : Effect.void).pipe(
124130
Effect.andThen(PubSub.publish(changes, { key, attached: false })),
125131
);
126132
}
133+
case "shown":
134+
if (event.shown) shownTabs.add(id);
135+
else shownTabs.delete(id);
136+
return Effect.void;
127137
}
128138
}),
129139
Effect.catchCause((cause) => Effect.logWarning("desktop browser channel stopped", { cause })),
140+
Effect.ensuring(Effect.sync(() => shownTabs.clear())),
130141
Effect.forkScoped,
131142
);
132143

@@ -214,6 +225,7 @@ const make = Effect.gen(function* () {
214225
Stream.map((change) => change.key),
215226
),
216227
isAttached: (key) => Effect.sync(() => attachedTabs.has(keyOf(key))),
228+
isShown: (key) => shownTabs.has(keyOf(key)),
217229
endpoint,
218230
pointer: (key, pointer) => command({ type: "pointer", ...key, ...pointer }),
219231
});

‎apps/server/src/preview/ServerBrowser.test.ts‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,7 @@ const desktopRenders = (tabId: string) => {
230230
}
231231
return desktopTabs.has(tabId);
232232
};
233+
const shownDesktopTabs = new Set<string>();
233234
const releasedDesktopTabs: Array<string> = [];
234235
const desktopConnections: Array<{ endpoint: string; context: ReturnType<typeof makeContext> }> = [];
235236
const testThread = {
@@ -286,6 +287,7 @@ const dependencies = Layer.mergeAll(
286287
),
287288
),
288289
isAttached: (key) => Effect.sync(() => desktopRenders(key.tabId)),
290+
isShown: (key) => shownDesktopTabs.has(key.tabId),
289291
endpoint: (key) =>
290292
Effect.acquireRelease(Effect.succeed(`ws://desktop/${key.tabId}`), () =>
291293
Effect.sync(() => releasedDesktopTabs.push(key.tabId)),
@@ -329,6 +331,7 @@ beforeEach(() => {
329331
focusedFrame = null;
330332
desktopTabs.clear();
331333
desktopRendersNext = false;
334+
shownDesktopTabs.clear();
332335
releasedDesktopTabs.length = 0;
333336
desktopConnections.length = 0;
334337
});
@@ -991,6 +994,47 @@ it.live("only the current controller can answer a file picker", () =>
991994
).pipe(Effect.provide(layer)),
992995
);
993996

997+
it.live("the tab the desktop shows is visible and picked for agents without a tab", () =>
998+
Effect.scoped(
999+
Effect.gen(function* () {
1000+
const broker = yield* Broker.PreviewAutomationBroker;
1001+
const manager = yield* Manager.PreviewManager;
1002+
yield* ServerBrowser.ServerBrowser;
1003+
yield* Effect.yieldNow;
1004+
const openDesktopTab = Effect.fn("openDesktopTab")(function* (url: string) {
1005+
desktopRendersNext = true;
1006+
const opened = yield* manager.open({
1007+
threadId: scope.thread.threadId,
1008+
url,
1009+
runtime: "server",
1010+
});
1011+
const tabId = PreviewTabId.make(opened.tabId);
1012+
yield* broker.invoke({ scope, tabId, operation: "status", input: {} });
1013+
return tabId;
1014+
});
1015+
const recentTab = yield* openDesktopTab("http://localhost:5173/recent");
1016+
const shownTab = yield* openDesktopTab("http://localhost:5173/shown");
1017+
// Opened first and used last, so recency or insertion order alone picks it.
1018+
yield* broker.invoke({ scope, tabId: recentTab, operation: "status", input: {} });
1019+
1020+
shownDesktopTabs.add(shownTab);
1021+
const status = yield* broker.invoke<PreviewAutomationStatus>({
1022+
scope: asSession("agent-b"),
1023+
operation: "status",
1024+
input: {},
1025+
});
1026+
expect(status).toMatchObject({
1027+
tabId: shownTab,
1028+
visible: true,
1029+
tabs: [
1030+
expect.objectContaining({ tabId: recentTab, visible: false }),
1031+
expect.objectContaining({ tabId: shownTab, visible: true }),
1032+
],
1033+
});
1034+
}),
1035+
).pipe(Effect.provide(layer)),
1036+
);
1037+
9941038
it.live("agents read a human's tab with no arguments and act on it only while nobody drives", () =>
9951039
Effect.scoped(
9961040
Effect.gen(function* () {

‎apps/server/src/preview/ServerBrowser.ts‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,9 @@ const make = Effect.gen(function* () {
468468

469469
const tabs = new Map<string, ServerTab>();
470470
const pendingTabs = new Map<string, Promise<ServerTab>>();
471+
// The desktop draws its tabs in its own webview, so they never get a streamed viewer.
472+
const isVisible = (tab: ServerTab) =>
473+
tab.viewers.size > 0 || (tab.desktop !== null && desktopChannel.isShown(tab));
471474

472475
const closedPendingTabs = new Set<string>();
473476
const adoptedPages = new Map<string, { readonly page: Page; readonly openerTabId: string }>();
@@ -589,7 +592,7 @@ const make = Effect.gen(function* () {
589592
liveTabs: [...tabs.values()].map((tab) => ({
590593
threadId: tab.threadId,
591594
tabId: tab.tabId,
592-
visible: tab.viewers.size > 0,
595+
visible: isVisible(tab),
593596
})),
594597
}),
595598
),
@@ -1296,8 +1299,7 @@ const make = Effect.gen(function* () {
12961299
.sort((left, right) => right.createdAt - left.createdAt)[0] ??
12971300
threadTabs.sort(
12981301
(left, right) =>
1299-
Number(right.viewers.size > 0) - Number(left.viewers.size > 0) ||
1300-
right.usedAt - left.usedAt,
1302+
Number(isVisible(right)) - Number(isVisible(left)) || right.usedAt - left.usedAt,
13011303
)[0]
13021304
);
13031305
};
@@ -1371,7 +1373,7 @@ const make = Effect.gen(function* () {
13711373
const viewport = tab.page.viewportSize();
13721374
const status = {
13731375
available: true,
1374-
visible: tab.viewers.size > 0,
1376+
visible: isVisible(tab),
13751377
tabId: tab.tabId,
13761378
url: url === "about:blank" ? null : url,
13771379
title: null,
@@ -1395,7 +1397,7 @@ const make = Effect.gen(function* () {
13951397
...(candidate.openerTabId === undefined ? {} : { openerTabId: candidate.openerTabId }),
13961398
owner: tabOwner(candidate),
13971399
ownedByCaller: candidate.control.agentId === agentSessionId,
1398-
visible: candidate.viewers.size > 0,
1400+
visible: isVisible(candidate),
13991401
...(candidate.profileId === undefined ? {} : { profileId: candidate.profileId }),
14001402
})),
14011403
...profileStatus(),

0 commit comments

Comments
 (0)