Skip to content

Commit 8f9bef1

Browse files
PR Batch Testerclaude
andcommitted
fix(desktop): agents see visible:true while the desktop shows their tab
The server counted a tab as visible only when a client streamed its frames. The desktop draws server tabs in its own webview and never streams them, so preview_open, preview_status and tabs[].visible said false while the tab was on screen, and tabs without an explicit tabId fell back to recency. The desktop renderer now reports when a server-driven webview is on screen (Browser panel or picture-in-picture). Electron's main process passes that over the existing desktop browser channel as a "shown" event, and the server counts a shown desktop tab as visible. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent ed3f965 commit 8f9bef1

12 files changed

Lines changed: 133 additions & 5 deletions

File tree

‎apps/desktop/src/ipc/channels.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ export const PREVIEW_SET_ZOOM_FACTOR_CHANNEL = "desktop:preview-set-zoom-factor"
9090
export const PREVIEW_HARD_RELOAD_CHANNEL = "desktop:preview-hard-reload";
9191
export const PREVIEW_SET_COLOR_SCHEME_CHANNEL = "desktop:preview-set-color-scheme";
9292
export const PREVIEW_SET_AUDIO_MUTED_CHANNEL = "desktop:preview-set-audio-muted";
93+
export const PREVIEW_SET_SHOWN_CHANNEL = "desktop:preview-set-shown";
9394
export const PREVIEW_OPEN_DEVTOOLS_CHANNEL = "desktop:preview-open-devtools";
9495
export const PREVIEW_CLEAR_COOKIES_CHANNEL = "desktop:preview-clear-cookies";
9596
export const PREVIEW_CLEAR_CACHE_CHANNEL = "desktop:preview-clear-cache";

‎apps/desktop/src/ipc/methods/preview.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
DesktopPreviewRegisterWebviewInputSchema,
1010
DesktopPreviewScreenshotArtifactSchema,
1111
DesktopPreviewSetAudioMutedInputSchema,
12+
DesktopPreviewSetShownInputSchema,
1213
DesktopPreviewSetColorSchemeInputSchema,
1314
DesktopPreviewSetZoomFactorInputSchema,
1415
BrowserImportResult,
@@ -196,6 +197,15 @@ const setAudioMuted = DesktopIpc.makeIpcMethod({
196197
yield* manager.setAudioMuted(tabId, audioMuted);
197198
}),
198199
});
200+
const setShown = DesktopIpc.makeIpcMethod({
201+
channel: IpcChannels.PREVIEW_SET_SHOWN_CHANNEL,
202+
payload: DesktopPreviewSetShownInputSchema,
203+
result: Schema.Void,
204+
handler: Effect.fn("desktop.ipc.preview.setShown")(function* ({ tabId, shown }) {
205+
const manager = yield* PreviewManager.PreviewManager;
206+
yield* manager.setShown(tabId, shown);
207+
}),
208+
});
199209
const openDevTools = tabMethod(
200210
IpcChannels.PREVIEW_OPEN_DEVTOOLS_CHANNEL,
201211
"desktop.ipc.preview.openDevTools",
@@ -452,6 +462,7 @@ export const methods = [
452462
setColorScheme,
453463
setZoomFactor,
454464
setAudioMuted,
465+
setShown,
455466
openDevTools,
456467
clearCookies,
457468
clearCache,

‎apps/desktop/src/preload.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,8 @@ contextBridge.exposeInMainWorld("desktopBridge", {
346346
ipcRenderer.invoke(IpcChannels.PREVIEW_SET_COLOR_SCHEME_CHANNEL, { tabId, colorScheme }),
347347
setAudioMuted: (tabId, audioMuted) =>
348348
ipcRenderer.invoke(IpcChannels.PREVIEW_SET_AUDIO_MUTED_CHANNEL, { tabId, audioMuted }),
349+
setShown: (tabId, shown) =>
350+
ipcRenderer.invoke(IpcChannels.PREVIEW_SET_SHOWN_CHANNEL, { tabId, shown }),
349351
openDevTools: (tabId) =>
350352
ipcRenderer.invoke(IpcChannels.PREVIEW_OPEN_DEVTOOLS_CHANNEL, { tabId }),
351353
listBrowserImportSources: () => ipcRenderer.invoke(IpcChannels.PREVIEW_IMPORT_SOURCES_CHANNEL),

‎apps/desktop/src/preview/DesktopBrowserHost.test.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,15 @@ describe("DesktopBrowserHost", () => {
6262
}),
6363
);
6464

65+
it.effect("tells a backend that starts later which tabs are on screen", () =>
66+
Effect.gen(function* () {
67+
const host = yield* DesktopBrowserHost.make;
68+
host.setShown(key, true);
69+
70+
expect(yield* takeEvents(host, 1)).toEqual([{ type: "shown", ...key, shown: true }]);
71+
}),
72+
);
73+
6574
it.effect("drops replies from a relay the server released", () =>
6675
Effect.gen(function* () {
6776
const host = yield* DesktopBrowserHost.make;

‎apps/desktop/src/preview/DesktopBrowserHost.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ export class DesktopBrowserHost extends Context.Service<
6161
readonly handleCommandLine: (line: string) => Effect.Effect<void>;
6262
readonly attach: (key: DesktopBrowserTabKey, debuggee: DesktopBrowserTabDebugger) => void;
6363
readonly detach: (key: DesktopBrowserTabKey) => void;
64+
readonly setShown: (key: DesktopBrowserTabKey, shown: boolean) => void;
6465
readonly humanStartedDownload: (source: Electron.WebContents) => boolean;
6566
readonly placeDownload: (source: Electron.WebContents, item: Electron.DownloadItem) => boolean;
6667
readonly pointers: Stream.Stream<{
@@ -82,6 +83,8 @@ export const make = Effect.gen(function* () {
8283
}>(16);
8384
const runFork = Effect.runForkWith(yield* Effect.context<never>());
8485
const tabs = new Map<string, AttachedTab>();
86+
// Kept apart from attachment: the renderer can show a webview before its debugger attaches.
87+
const shownTabs = new Map<string, DesktopBrowserTabKey>();
8588
const emit = (event: DesktopBrowserEventType) => runFork(PubSub.publish(outbox, event));
8689

8790
const relayFor = (tab: AttachedTab) => {
@@ -161,6 +164,14 @@ export const make = Effect.gen(function* () {
161164
emit({ type: "attached", ...key });
162165
};
163166

167+
const setShown = (key: DesktopBrowserTabKey, shown: boolean) => {
168+
const id = keyOf(key);
169+
if (shownTabs.has(id) === shown) return;
170+
if (shown) shownTabs.set(id, key);
171+
else shownTabs.delete(id);
172+
emit({ type: "shown", ...key, shown });
173+
};
174+
164175
const handleCommandLine = (line: string) =>
165176
Effect.sync(() => {
166177
const command = decodeCommand(line);
@@ -193,6 +204,14 @@ export const make = Effect.gen(function* () {
193204
return PubSub.publish(outbox, { type: "attached", ...tab.key });
194205
},
195206
{ discard: true },
207+
).pipe(
208+
Effect.andThen(
209+
Effect.forEach(
210+
[...shownTabs.values()],
211+
(key) => PubSub.publish(outbox, { type: "shown", ...key, shown: true }),
212+
{ discard: true },
213+
),
214+
),
196215
),
197216
);
198217

@@ -209,6 +228,7 @@ export const make = Effect.gen(function* () {
209228
handleCommandLine,
210229
attach,
211230
detach,
231+
setShown,
212232
placeDownload,
213233
humanStartedDownload,
214234
});

‎apps/desktop/src/preview/Manager.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1767,6 +1767,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
17671767
});
17681768
if (Option.isNone(tab)) return;
17691769
const closedTab = tab.value;
1770+
if (closedTab.serverTab) browserHost.setShown(closedTab.serverTab, false);
17701771
if (closedTab.webContentsId != null) {
17711772
yield* Effect.all(
17721773
[
@@ -2449,6 +2450,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
24492450
yield* applyColorScheme(tabId, wc, colorScheme);
24502451
});
24512452

2453+
const setShown = Effect.fn("PreviewManager.setShown")(function* (tabId: string, shown: boolean) {
2454+
const serverTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId)?.serverTab;
2455+
if (serverTab) browserHost.setShown(serverTab, shown);
2456+
});
2457+
24522458
const setAudioMuted = Effect.fn("PreviewManager.setAudioMuted")(function* (
24532459
tabId: string,
24542460
audioMuted: boolean,
@@ -3416,6 +3422,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
34163422
setAnnotationTheme,
34173423
setAnnotationSendEnabled,
34183424
setAudioMuted,
3425+
setShown,
34193426
setColorScheme,
34203427
setMainWindow,
34213428
setForwardedShortcuts: (shortcuts: ReadonlyArray<PreviewForwardedShortcut>) =>
@@ -3623,6 +3630,7 @@ export class PreviewManager extends Context.Service<
36233630
tabId: string,
36243631
audioMuted: boolean,
36253632
) => Effect.Effect<void, PreviewManagerError>;
3633+
readonly setShown: (tabId: string, shown: boolean) => Effect.Effect<void>;
36263634
readonly openDevTools: (tabId: string) => Effect.Effect<void, PreviewManagerError>;
36273635
readonly clearCookies: (
36283636
partitions?: ReadonlyArray<string>,
@@ -3766,6 +3774,7 @@ export const make = Effect.gen(function* PreviewManagerMake() {
37663774
setColorScheme: operations.setColorScheme,
37673775
setZoomFactor: operations.setZoomFactor,
37683776
setAudioMuted: operations.setAudioMuted,
3777+
setShown: operations.setShown,
37693778
openDevTools: operations.openDevTools,
37703779
clearCookies: Effect.fn("PreviewManager.clearCookies")(function* (partitions) {
37713780
yield* browserSession

‎apps/server/src/preview/DesktopBrowserChannel.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ export class DesktopBrowserChannel extends Context.Service<
5252
readonly detached: Stream.Stream<DesktopTabKey>;
5353
readonly attached: Stream.Stream<DesktopTabKey>;
5454
readonly isAttached: (key: DesktopTabKey) => Effect.Effect<boolean>;
55+
/** Whether a desktop window draws the tab on screen, which no streamed viewer counts. */
56+
readonly isShown: (key: DesktopTabKey) => boolean;
57+
readonly shownChanged: Stream.Stream<DesktopTabKey>;
5558
readonly endpoint: (key: DesktopTabKey) => Effect.Effect<string, never, Scope.Scope>;
5659
readonly pointer: (
5760
key: DesktopTabKey,
@@ -66,6 +69,8 @@ const make = Effect.gen(function* () {
6669
const controlFd = config.desktopBrowserControlFd;
6770
const changes = yield* PubSub.unbounded<{ key: DesktopTabKey; attached: boolean }>();
6871
const attachedTabs = new Set<string>();
72+
const shownTabs = new Set<string>();
73+
const shownChanges = yield* PubSub.unbounded<DesktopTabKey>();
6974
const inbound = new Map<string, Queue.Queue<string>>();
7075
const writeLock = yield* Semaphore.make(1);
7176

@@ -76,6 +81,8 @@ const make = Effect.gen(function* () {
7681
detached: Stream.empty,
7782
attached: Stream.empty,
7883
isAttached: () => Effect.succeed(false),
84+
isShown: () => false,
85+
shownChanged: Stream.empty,
7986
endpoint: () => Effect.die("No desktop app is attached to this server."),
8087
pointer: () => Effect.void,
8188
});
@@ -124,6 +131,11 @@ const make = Effect.gen(function* () {
124131
Effect.andThen(PubSub.publish(changes, { key, attached: false })),
125132
);
126133
}
134+
case "shown":
135+
if (shownTabs.has(id) === event.shown) return Effect.void;
136+
if (event.shown) shownTabs.add(id);
137+
else shownTabs.delete(id);
138+
return PubSub.publish(shownChanges, key);
127139
}
128140
}),
129141
Effect.catchCause((cause) => Effect.logWarning("desktop browser channel stopped", { cause })),
@@ -214,6 +226,8 @@ const make = Effect.gen(function* () {
214226
Stream.map((change) => change.key),
215227
),
216228
isAttached: (key) => Effect.sync(() => attachedTabs.has(keyOf(key))),
229+
isShown: (key) => shownTabs.has(keyOf(key)),
230+
shownChanged: Stream.fromPubSub(shownChanges),
217231
endpoint,
218232
pointer: (key, pointer) => command({ type: "pointer", ...key, ...pointer }),
219233
});

‎apps/server/src/preview/ServerBrowser.test.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,7 @@ const desktopRenders = (tabId: string) => {
230230
}
231231
return desktopTabs.has(tabId);
232232
};
233+
const shownDesktopTabs = new Set<string>();
233234
const releasedDesktopTabs: Array<string> = [];
234235
const desktopConnections: Array<{ endpoint: string; context: ReturnType<typeof makeContext> }> = [];
235236
const testThread = {
@@ -286,6 +287,8 @@ const dependencies = Layer.mergeAll(
286287
),
287288
),
288289
isAttached: (key) => Effect.sync(() => desktopRenders(key.tabId)),
290+
isShown: (key) => shownDesktopTabs.has(key.tabId),
291+
shownChanged: Stream.empty,
289292
endpoint: (key) =>
290293
Effect.acquireRelease(Effect.succeed(`ws://desktop/${key.tabId}`), () =>
291294
Effect.sync(() => releasedDesktopTabs.push(key.tabId)),
@@ -329,6 +332,7 @@ beforeEach(() => {
329332
focusedFrame = null;
330333
desktopTabs.clear();
331334
desktopRendersNext = false;
335+
shownDesktopTabs.clear();
332336
releasedDesktopTabs.length = 0;
333337
desktopConnections.length = 0;
334338
});
@@ -991,6 +995,41 @@ it.live("only the current controller can answer a file picker", () =>
991995
).pipe(Effect.provide(layer)),
992996
);
993997

998+
it.live("a tab the desktop draws is visible only while the desktop shows it", () =>
999+
Effect.scoped(
1000+
Effect.gen(function* () {
1001+
const broker = yield* Broker.PreviewAutomationBroker;
1002+
const manager = yield* Manager.PreviewManager;
1003+
yield* ServerBrowser.ServerBrowser;
1004+
yield* Effect.yieldNow;
1005+
desktopRendersNext = true;
1006+
const opened = yield* manager.open({
1007+
threadId: scope.thread.threadId,
1008+
url: "http://localhost:5173/desktop",
1009+
runtime: "server",
1010+
});
1011+
const tabId = PreviewTabId.make(opened.tabId);
1012+
const status = broker.invoke<PreviewAutomationStatus>({
1013+
scope,
1014+
tabId,
1015+
operation: "status",
1016+
input: {},
1017+
});
1018+
expect(yield* status).toMatchObject({
1019+
visible: false,
1020+
tabs: [expect.objectContaining({ tabId, visible: false })],
1021+
});
1022+
expect(desktopConnections).toHaveLength(1);
1023+
1024+
shownDesktopTabs.add(tabId);
1025+
expect(yield* status).toMatchObject({
1026+
visible: true,
1027+
tabs: [expect.objectContaining({ tabId, visible: true })],
1028+
});
1029+
}),
1030+
).pipe(Effect.provide(layer)),
1031+
);
1032+
9941033
it.live("agents read a human's tab with no arguments and act on it only while nobody drives", () =>
9951034
Effect.scoped(
9961035
Effect.gen(function* () {

‎apps/server/src/preview/ServerBrowser.ts‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,9 @@ const make = Effect.gen(function* () {
468468

469469
const tabs = new Map<string, ServerTab>();
470470
const pendingTabs = new Map<string, Promise<ServerTab>>();
471+
// The desktop draws its tabs in its own webview, so they never get a streamed viewer.
472+
const isVisible = (tab: ServerTab) =>
473+
tab.viewers.size > 0 || (tab.desktop !== null && desktopChannel.isShown(tab));
471474

472475
const closedPendingTabs = new Set<string>();
473476
const adoptedPages = new Map<string, { readonly page: Page; readonly openerTabId: string }>();
@@ -589,7 +592,7 @@ const make = Effect.gen(function* () {
589592
liveTabs: [...tabs.values()].map((tab) => ({
590593
threadId: tab.threadId,
591594
tabId: tab.tabId,
592-
visible: tab.viewers.size > 0,
595+
visible: isVisible(tab),
593596
})),
594597
}),
595598
),
@@ -1296,8 +1299,7 @@ const make = Effect.gen(function* () {
12961299
.sort((left, right) => right.createdAt - left.createdAt)[0] ??
12971300
threadTabs.sort(
12981301
(left, right) =>
1299-
Number(right.viewers.size > 0) - Number(left.viewers.size > 0) ||
1300-
right.usedAt - left.usedAt,
1302+
Number(isVisible(right)) - Number(isVisible(left)) || right.usedAt - left.usedAt,
13011303
)[0]
13021304
);
13031305
};
@@ -1371,7 +1373,7 @@ const make = Effect.gen(function* () {
13711373
const viewport = tab.page.viewportSize();
13721374
const status = {
13731375
available: true,
1374-
visible: tab.viewers.size > 0,
1376+
visible: isVisible(tab),
13751377
tabId: tab.tabId,
13761378
url: url === "about:blank" ? null : url,
13771379
title: null,
@@ -1395,7 +1397,7 @@ const make = Effect.gen(function* () {
13951397
...(candidate.openerTabId === undefined ? {} : { openerTabId: candidate.openerTabId }),
13961398
owner: tabOwner(candidate),
13971399
ownedByCaller: candidate.control.agentId === agentSessionId,
1398-
visible: candidate.viewers.size > 0,
1400+
visible: isVisible(candidate),
13991401
...(candidate.profileId === undefined ? {} : { profileId: candidate.profileId }),
14001402
})),
14011403
...profileStatus(),
@@ -2470,6 +2472,10 @@ const make = Effect.gen(function* () {
24702472
),
24712473
Effect.forkScoped,
24722474
);
2475+
yield* desktopChannel.shownChanged.pipe(
2476+
Stream.runForEach(() => Effect.sync(reportLiveTabs)),
2477+
Effect.forkScoped,
2478+
);
24732479
// The page came back, for example after its DevTools closed. Reconnect now
24742480
// rather than on the next viewer or agent, or the tab's URL and title stop
24752481
// reaching every client in the meantime.

‎apps/web/src/browser/HostedBrowserWebview.tsx‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,14 @@ export function HostedBrowserWebview(props: {
291291
viewportWidth,
292292
]);
293293

294+
const shown = active || pictureInPicture;
295+
useEffect(() => {
296+
const bridge = window.desktopBridge?.preview;
297+
if (!bridge || !serverDriven || !shown) return;
298+
withDesktopTab(runtimeTabId, () => bridge.setShown(runtimeTabId, true));
299+
return () => withDesktopTab(runtimeTabId, () => bridge.setShown(runtimeTabId, false));
300+
}, [runtimeTabId, serverDriven, shown]);
301+
294302
if (!clientSettingsHydrated || !config) return null;
295303

296304
const renderingActive = active || backgroundActivity || pictureInPicture || recordingActive;

0 commit comments

Comments
 (0)