Skip to content

Commit c6e0cb4

Browse files
feat(web): float device streams over chat
The floating preview mirrored only browser tabs, so watching an agent drive a simulator meant keeping the right panel open on the device tab. The mini player now takes a source: a browser tab or a device stream. Agent-opened devices float over chat like agent-driven browsers (a right-panel tab when auto-show is off), the device panel toolbar can float the active device, and closing the panel on a device floats it instead of dropping it. The floating device follows the stream's reported screen size, rotates with the device, and closes when its session ends. Co-Authored-By: Claude Code <[email protected]>
1 parent 05d4042 commit c6e0cb4

15 files changed

Lines changed: 550 additions & 163 deletions

‎apps/web/src/components/ChatView.logic.test.ts‎

Lines changed: 44 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ describe("floating browser preview", () => {
126126
const ref = scopeThreadRef(EnvironmentId.make("env-1"), ThreadId.make("thread-1"));
127127
const panels = useRightPanelStore.getState();
128128
const revision = panels.getUserActionRevision(ref);
129-
usePreviewMiniPlayerStore.getState().open(ref, "agent-tab");
129+
usePreviewMiniPlayerStore.getState().open(ref, { kind: "browser", tabId: "agent-tab" });
130130
panels.reconcileBrowserSurfaces(ref, ["agent-tab"]);
131131
const intent = selectThreadPreviewMiniPlayer(
132132
usePreviewMiniPlayerStore.getState().byThreadKey,
@@ -135,7 +135,7 @@ describe("floating browser preview", () => {
135135
const isFloating = () =>
136136
shouldRenderPreviewMiniPlayer(
137137
selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, ref)
138-
?.tabId ?? null,
138+
?.source ?? null,
139139
selectActiveRightPanelSurface(useRightPanelStore.getState().byThreadKey, ref),
140140
);
141141

@@ -152,22 +152,61 @@ describe("floating browser preview", () => {
152152
});
153153

154154
it("only hides the duplicate while the same browser is rendered in the panel", () => {
155+
const tab = { kind: "browser", tabId: "tab-1" } as const;
155156
expect(shouldRenderPreviewMiniPlayer(null, null)).toBe(false);
156157
expect(
157-
shouldRenderPreviewMiniPlayer("tab-1", {
158+
shouldRenderPreviewMiniPlayer(tab, {
158159
id: "browser:one",
159160
kind: "preview",
160161
resourceId: "tab-1",
161162
}),
162163
).toBe(false);
163164
expect(
164-
shouldRenderPreviewMiniPlayer("tab-1", {
165+
shouldRenderPreviewMiniPlayer(tab, {
165166
id: "browser:two",
166167
kind: "preview",
167168
resourceId: "tab-2",
168169
}),
169170
).toBe(true);
170-
expect(shouldRenderPreviewMiniPlayer("tab-1", { id: "diff", kind: "diff" })).toBe(true);
171+
expect(shouldRenderPreviewMiniPlayer(tab, { id: "diff", kind: "diff" })).toBe(true);
172+
});
173+
174+
it("only hides a floating device while that device is rendered in the panel", () => {
175+
const pixel = {
176+
kind: "device",
177+
hostId: "nucbox",
178+
deviceId: "emulator-5580",
179+
platform: "android",
180+
name: "Pixel",
181+
} as const;
182+
const target = {
183+
hostId: "nucbox",
184+
deviceId: "emulator-5580",
185+
platform: "android",
186+
name: "Pixel",
187+
} as const;
188+
expect(
189+
shouldRenderPreviewMiniPlayer(pixel, {
190+
id: "device:nucbox:emulator-5580",
191+
kind: "device",
192+
target,
193+
}),
194+
).toBe(false);
195+
expect(
196+
shouldRenderPreviewMiniPlayer(pixel, {
197+
id: "device:nucbox:emulator-5554",
198+
kind: "device",
199+
target: { ...target, deviceId: "emulator-5554" },
200+
}),
201+
).toBe(true);
202+
expect(shouldRenderPreviewMiniPlayer(pixel, { id: "device", kind: "device" })).toBe(true);
203+
expect(
204+
shouldRenderPreviewMiniPlayer(pixel, {
205+
id: "browser:one",
206+
kind: "preview",
207+
resourceId: "emulator-5580",
208+
}),
209+
).toBe(true);
171210
});
172211
});
173212

‎apps/web/src/components/ChatView.logic.ts‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ import {
5050
import type { DraftThreadEnvMode } from "../composerDraftStore";
5151
import type { ComposerSubmissionIntent } from "../composer-logic";
5252
import type { TimelineEntry } from "../session-logic";
53+
import type { PreviewMiniPlayerSource } from "../previewMiniPlayerStore";
5354
import type { DesktopPreviewOverlay } from "../previewStateStore";
5455
import type { RightPanelSurface } from "../rightPanelStore";
5556
import {
@@ -88,16 +89,22 @@ export function agentControlledBrowserCloseConfirmation(
8889
].join("\n");
8990
}
9091

92+
/** The floating player hides only while the same source is rendered in the panel. */
9193
export function shouldRenderPreviewMiniPlayer(
92-
miniPlayerTabId: string | null,
94+
source: PreviewMiniPlayerSource | null,
9395
renderedRightPanelSurface: RightPanelSurface | null,
9496
): boolean {
95-
return (
96-
miniPlayerTabId !== null &&
97-
!(
97+
if (source === null) return false;
98+
if (source.kind === "browser") {
99+
return !(
98100
renderedRightPanelSurface?.kind === "preview" &&
99-
renderedRightPanelSurface.resourceId === miniPlayerTabId
100-
)
101+
renderedRightPanelSurface.resourceId === source.tabId
102+
);
103+
}
104+
return !(
105+
renderedRightPanelSurface?.kind === "device" &&
106+
renderedRightPanelSurface.target?.hostId === source.hostId &&
107+
renderedRightPanelSurface.target.deviceId === source.deviceId
101108
);
102109
}
103110

‎apps/web/src/components/ChatView.tsx‎

Lines changed: 55 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,8 @@ import {
194194
useSidebarPendingFileDropStore,
195195
} from "../sidebarPendingFileDropStore";
196196
import {
197+
browserMiniPlayerSource,
198+
previewMiniPlayerSourceKey,
197199
selectThreadPreviewMiniPlayer,
198200
usePreviewMiniPlayerStore,
199201
} from "../previewMiniPlayerStore";
@@ -566,6 +568,8 @@ const PreviewPanel = lazy(() =>
566568
import("./preview/PreviewPanel").then((module) => ({ default: module.PreviewPanel })),
567569
);
568570
const DiffPanel = lazy(() => import("./DiffPanel"));
571+
const selectAutoShowFloatingPreview = (settings: { browserAutoShowFloatingPreview: boolean }) =>
572+
settings.browserAutoShowFloatingPreview;
569573
const DevicePanel = lazy(() =>
570574
import("./device/DevicePanel").then((module) => ({ default: module.DevicePanel })),
571575
);
@@ -1962,7 +1966,7 @@ export default function ChatView(props: ChatViewProps) {
19621966
const renderedRightPanelSurface = rightPanelPresence.value?.activeSurface ?? null;
19631967
const renderedRightPanelSurfaces = rightPanelPresence.value?.surfaces ?? [];
19641968
const previewMiniPlayerVisible = shouldRenderPreviewMiniPlayer(
1965-
activePreviewMiniPlayer?.tabId ?? null,
1969+
activePreviewMiniPlayer?.source ?? null,
19661970
renderedRightPanelSurface,
19671971
);
19681972
const canMaximizeRightPanel = rightPanelOpen && !shouldUseRightPanelSheet;
@@ -1978,8 +1982,10 @@ export default function ChatView(props: ChatViewProps) {
19781982
}, [activePreviewState.sessions, activeThreadRef]);
19791983

19801984
useEffect(() => {
1981-
if (!activeThreadRef || !activePreviewMiniPlayer) return;
1982-
const miniTabStillExists = Boolean(activePreviewState.sessions[activePreviewMiniPlayer.tabId]);
1985+
if (!activeThreadRef || activePreviewMiniPlayer?.source.kind !== "browser") return;
1986+
const miniTabStillExists = Boolean(
1987+
activePreviewState.sessions[activePreviewMiniPlayer.source.tabId],
1988+
);
19831989
if (!miniTabStillExists) {
19841990
usePreviewMiniPlayerStore.getState().close(activeThreadRef);
19851991
}
@@ -4195,9 +4201,12 @@ export default function ChatView(props: ChatViewProps) {
41954201
}
41964202
useRightPanelStore.getState().open(activeThreadRef, "device");
41974203
}, [activeThreadRef, deviceState.onboardingCompleted, deviceState.hostStatus]);
4198-
// Reconcile new server sessions into separate tabs, including sessions opened
4199-
// by an agent or another client. The first snapshot is a baseline: persisted
4200-
// tabs restore themselves, and existing sessions must not resurrect closed tabs.
4204+
// A device the agent opens floats over chat like an agent-driven browser,
4205+
// or becomes a panel tab when floating previews are off. Sessions opened by
4206+
// another client arrive the same way. The first snapshot is a baseline:
4207+
// persisted tabs restore themselves, and existing sessions must not
4208+
// resurrect closed tabs.
4209+
const autoShowFloatingPreview = useClientSettings(selectAutoShowFloatingPreview);
42014210
const previousDeviceSessions = useRef(new Map<string, Set<string>>());
42024211
useEffect(() => {
42034212
if (!activeThreadRef || !deviceStateLoaded) return;
@@ -4208,9 +4217,24 @@ export default function ChatView(props: ChatViewProps) {
42084217
const key = (session: (typeof sessions)[number]) => `${session.hostId}:${session.deviceId}`;
42094218
const previous = previousDeviceSessions.current.get(threadKey);
42104219
previousDeviceSessions.current.set(threadKey, new Set(sessions.map(key)));
4211-
if (!previous || shouldUseRightPanelSheet) return;
4220+
if (!previous) return;
42124221
for (const session of sessions) {
42134222
if (previous?.has(key(session))) continue;
4223+
const device = deviceState.devices.find(
4224+
(entry) => entry.hostId === session.hostId && entry.id === session.deviceId,
4225+
);
4226+
if (!device) continue;
4227+
const target = {
4228+
hostId: session.hostId,
4229+
deviceId: session.deviceId,
4230+
platform: device.platform,
4231+
name: device.name,
4232+
};
4233+
if (autoShowFloatingPreview) {
4234+
usePreviewMiniPlayerStore.getState().open(activeThreadRef, { kind: "device", ...target });
4235+
continue;
4236+
}
4237+
if (shouldUseRightPanelSheet) continue;
42144238
const existing = useRightPanelStore
42154239
.getState()
42164240
.byThreadKey[scopedThreadKey(activeThreadRef)]?.surfaces.some(
@@ -4220,28 +4244,30 @@ export default function ChatView(props: ChatViewProps) {
42204244
surface.target.deviceId === session.deviceId,
42214245
);
42224246
if (existing) continue;
4223-
const device = deviceState.devices.find(
4224-
(entry) => entry.hostId === session.hostId && entry.id === session.deviceId,
4225-
);
4226-
if (!device) continue;
4227-
useRightPanelStore.getState().openDevice(
4228-
activeThreadRef,
4229-
{
4230-
hostId: session.hostId,
4231-
deviceId: session.deviceId,
4232-
platform: device.platform,
4233-
name: device.name,
4234-
},
4235-
true,
4236-
);
4247+
useRightPanelStore.getState().openDevice(activeThreadRef, target, true);
42374248
}
42384249
}, [
42394250
activeThreadRef,
4251+
autoShowFloatingPreview,
42404252
deviceStateLoaded,
42414253
shouldUseRightPanelSheet,
42424254
deviceState.sessions,
42434255
deviceState.devices,
42444256
]);
4257+
// A floating device follows its session: once the agent or another client
4258+
// closes the device there is nothing left to stream.
4259+
useEffect(() => {
4260+
if (!activeThreadRef || !deviceStateLoaded) return;
4261+
const source = activePreviewMiniPlayer?.source;
4262+
if (source?.kind !== "device") return;
4263+
const sessionStillExists = deviceState.sessions.some(
4264+
(session) =>
4265+
session.threadId === activeThreadRef.threadId &&
4266+
session.hostId === source.hostId &&
4267+
session.deviceId === source.deviceId,
4268+
);
4269+
if (!sessionStillExists) usePreviewMiniPlayerStore.getState().close(activeThreadRef);
4270+
}, [activePreviewMiniPlayer, activeThreadRef, deviceState.sessions, deviceStateLoaded]);
42454271
const openFileSurface = useCallback(
42464272
(relativePath: string) => {
42474273
if (!activeThreadRef || !activeProject) return;
@@ -4401,10 +4427,15 @@ export default function ChatView(props: ChatViewProps) {
44014427
]);
44024428
const closePreviewPanel = useCallback(() => {
44034429
if (activeThreadRef) {
4430+
// Closing the panel on a live browser or device floats it instead of dropping it.
44044431
if (activeRightPanelSurface?.kind === "preview" && activeRightPanelSurface.resourceId) {
44054432
usePreviewMiniPlayerStore
44064433
.getState()
4407-
.open(activeThreadRef, activeRightPanelSurface.resourceId);
4434+
.open(activeThreadRef, browserMiniPlayerSource(activeRightPanelSurface.resourceId));
4435+
} else if (activeRightPanelSurface?.kind === "device" && activeRightPanelSurface.target) {
4436+
usePreviewMiniPlayerStore
4437+
.getState()
4438+
.open(activeThreadRef, { kind: "device", ...activeRightPanelSurface.target });
44084439
}
44094440
setMaximizedRightPanelThreadKey(null);
44104441
useRightPanelStore.getState().close(activeThreadRef);
@@ -8814,9 +8845,9 @@ export default function ChatView(props: ChatViewProps) {
88148845

88158846
{activeThreadRef && activePreviewMiniPlayer && previewMiniPlayerVisible ? (
88168847
<ThreadPreviewMiniPlayer
8817-
key={`${activeThreadKey}:${activePreviewMiniPlayer.tabId}`}
8848+
key={`${activeThreadKey}:${previewMiniPlayerSourceKey(activePreviewMiniPlayer.source)}`}
88188849
threadRef={activeThreadRef}
8819-
tabId={activePreviewMiniPlayer.tabId}
8850+
miniPlayer={activePreviewMiniPlayer}
88208851
bottomInset={isDraftHeroState ? 0 : composerOverlayHeight}
88218852
/>
88228853
) : null}

‎apps/web/src/components/device/DevicePanel.tsx‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import type {
77
import {
88
ChevronLeft,
99
Home,
10+
PictureInPicture2,
1011
Power,
1112
RotateCcw,
1213
SlidersHorizontal,
@@ -16,6 +17,7 @@ import {
1617
} from "lucide-react";
1718
import { useEffect, useMemo, useState } from "react";
1819

20+
import { usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore";
1921
import { useRightPanelStore, type RightPanelSurface } from "~/rightPanelStore";
2022
import { Button } from "~/components/ui/button";
2123
import { DiscoveryList, DiscoveryListRow } from "~/components/ui/discovery-list";
@@ -116,6 +118,19 @@ export function DevicePanel(props: {
116118
}
117119
};
118120

121+
// Floating the device closes the panel, like the browser's floating preview.
122+
const floatActive = () => {
123+
if (!activeDevice) return;
124+
usePreviewMiniPlayerStore.getState().open(props.threadRef, {
125+
kind: "device",
126+
hostId: activeDevice.hostId,
127+
deviceId: activeDevice.id,
128+
platform: activeDevice.platform,
129+
name: activeDevice.name,
130+
});
131+
useRightPanelStore.getState().close(props.threadRef);
132+
};
133+
119134
const closeActive = (powerOff: boolean) => {
120135
if (!powerOff) {
121136
useRightPanelStore.getState().closeSurface(props.threadRef, props.surface.id);
@@ -218,6 +233,9 @@ export function DevicePanel(props: {
218233
>
219234
<SlidersHorizontal />
220235
</Toggle>
236+
<DeviceButton label="Float device over chat" onClick={floatActive}>
237+
<PictureInPicture2 />
238+
</DeviceButton>
221239
<DeviceButton label="Power off" onClick={() => closeActive(true)}>
222240
<Power />
223241
</DeviceButton>

‎apps/web/src/components/preview/PreviewAutomationHosts.tsx‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,11 @@ import {
2929
reconcilePreviewServerSessions,
3030
updatePreviewServerSnapshot,
3131
} from "~/previewStateStore";
32-
import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore";
32+
import {
33+
browserMiniPlayerSource,
34+
selectThreadPreviewMiniPlayerTabId,
35+
usePreviewMiniPlayerStore,
36+
} from "~/previewMiniPlayerStore";
3337
import { resolveBrowserNavigationTarget } from "~/browser/browserTargetResolver";
3438
import {
3539
readActiveBrowserRecordingTargets,
@@ -378,7 +382,9 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
378382
?.has(runtimeTabId) ?? false,
379383
})
380384
) {
381-
usePreviewMiniPlayerStore.getState().open(threadRef, readyTabId);
385+
usePreviewMiniPlayerStore
386+
.getState()
387+
.open(threadRef, browserMiniPlayerSource(readyTabId));
382388
}
383389
}
384390
browserActivity.release ??= acquireBrowserSurfaceActivity(runtimeTabId);
@@ -493,11 +499,11 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
493499
new Set([activeRuntimeTabId]),
494500
);
495501
}
496-
const miniPlayer = selectThreadPreviewMiniPlayer(
502+
const miniPlayerTabId = selectThreadPreviewMiniPlayerTabId(
497503
usePreviewMiniPlayerStore.getState().byThreadKey,
498504
threadRef,
499505
);
500-
if (miniPlayer?.tabId === activeTabId) {
506+
if (miniPlayerTabId === activeTabId) {
501507
usePreviewMiniPlayerStore.getState().close(threadRef);
502508
}
503509
} else if (shouldPresentPreview) {
@@ -507,7 +513,9 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId })
507513
}
508514
}
509515
if (shouldPresentPreview) {
510-
usePreviewMiniPlayerStore.getState().open(threadRef, activeTabId);
516+
usePreviewMiniPlayerStore
517+
.getState()
518+
.open(threadRef, browserMiniPlayerSource(activeTabId));
511519
}
512520
if (activeSnapshot && previewAutomationOpenNeedsOverlay(input, activeSnapshot)) {
513521
await requireReadyTab();

0 commit comments

Comments
 (0)