Skip to content

Commit 9957349

Browse files
feat(web): control Android foldables in the Device panel (pingdotgg#13534)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 6391be2 commit 9957349

11 files changed

Lines changed: 985 additions & 15 deletions

File tree

‎apps/server/src/device/DeviceHubProxy.test.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,32 @@ describe("device hub proxy", () => {
111111
await response.text();
112112
});
113113

114+
it("reads Android fold state but requires operate scope to change it", async () => {
115+
const path = "http://t3.test/api/device-hub/vendor/serve-emu/api/fold?device=emulator-5554";
116+
const reader = fixture([AuthOrchestrationReadScope]);
117+
const read = await reader.handler(new Request(path));
118+
expect(read.status).toBe(200);
119+
await read.text();
120+
expect(reader.requests).toEqual([
121+
"http://hub.test/vendor/serve-emu/api/fold?device=emulator-5554",
122+
]);
123+
const denied = await reader.handler(
124+
new Request(path, { method: "POST", body: '{"posture":"closed"}' }),
125+
);
126+
expect(denied.status).toBe(403);
127+
expect(reader.requests).toHaveLength(1);
128+
129+
const operator = fixture([AuthOrchestrationOperateScope]);
130+
const changed = await operator.handler(
131+
new Request(path, { method: "POST", body: '{"posture":"closed"}' }),
132+
);
133+
expect(changed.status).toBe(200);
134+
await changed.text();
135+
expect(operator.requests).toEqual([
136+
"http://hub.test/vendor/serve-emu/api/fold?device=emulator-5554",
137+
]);
138+
});
139+
114140
it("never forwards the vendor shell endpoint", async () => {
115141
const { handler, requests } = fixture([AuthOrchestrationOperateScope]);
116142
expect(

‎apps/server/src/device/DeviceHubProxy.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,15 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
4444
/^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/,
4545
/^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/,
4646
/^\/vendor\/serve-sim\/appstate$/,
47-
/^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility)$/,
47+
/^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility|fold)$/,
4848
/^\/vendor\/serve-emu\/health$/,
4949
];
5050

5151
/** Read paths are GET-only; only these accept other methods (screenshot captures, stream tuning). */
5252
const MUTABLE_PATHS: ReadonlyArray<RegExp> = [
5353
/^\/vendor\/serve-sim\/api\/screenshot$/,
5454
/^\/vendor\/serve-emu\/api\/(screenshot|stream-mode|stream-settings)$/,
55+
/^\/vendor\/serve-emu\/api\/fold$/,
5556
];
5657

5758
const ALLOWED_WS_PATHS: ReadonlyArray<RegExp> = [
@@ -204,7 +205,7 @@ const handler = Effect.gen(function* () {
204205
}
205206
const controlsDevice =
206207
(upgrade && hubPath !== "/api/devices/ws") ||
207-
(!readOnly && /\/api\/stream-(mode|settings)$/.test(hubPath));
208+
(!readOnly && /\/api\/(stream-(mode|settings)|fold)$/.test(hubPath));
208209
yield* authenticate(controlsDevice ? AuthOrchestrationOperateScope : AuthOrchestrationReadScope);
209210
const devices = yield* DeviceService.DeviceService;
210211
const ready = yield* devices.currentReadiness(url.value.searchParams.get("hostId") ?? undefined);
Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess";
2+
import { useEffect, useRef, useState } from "react";
3+
import { Button } from "~/components/ui/button";
4+
import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip";
5+
import {
6+
readAndroidFold,
7+
setAndroidFold,
8+
type AndroidFoldPosture,
9+
type AndroidFoldState,
10+
} from "./deviceFold";
11+
import { DeviceDuoGlyph } from "./DeviceDuoGlyph";
12+
13+
/** Capability comes from the emulator, not its AVD name or screen dimensions. */
14+
export function DeviceAndroidFoldControls(props: {
15+
readonly access: DeviceHubAccess;
16+
readonly deviceId: string;
17+
readonly visible: boolean;
18+
readonly enabled: boolean;
19+
readonly screenWidth: number | undefined;
20+
readonly screenHeight: number | undefined;
21+
readonly onFoldAngle: (angle: number | null) => void;
22+
}) {
23+
const [fold, setFold] = useState<AndroidFoldState | null>(null);
24+
const [pending, setPending] = useState(false);
25+
const [error, setError] = useState<string | null>(null);
26+
const busy = useRef(false);
27+
const mounted = useRef(true);
28+
const { access, deviceId, visible, enabled, screenWidth, screenHeight, onFoldAngle } = props;
29+
30+
useEffect(() => {
31+
mounted.current = true;
32+
return () => {
33+
mounted.current = false;
34+
};
35+
}, []);
36+
37+
useEffect(() => {
38+
if (!visible || !enabled || pending || !screenWidth || !screenHeight) return;
39+
const controller = new AbortController();
40+
let retry: number | undefined;
41+
const read = () => {
42+
void readAndroidFold(access, deviceId, controller.signal)
43+
.then((next) => {
44+
if (!controller.signal.aborted) {
45+
setFold(next);
46+
onFoldAngle(
47+
next.supported ? (next.hingeAngle ?? (next.posture === "closed" ? 0 : 180)) : null,
48+
);
49+
}
50+
})
51+
.catch(() => {
52+
if (!controller.signal.aborted) retry = window.setTimeout(read, 3_000);
53+
});
54+
};
55+
read();
56+
return () => {
57+
controller.abort();
58+
window.clearTimeout(retry);
59+
};
60+
}, [access, deviceId, visible, enabled, screenWidth, screenHeight, pending, onFoldAngle]);
61+
62+
if (!fold?.supported || !props.visible) return null;
63+
64+
const change = (posture: AndroidFoldPosture) => {
65+
if (busy.current || !props.enabled) return;
66+
busy.current = true;
67+
setPending(true);
68+
setError(null);
69+
props.onFoldAngle(posture === "closed" ? 0 : 180);
70+
const controller = new AbortController();
71+
const timeout = window.setTimeout(() => controller.abort(), 12_000);
72+
void setAndroidFold(props.access, props.deviceId, posture, controller.signal)
73+
.then((next) => {
74+
if (mounted.current) {
75+
setFold(next);
76+
props.onFoldAngle(next.hingeAngle ?? (next.posture === "closed" ? 0 : 180));
77+
}
78+
})
79+
.catch((cause: unknown) => {
80+
if (mounted.current) {
81+
props.onFoldAngle(fold.hingeAngle ?? (fold.posture === "closed" ? 0 : 180));
82+
setError(
83+
controller.signal.aborted
84+
? "Fold command timed out."
85+
: cause instanceof Error
86+
? cause.message
87+
: "Could not change fold posture.",
88+
);
89+
}
90+
})
91+
.finally(() => {
92+
window.clearTimeout(timeout);
93+
busy.current = false;
94+
if (mounted.current) setPending(false);
95+
});
96+
};
97+
98+
return (
99+
<div aria-label="Android fold controls" className="flex flex-col items-center gap-2">
100+
<div className="pointer-events-auto flex shrink-0 flex-col items-center gap-1 rounded-full border border-border/50 bg-background/80 p-1 shadow-sm">
101+
<Tooltip>
102+
<TooltipTrigger
103+
render={
104+
<Button
105+
size="icon"
106+
variant={fold.posture === "closed" ? "secondary" : "ghost"}
107+
aria-label="Fold device"
108+
aria-pressed={fold.posture === "closed"}
109+
disabled={pending || !props.enabled}
110+
onClick={() => change("closed")}
111+
/>
112+
}
113+
>
114+
<DeviceDuoGlyph pose="closed" />
115+
</TooltipTrigger>
116+
<TooltipPopup side="left">Fold device</TooltipPopup>
117+
</Tooltip>
118+
<Tooltip>
119+
<TooltipTrigger
120+
render={
121+
<Button
122+
size="icon"
123+
variant={fold.posture === "opened" ? "secondary" : "ghost"}
124+
aria-label="Unfold device"
125+
aria-pressed={fold.posture === "opened"}
126+
disabled={pending || !props.enabled}
127+
onClick={() => change("opened")}
128+
/>
129+
}
130+
>
131+
<DeviceDuoGlyph pose="open" />
132+
</TooltipTrigger>
133+
<TooltipPopup side="left">Unfold device</TooltipPopup>
134+
</Tooltip>
135+
</div>
136+
{error ? (
137+
<Tooltip>
138+
<TooltipTrigger
139+
render={
140+
<span
141+
tabIndex={0}
142+
role="alert"
143+
aria-label={error}
144+
className="pointer-events-auto text-xs text-destructive"
145+
>
146+
!
147+
</span>
148+
}
149+
/>
150+
<TooltipPopup side="left">{error}</TooltipPopup>
151+
</Tooltip>
152+
) : null}
153+
</div>
154+
);
155+
}

‎apps/web/src/components/device/DevicePhoneViewport.tsx‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,18 +22,25 @@ export function DevicePhoneViewport(props: {
2222
readonly onInputCancel: (cancel: (() => void) | null) => void;
2323
readonly client: RefObject<DeviceStreamClient | null>;
2424
readonly screen: DeviceScreenSize | null;
25+
readonly foldAngle: number | null;
2526
readonly onUnavailable: () => void;
2627
}) {
2728
const hostRef = useRef<HTMLDivElement | null>(null);
2829
const canvasRef = useRef<HTMLCanvasElement | null>(null);
2930
const viewerRef = useRef<PhoneViewer | null>(null);
3031
const interactionRef = useRef<ReturnType<typeof createPhoneInteraction> | null>(null);
3132
const screenRef = useRef(props.screen);
33+
const foldAngleRef = useRef(props.foldAngle);
3234
const profileRef = useRef(props.profile);
3335
const modelRef = useRef(props.model);
3436
const accessoryRef = useRef(props.accessory);
3537
const { source, onFrameListener, client, onInputCancel, onResetReady, onUnavailable } = props;
3638

39+
useEffect(() => {
40+
foldAngleRef.current = props.foldAngle;
41+
viewerRef.current?.setFoldAngle(props.foldAngle);
42+
}, [props.foldAngle]);
43+
3744
useEffect(() => {
3845
screenRef.current = props.screen;
3946
profileRef.current = props.profile;
@@ -76,6 +83,7 @@ export function DevicePhoneViewport(props: {
7683
profile: profileRef.current,
7784
model: modelRef.current,
7885
accessory: accessoryRef.current,
86+
foldAngle: foldAngleRef.current,
7987
});
8088
viewerRef.current = viewer;
8189
onResetReady(viewer.resetPose);

‎apps/web/src/components/device/DeviceStreamView.tsx‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { deviceKeyboard, deviceModel } from "./deviceModels";
1010
import { fitDeviceFrame } from "./deviceFrameLayout";
1111
import { DeviceDuoViewport } from "./DeviceDuoViewport";
1212
import { DeviceDuoControls } from "./DeviceDuoControls";
13+
import { DeviceAndroidFoldControls } from "./DeviceAndroidFoldControls";
1314
import type { DuoControlState } from "@t3tools/client-runtime/device/duo-control";
1415
import { DevicePhoneViewport } from "./DevicePhoneViewport";
1516
import { DeviceLoadingView } from "./DeviceLoadingView";
@@ -94,7 +95,9 @@ export function DeviceStreamView(props: {
9495
const clientRef = useRef<DeviceStreamClient | null>(null);
9596
const [status, setStatus] = useState<DeviceStreamStatus>("connecting");
9697
const [detail, setDetail] = useState<string | undefined>(undefined);
98+
const [showRestartNotice, setShowRestartNotice] = useState(false);
9799
const [screen, setScreen] = useState<DeviceScreenSize | null>(null);
100+
const [foldAngle, setFoldAngle] = useState<number | null>(null);
98101
const [mjpegUrl, setMjpegUrl] = useState<string | null>(null);
99102
const [mjpegGeneration, setMjpegGeneration] = useState(0);
100103
const attachMjpegImage = useCallback((image: HTMLImageElement | null) => {
@@ -121,6 +124,7 @@ export function DeviceStreamView(props: {
121124
onStatus: (next, nextDetail) => {
122125
setStatus(next);
123126
setDetail(nextDetail);
127+
if (next !== "connecting") setShowRestartNotice(false);
124128
},
125129
onScreen: (next) => {
126130
setScreen(next);
@@ -136,6 +140,7 @@ export function DeviceStreamView(props: {
136140
},
137141
onInputConnected: (connected, detail) => {
138142
setInputState({ connected, ...(detail ? { detail } : {}) });
143+
if (!connected) setShowRestartNotice(false);
139144
onHandle?.({
140145
pressButton: client.pressButton,
141146
rotate: client.rotate,
@@ -183,15 +188,27 @@ export function DeviceStreamView(props: {
183188
return w / h;
184189
}, [props.platform, screen]);
185190

191+
// Android restarts its encoder when a fold changes the framebuffer size.
192+
// Keep the last decoded frame and viewer mounted while the next keyframe arrives.
193+
const retainingAndroidFrame =
194+
props.platform === "android" &&
195+
status === "connecting" &&
196+
inputState.connected &&
197+
screen !== null;
186198
const showPhone =
187199
props.allowPhoneView &&
188-
status === "streaming" &&
200+
(status === "streaming" || retainingAndroidFrame) &&
189201
props.visible &&
190202
presentation === "phone" &&
191203
!phoneUnavailable &&
192204
!mjpegUrl &&
193205
!props.axOverlay &&
194206
(!isDuo || screen?.supportsHingeAngle === true);
207+
useEffect(() => {
208+
if (!retainingAndroidFrame || !showPhone) return;
209+
const timeout = window.setTimeout(() => setShowRestartNotice(true), 2_000);
210+
return () => window.clearTimeout(timeout);
211+
}, [retainingAndroidFrame, showPhone]);
195212
const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0;
196213

197214
// The frame is the largest box at `aspect` that fits the container, so a
@@ -330,7 +347,18 @@ export function DeviceStreamView(props: {
330347
streaming: status === "streaming",
331348
phoneUnavailableReason,
332349
foldingControls:
333-
showPhone && isDuo && screen?.supportsHingeAngle ? (
350+
props.platform === "android" && access ? (
351+
<DeviceAndroidFoldControls
352+
key={`${props.hostId}:${props.deviceId}`}
353+
access={access}
354+
deviceId={props.deviceId}
355+
visible={props.visible}
356+
enabled={status === "streaming"}
357+
screenWidth={screen?.width}
358+
screenHeight={screen?.height}
359+
onFoldAngle={setFoldAngle}
360+
/>
361+
) : showPhone && isDuo && screen?.supportsHingeAngle ? (
334362
<DeviceDuoControls
335363
screen={screen}
336364
state={duoControl}
@@ -471,6 +499,7 @@ export function DeviceStreamView(props: {
471499
onInputCancel={onInputCancel}
472500
onResetReady={onResetReady}
473501
screen={screen}
502+
foldAngle={foldAngle}
474503
onUnavailable={onPhoneUnavailable}
475504
/>
476505
) : null}
@@ -503,7 +532,14 @@ export function DeviceStreamView(props: {
503532
</span>
504533
</div>
505534
) : null}
506-
{status !== "streaming" ? (
535+
{retainingAndroidFrame && showPhone && showRestartNotice ? (
536+
<div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2">
537+
<span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">
538+
Waiting for device video…
539+
</span>
540+
</div>
541+
) : null}
542+
{status !== "streaming" && !(retainingAndroidFrame && showPhone) ? (
507543
<div className="absolute inset-0">
508544
<DeviceLoadingView
509545
name={props.deviceName ?? "Device"}

0 commit comments

Comments
 (0)