Skip to content

Commit 46f3c2c

Browse files
feat(web): add iPhone Duo 3D controls (pingdotgg#12813)
1 parent fd46510 commit 46f3c2c

25 files changed

Lines changed: 2951 additions & 44 deletions

‎apps/server/src/device/DeviceHubProxy.test.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,3 +139,29 @@ it.each([
139139
expect(await response.text()).not.toContain("private credential diagnostic");
140140
expect(requests).toEqual([]);
141141
});
142+
143+
it.each([1, 3])(
144+
"forwards fixed Duo display %s through the authenticated read proxy",
145+
async (panel) => {
146+
const { handler, requests } = fixture([AuthOrchestrationReadScope]);
147+
const route = `/vendor/serve-sim/helper/duo/panel/${panel}/stream.avcc`;
148+
const response = await handler(
149+
new Request(`http://t3.test/api/device-hub${route}?wsTicket=secret`),
150+
);
151+
expect(response.status).toBe(200);
152+
await response.text();
153+
expect(requests).toEqual([`http://hub.test${route}`]);
154+
},
155+
);
156+
157+
it.each(["/panel/2/stream.avcc", "/panel/1/webrtc/offer", "/panel/3/exec"])(
158+
"rejects unsupported Duo route %s",
159+
async (route) => {
160+
const { handler, requests } = fixture([AuthOrchestrationReadScope]);
161+
const response = await handler(
162+
new Request(`http://t3.test/api/device-hub/vendor/serve-sim/helper/duo${route}`),
163+
);
164+
expect(response.status).toBe(404);
165+
expect(requests).toEqual([]);
166+
},
167+
);

‎apps/server/src/device/DeviceHubProxy.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ const ALLOWED_PATHS: ReadonlyArray<RegExp> = [
4242
/^\/vendor\/serve-sim\/api\/screenshot$/,
4343
/^\/vendor\/serve-sim\/api\/event-log(\/events)?$/,
4444
/^\/vendor\/serve-sim\/helper\/[^/]+\/(stream\.mjpeg|stream\.avcc|config|health|ax|foreground)$/,
45+
/^\/vendor\/serve-sim\/helper\/[^/]+\/panel\/(1|3)\/stream\.avcc$/,
4546
/^\/vendor\/serve-sim\/appstate$/,
4647
/^\/vendor\/serve-emu\/api\/(devices|screenshot|stream-mode|stream-settings|accessibility)$/,
4748
/^\/vendor\/serve-emu\/health$/,

‎apps/server/src/device/DeviceToolchain.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import * as Semaphore from "effect/Semaphore";
2626
import * as ProcessRunner from "../processRunner.ts";
2727

2828
const DEVICE_HUB_PACKAGE = "expo-device-hub";
29-
export const DEVICE_HUB_VERSION = "0.10.1";
29+
export const DEVICE_HUB_VERSION = "0.12.0";
3030
const AGENT_DEVICE_PACKAGE = "agent-device";
3131
export const AGENT_DEVICE_VERSION = "0.21.12";
3232

‎apps/web/src/components/device/DeviceControlsRail.tsx‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,9 @@ export function DeviceControlsRail(props: {
5757
<aside
5858
aria-label="Device controls"
5959
data-layout="rail"
60-
className="pointer-events-none absolute inset-y-0 right-0 z-10 flex w-14 flex-col items-center justify-center py-3 pr-2"
60+
className="pointer-events-none absolute inset-y-0 right-0 z-10 flex w-14 flex-col items-center gap-2 overflow-y-auto [justify-content:safe_center] py-3 pr-2 [scrollbar-width:none]"
6161
>
62-
<div className="pointer-events-auto flex max-h-full shrink-0 flex-col items-center gap-1 overflow-y-auto rounded-full border border-border/50 bg-background/80 p-2 shadow-sm [scrollbar-width:none]">
62+
<div className="pointer-events-auto flex shrink-0 flex-col items-center gap-1 overflow-y-auto rounded-full border border-border/50 bg-background/80 p-2 shadow-sm [scrollbar-width:none]">
6363
<RailButton
6464
tooltipSide={popupSide}
6565
label="Home"
@@ -255,11 +255,12 @@ export function DeviceControlsRail(props: {
255255
</RailButton>
256256
) : null}
257257
{view.phone ? (
258-
<RailButton tooltipSide={popupSide} label="Reset phone view" onClick={view.resetView}>
258+
<RailButton tooltipSide={popupSide} label="Restore 3D view" onClick={view.resetView}>
259259
<Maximize />
260260
</RailButton>
261261
) : null}
262262
</div>
263+
{view.foldingControls}
263264
</aside>
264265
);
265266
}
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
import {
2+
DUO_POSES,
3+
type DuoCommand,
4+
type DuoControlState,
5+
} from "@t3tools/client-runtime/device/duo-control";
6+
import type { DeviceScreenSize } from "@t3tools/client-runtime/device/stream";
7+
import { DeviceDuoGlyph } from "./DeviceDuoGlyph";
8+
import { Button } from "~/components/ui/button";
9+
import { Tooltip, TooltipPopup, TooltipTrigger } from "~/components/ui/tooltip";
10+
11+
/** Physical presets live beside the device. Pinching supplies continuous hinge control. */
12+
export function DeviceDuoControls(props: {
13+
screen: DeviceScreenSize;
14+
state: DuoControlState;
15+
enabled: boolean;
16+
onCommand: (command: DuoCommand) => void;
17+
}) {
18+
const angle = props.screen.hingeAngle;
19+
const fold = angle == null ? null : angle === 0 ? "closed" : angle === 180 ? "open" : "book";
20+
const selected = (id: (typeof DUO_POSES)[number]["id"]) =>
21+
id === "laptop" || id === "tent" ? props.screen.hingePose === id : fold === id;
22+
return (
23+
<div aria-label="iPhone Duo stands" className="flex flex-col items-center gap-2">
24+
{([DUO_POSES.slice(0, 3), DUO_POSES.slice(3)] as const).map((poses, index) => (
25+
<div
26+
key={poses[0]?.id}
27+
role="group"
28+
aria-label={index === 0 ? "Fold shape" : "Device stance"}
29+
className="pointer-events-auto flex shrink-0 flex-col items-center gap-1 rounded-full border border-border/50 bg-background/80 p-1 shadow-sm"
30+
>
31+
{poses.map((pose) => (
32+
<Tooltip key={pose.id}>
33+
<TooltipTrigger
34+
render={
35+
<Button
36+
size="icon"
37+
variant={selected(pose.id) ? "secondary" : "ghost"}
38+
disabled={!props.enabled}
39+
aria-label={`${pose.label} stand`}
40+
aria-pressed={selected(pose.id)}
41+
data-pressed={selected(pose.id) ? "" : undefined}
42+
onClick={() => props.onCommand({ control: "pose", value: pose.id })}
43+
/>
44+
}
45+
>
46+
<DeviceDuoGlyph pose={pose.id} />
47+
</TooltipTrigger>
48+
<TooltipPopup side="left">
49+
{pose.label}
50+
{pose.id === "book" ? " / bookshelf" : ""}
51+
</TooltipPopup>
52+
</Tooltip>
53+
))}
54+
</div>
55+
))}
56+
{props.state.error ? (
57+
<Tooltip>
58+
<TooltipTrigger
59+
render={
60+
<span
61+
tabIndex={0}
62+
role="alert"
63+
aria-label={props.state.error}
64+
className="pointer-events-auto text-xs text-destructive"
65+
>
66+
!
67+
</span>
68+
}
69+
/>
70+
<TooltipPopup side="left">{props.state.error}</TooltipPopup>
71+
</Tooltip>
72+
) : null}
73+
</div>
74+
);
75+
}
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import type { DuoPose } from "@t3tools/client-runtime/device/duo-control";
2+
3+
// Bitrig 0.25's toolbar glyphs. The fold outlines follow the native toolbar;
4+
// the rounded stance contours are ported from SimulatorFoldingPoseGlyph's SwiftUI paths.
5+
const stancePaths = {
6+
laptop:
7+
"M2.7351 12.9857 L15.8128 11.4159 L14.4114 3.3787 C14.1943 2.134 13.1275 1.2319 12.0283 1.3639 L2.9308 2.4559 C1.8317 2.5879 1.1167 3.7038 1.3337 4.9485 L2.7351 12.9857 Z M2.7351 12.9857 L15.8128 11.4159 L19.9103 12.8914 C20.5449 13.1199 20.1683 13.4121 19.0692 13.5441 L9.9717 14.6361 C8.8725 14.7681 7.4672 14.6898 6.8326 14.4612 L2.7351 12.9857 Z",
8+
tent: "M3.2522 2.7267 L18.7478 1.4816 L16.696 9.7017 C16.3782 10.9748 15.0649 12.0915 13.7626 12.1962 L2.983 13.0623 C1.6807 13.167 0.8826 12.2199 1.2004 10.9468 L3.2522 2.7267 Z M3.2522 2.7267 L18.7478 1.4816 L20.7996 10.6909 C21.1174 12.1172 20.3193 13.3581 19.017 13.4628 L8.2374 14.3289 C6.9351 14.4336 5.6218 13.3623 5.304 11.936 L3.2522 2.7267 Z",
9+
} as const;
10+
11+
export function DeviceDuoGlyph({ pose }: { pose: DuoPose }) {
12+
const stance = pose === "laptop" || pose === "tent";
13+
return (
14+
<svg
15+
viewBox={stance ? "0 0 22 16" : "0 0 24 32"}
16+
fill="none"
17+
stroke="currentColor"
18+
strokeWidth={stance ? 1.25 : 1.8}
19+
strokeLinecap="round"
20+
strokeLinejoin="round"
21+
className="size-7 shrink-0"
22+
aria-hidden
23+
>
24+
{stance ? <path d={stancePaths[pose]} /> : null}
25+
{pose === "closed" ? (
26+
<>
27+
<rect x="2" y="9" width="20" height="13" rx="2.2" />
28+
<path d="M19.3 12.5v6" />
29+
<circle cx="5" cy="12" r=".5" fill="currentColor" stroke="none" />
30+
</>
31+
) : null}
32+
{pose === "book" ? (
33+
<path d="M4.2 6.5h15.6c1 0 1.6.8 1.2 1.7L17.7 16l3.3 7.8c.4.9-.2 1.7-1.2 1.7H4.2c-1 0-1.6-.8-1.2-1.7L6.3 16 3 8.2c-.4-.9.2-1.7 1.2-1.7Z" />
34+
) : null}
35+
{pose === "open" ? (
36+
<>
37+
<rect x="2" y="2" width="20" height="28" rx="2.4" />
38+
<path d="M10 27h4" />
39+
</>
40+
) : null}
41+
</svg>
42+
);
43+
}
Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
import { useEffect, useRef, type RefObject } from "react";
2+
import type { DuoViewer } from "@t3tools/client-runtime/device/duo-viewer";
3+
import type { DeviceModelSource } from "@t3tools/client-runtime/device/model";
4+
import { createPhoneInteraction } from "@t3tools/client-runtime/device/phone-interaction";
5+
import { createCanvasFrameSink } from "@t3tools/client-runtime/device/frame";
6+
import type { DeviceScreenSize, DeviceStreamClient } from "@t3tools/client-runtime/device/stream";
7+
import { createDuoPinch } from "@t3tools/client-runtime/device/duo-control";
8+
import { bindPhoneTrackpad } from "./phoneTrackpad";
9+
10+
const loadDuoViewer = () => import("@t3tools/client-runtime/device/duo-viewer");
11+
12+
/** Web shell for the framework-independent viewer. The decoded screen and input connection remain owned by DeviceStreamView. */
13+
export function DeviceDuoViewport(props: {
14+
readonly model: DeviceModelSource;
15+
readonly source: RefObject<HTMLCanvasElement | null>;
16+
readonly onFrameListener: (listener: (() => void) | null) => void;
17+
readonly onResetReady: (reset: (() => void) | null) => void;
18+
readonly onInputCancel: (cancel: (() => void) | null) => void;
19+
readonly client: RefObject<DeviceStreamClient | null>;
20+
readonly screen: DeviceScreenSize | null;
21+
readonly hingePreview: number | null;
22+
readonly controlError: string | null;
23+
readonly onUnavailable: () => void;
24+
}) {
25+
const hostRef = useRef<HTMLDivElement | null>(null);
26+
const canvasRef = useRef<HTMLCanvasElement | null>(null);
27+
const viewerRef = useRef<DuoViewer | null>(null);
28+
const interactionRef = useRef<ReturnType<typeof createPhoneInteraction> | null>(null);
29+
const pinchRef = useRef<ReturnType<typeof createDuoPinch> | null>(null);
30+
const trackpadRef = useRef<ReturnType<typeof bindPhoneTrackpad> | null>(null);
31+
const screenRef = useRef(props.screen);
32+
const modelRef = useRef(props.model);
33+
const previewRef = useRef(props.hingePreview);
34+
const { source, client, onInputCancel, onResetReady, onUnavailable, onFrameListener } = props;
35+
36+
useEffect(() => {
37+
screenRef.current = props.screen;
38+
modelRef.current = props.model;
39+
interactionRef.current?.end();
40+
viewerRef.current?.setScreen(props.screen);
41+
}, [props.screen, props.model]);
42+
43+
useEffect(() => {
44+
if (props.controlError || !props.screen) {
45+
trackpadRef.current?.cancel();
46+
viewerRef.current?.rejectOrientation();
47+
}
48+
}, [props.controlError, props.screen]);
49+
50+
useEffect(() => {
51+
previewRef.current = props.hingePreview;
52+
if (!pinchRef.current?.active) viewerRef.current?.setHingePreview(props.hingePreview);
53+
}, [props.hingePreview]);
54+
55+
useEffect(() => {
56+
const host = hostRef.current;
57+
const canvas = canvasRef.current;
58+
const decoded = source.current;
59+
if (!host || !canvas || !decoded) return;
60+
let disposed = false;
61+
let trackpad: ReturnType<typeof bindPhoneTrackpad> | null = null;
62+
let stopTrackpadEnd: (() => void) | undefined;
63+
const resize = () => {
64+
const { width, height } = host.getBoundingClientRect();
65+
viewerRef.current?.resize(width, height, window.devicePixelRatio);
66+
};
67+
const observer = new ResizeObserver(resize);
68+
observer.observe(host);
69+
const blur = () => {
70+
interactionRef.current?.end();
71+
trackpad?.cancel();
72+
viewerRef.current?.cancelInput();
73+
};
74+
onInputCancel(blur);
75+
window.addEventListener("blur", blur);
76+
void loadDuoViewer()
77+
.then(({ createDuoViewer }) => {
78+
if (disposed) return;
79+
const sources = {
80+
1: document.createElement("canvas"),
81+
3: document.createElement("canvas"),
82+
};
83+
const viewer = createDuoViewer({
84+
canvas,
85+
sources,
86+
onUnavailable,
87+
onPanelRequested: (panel) =>
88+
client.current?.controlDuo({
89+
control: "physical",
90+
value: panel === 1 ? "facedown" : "faceup",
91+
}),
92+
onOrientationRequested: (value) =>
93+
client.current?.controlDuo({ control: "orientation", value }),
94+
onModelError: (cause) => console.warn("Device 3D asset could not load", cause),
95+
model: modelRef.current,
96+
});
97+
viewerRef.current = viewer;
98+
onResetReady(viewer.resetPose);
99+
viewer.setScreen(screenRef.current);
100+
const primaryFrame = () => {
101+
const panel = screenRef.current?.screenId;
102+
if (panel === 1 || panel === 3) viewer.frameUpdated(panel, decoded);
103+
};
104+
onFrameListener(primaryFrame);
105+
primaryFrame();
106+
viewer.setHingePreview(previewRef.current);
107+
client.current?.setDuoPanels({
108+
onScreen(next) {
109+
screenRef.current = next;
110+
interactionRef.current?.end();
111+
viewer.setScreen(next);
112+
},
113+
cover: createCanvasFrameSink(sources[1], () => viewer.frameUpdated(1)),
114+
inner: createCanvasFrameSink(sources[3], () => viewer.frameUpdated(3)),
115+
});
116+
interactionRef.current = createPhoneInteraction({
117+
screenPoint: (point, captured) => viewer.screenPoint(point.x, point.y, captured),
118+
touch: (phase, point) => client.current?.sendRawTouch(phase, point.x, point.y),
119+
orbit: viewer.orbit,
120+
zoomBy: () => {},
121+
onInteractionActive: viewer.setInteractionActive,
122+
});
123+
const pinch = createDuoPinch({
124+
angle: () =>
125+
previewRef.current ??
126+
screenRef.current?.hingeAngle ??
127+
(screenRef.current?.screenId === 1 ? 0 : 180),
128+
contains: (x, y) => !!screenRef.current && viewer.beginHinge(x, y),
129+
change: (angle) => {
130+
viewer.setHingePreview(angle ?? previewRef.current);
131+
if (angle !== null) client.current?.controlDuo({ control: "angle", value: angle });
132+
},
133+
});
134+
pinchRef.current = pinch;
135+
trackpad = bindPhoneTrackpad(canvas, interactionRef.current, {
136+
begin(x, y) {
137+
interactionRef.current?.end();
138+
return pinch.begin(x, y);
139+
},
140+
move: pinch.move,
141+
end: pinch.end,
142+
});
143+
trackpadRef.current = trackpad;
144+
stopTrackpadEnd = window.desktopBridge?.onTrackpadScrollEnd?.(() => trackpad?.endOrbit());
145+
resize();
146+
})
147+
.catch(() => {
148+
if (!disposed) onUnavailable();
149+
});
150+
return () => {
151+
disposed = true;
152+
stopTrackpadEnd?.();
153+
trackpad?.dispose();
154+
trackpadRef.current = null;
155+
pinchRef.current = null;
156+
interactionRef.current?.end();
157+
interactionRef.current = null;
158+
onInputCancel(null);
159+
onResetReady(null);
160+
onFrameListener(null);
161+
client.current?.setDuoPanels(null);
162+
observer.disconnect();
163+
window.removeEventListener("blur", blur);
164+
viewerRef.current?.dispose();
165+
viewerRef.current = null;
166+
};
167+
}, [onInputCancel, onResetReady, client, onUnavailable, source, onFrameListener]);
168+
169+
const point = (event: React.PointerEvent<HTMLCanvasElement>) => {
170+
const rect = event.currentTarget.getBoundingClientRect();
171+
return {
172+
x: (event.clientX - rect.left) / rect.width,
173+
y: (event.clientY - rect.top) / rect.height,
174+
};
175+
};
176+
return (
177+
<div className="absolute inset-0">
178+
<div ref={hostRef} className="absolute inset-0">
179+
<div
180+
aria-hidden
181+
className="pointer-events-none absolute bottom-[6%] left-1/2 h-5 w-2/5 -translate-x-1/2 rounded-full bg-foreground/10 blur-xl"
182+
/>
183+
<canvas
184+
ref={canvasRef}
185+
aria-label="Interactive 3D iPhone Duo. Drag the screen to interact. Drag outside it or swipe with two fingers to turn. Pinch over the device to open or close its hinge."
186+
className="size-full touch-none"
187+
onPointerDown={(event) => {
188+
if (event.button !== 0) return;
189+
if (!interactionRef.current?.begin(event.pointerId, point(event), event.altKey)) return;
190+
event.preventDefault();
191+
event.currentTarget.setPointerCapture(event.pointerId);
192+
(event.currentTarget.closest('[role="application"]') as HTMLElement | null)?.focus();
193+
}}
194+
onPointerMove={(event) => interactionRef.current?.move(event.pointerId, point(event))}
195+
onPointerUp={(event) => {
196+
interactionRef.current?.move(event.pointerId, point(event));
197+
interactionRef.current?.end(event.pointerId);
198+
}}
199+
onPointerCancel={(event) => interactionRef.current?.end(event.pointerId)}
200+
onLostPointerCapture={(event) => interactionRef.current?.end(event.pointerId)}
201+
/>
202+
</div>
203+
</div>
204+
);
205+
}

0 commit comments

Comments
 (0)