Skip to content

Commit ecfb734

Browse files
fix(devices): recover stalled video without losing simulator input (pingdotgg#17566)
1 parent 91a6646 commit ecfb734

6 files changed

Lines changed: 272 additions & 31 deletions

File tree

‎apps/mobile/src/features/devices/device-stream.browser.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ async function setup() {
3838
static OPEN = 1;
3939
readyState = 1;
4040
onopen: (() => void) | null = null;
41+
onmessage: ((event: { data: ArrayBuffer }) => void) | null = null;
4142
close = vi.fn();
4243
send = vi.fn();
4344
constructor() {
@@ -86,6 +87,8 @@ it("bridges shared first-frame readiness, image failure, and a successful fresh
8687
const { elements, messages, sockets, configuration } = await setup();
8788
sockets[0]!.onopen?.();
8889
expect(messages()).not.toContainEqual({ type: "status", status: "streaming" });
90+
expect(messages()).not.toContainEqual({ type: "input", connected: true });
91+
sockets[0]!.onmessage?.({ data: new Uint8Array([0x83]).buffer });
8992
expect(messages()).toContainEqual({ type: "input", connected: true });
9093
const image = elements.find((element) => element.tag === "img")!;
9194
image.naturalWidth = 400;

‎apps/server/src/device/DeviceToolchain.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ import * as Semaphore from "effect/Semaphore";
2626
import * as ProcessRunner from "../processRunner.ts";
2727

2828
const DEVICE_HUB_PACKAGE = "expo-device-hub";
29-
export const DEVICE_HUB_VERSION = "0.12.0";
29+
export const DEVICE_HUB_VERSION = "0.15.3";
3030
const AGENT_DEVICE_PACKAGE = "agent-device";
31-
export const AGENT_DEVICE_VERSION = "0.21.12";
31+
export const AGENT_DEVICE_VERSION = "0.21.23";
3232

3333
const INSTALL_TIMEOUT = Duration.minutes(10);
3434
const installLock = Semaphore.makeUnsafe(1);

‎apps/web/src/components/device/DeviceStreamView.test.tsx‎

Lines changed: 99 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { act, useSyncExternalStore } from "react";
1+
import { act, useEffect, useSyncExternalStore } from "react";
22
import { create, type ReactTestRenderer } from "react-test-renderer";
33
import { EnvironmentId } from "@t3tools/contracts";
44
import { afterEach, beforeEach, expect, it, vi } from "vite-plus/test";
@@ -20,6 +20,20 @@ vi.mock("~/state/device", () => ({
2020
useDeviceHubAccess: () => useSyncExternalStore(accessStore.subscribe, () => accessStore.value),
2121
refreshDeviceHubAccess: () => accessStore.refresh(),
2222
}));
23+
// Replace GPU allocation while keeping the real React and stream lifecycles.
24+
let viewerMounts = 0;
25+
let viewerUnmounts = 0;
26+
vi.mock("./DevicePhoneViewport", () => ({
27+
DevicePhoneViewport: function Viewer() {
28+
useEffect(() => {
29+
viewerMounts++;
30+
return () => {
31+
viewerUnmounts++;
32+
};
33+
}, []);
34+
return null;
35+
},
36+
}));
2337
import { DeviceStreamView } from "./DeviceStreamView";
2438

2539
class Image extends EventTarget {
@@ -34,6 +48,8 @@ let renderer: ReactTestRenderer | undefined;
3448
let primes = 0;
3549
beforeEach(() => {
3650
primes = 0;
51+
viewerMounts = 0;
52+
viewerUnmounts = 0;
3753
});
3854
afterEach(async () => {
3955
await act(async () => renderer?.unmount());
@@ -42,18 +58,57 @@ afterEach(async () => {
4258
vi.unstubAllGlobals();
4359
});
4460

45-
async function setup() {
61+
async function setup(h264 = false) {
4662
vi.useFakeTimers();
63+
vi.stubGlobal("window", globalThis);
4764
vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true);
48-
vi.stubGlobal("fetch", () => {
65+
let videoBody: ReadableStreamDefaultController<Uint8Array> | undefined;
66+
let output: VideoFrameOutputCallback | undefined;
67+
if (h264) {
68+
vi.stubGlobal(
69+
"VideoDecoder",
70+
class {
71+
static isConfigSupported = async () => ({ supported: true });
72+
state = "unconfigured";
73+
constructor(callbacks: VideoDecoderInit) {
74+
output = callbacks.output;
75+
}
76+
configure() {
77+
this.state = "configured";
78+
}
79+
close() {
80+
this.state = "closed";
81+
}
82+
},
83+
);
84+
vi.stubGlobal("EncodedVideoChunk", vi.fn());
85+
}
86+
vi.stubGlobal("fetch", (url: string, init: RequestInit) => {
87+
if (url.endsWith("stream.avcc")) {
88+
return Promise.resolve(
89+
new Response(
90+
new ReadableStream<Uint8Array>({
91+
start(controller) {
92+
videoBody = controller;
93+
init.signal?.addEventListener("abort", () => controller.error(new Error("aborted")));
94+
},
95+
}),
96+
),
97+
);
98+
}
4999
primes++;
50100
return Promise.resolve(new Response("prime"));
51101
});
102+
const sockets: Array<{ onmessage?: (event: { data: ArrayBuffer }) => void }> = [];
52103
vi.stubGlobal(
53104
"WebSocket",
54105
class {
55106
static OPEN = 1;
56107
readyState = 1;
108+
constructor() {
109+
sockets.push(this);
110+
}
111+
onmessage?: (event: { data: ArrayBuffer }) => void;
57112
send() {}
58113
close() {}
59114
},
@@ -73,6 +128,7 @@ async function setup() {
73128
deviceId="test"
74129
platform="ios"
75130
visible={visible}
131+
allowPhoneView={h264}
76132
/>
77133
);
78134
await act(async () => {
@@ -84,13 +140,30 @@ async function setup() {
84140
return image;
85141
}
86142
return {
143+
getContext: () => ({ drawImage() {} }),
87144
style: { setProperty() {} },
88145
getBoundingClientRect: () => ({ width: 400, height: 800 }),
89146
};
90147
},
91148
});
92149
});
93-
return { images, view };
150+
return {
151+
images,
152+
view,
153+
configure() {
154+
const json = new TextEncoder().encode(
155+
JSON.stringify({ width: 400, height: 800, orientation: "portrait" }),
156+
);
157+
const packet = new Uint8Array(1 + json.length);
158+
packet[0] = 0x82;
159+
packet.set(json, 1);
160+
sockets[0]?.onmessage?.({ data: packet.buffer });
161+
videoBody?.enqueue(new Uint8Array([0, 0, 0, 5, 1, 1, 0x64, 0, 0x1f]));
162+
},
163+
frame() {
164+
output?.({ displayWidth: 400, displayHeight: 800, close() {} } as VideoFrame);
165+
},
166+
};
94167
}
95168

96169
it("removes MJPEG requests while hidden and reconnects when shown", async () => {
@@ -135,3 +208,25 @@ it("starts exactly one new stream per Reconnect press", async () => {
135208
await act(async () => renderer!.root.findByType("button").props.onClick());
136209
expect(primes).toBe(2);
137210
});
211+
212+
it("keeps the 3D viewer mounted across iOS video recovery and releases it when hidden", async () => {
213+
const { configure, frame, view } = await setup(true);
214+
await act(async () => configure());
215+
expect(viewerMounts).toBe(0);
216+
await act(async () => frame());
217+
expect(viewerMounts).toBe(1);
218+
await act(async () => {
219+
await vi.advanceTimersByTimeAsync(15_000);
220+
});
221+
expect(viewerUnmounts).toBe(0);
222+
expect(viewerMounts).toBe(1);
223+
await act(async () => {
224+
await vi.advanceTimersByTimeAsync(1_000);
225+
configure();
226+
});
227+
await act(async () => frame());
228+
expect(viewerMounts).toBe(1);
229+
await act(async () => renderer!.update(view(false)));
230+
expect(viewerUnmounts).toBe(1);
231+
expect(vi.getTimerCount()).toBe(0);
232+
});

‎apps/web/src/components/device/DeviceStreamView.tsx‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ export function DeviceStreamView(props: {
9494
const canvasRef = useRef<HTMLCanvasElement | null>(null);
9595
const clientRef = useRef<DeviceStreamClient | null>(null);
9696
const [status, setStatus] = useState<DeviceStreamStatus>("connecting");
97+
const [hasFrame, setHasFrame] = useState(false);
9798
const [detail, setDetail] = useState<string | undefined>(undefined);
9899
const [showRestartNotice, setShowRestartNotice] = useState(false);
99100
const [screen, setScreen] = useState<DeviceScreenSize | null>(null);
@@ -123,6 +124,7 @@ export function DeviceStreamView(props: {
123124
onDuoUnavailable: onPhoneUnavailable,
124125
onStatus: (next, nextDetail) => {
125126
setStatus(next);
127+
if (next === "streaming") setHasFrame(true);
126128
setDetail(nextDetail);
127129
if (next !== "connecting") setShowRestartNotice(false);
128130
},
@@ -150,6 +152,8 @@ export function DeviceStreamView(props: {
150152
},
151153
);
152154
clientRef.current = client;
155+
setPhoneUnavailable(false);
156+
setHasFrame(false);
153157
setMjpegUrl(null);
154158
setInputState({ connected: false });
155159
client.start();
@@ -188,27 +192,24 @@ export function DeviceStreamView(props: {
188192
return w / h;
189193
}, [props.platform, screen]);
190194

191-
// Android restarts its encoder when a fold changes the framebuffer size.
192-
// Keep the last decoded frame and viewer mounted while the next keyframe arrives.
193-
const retainingAndroidFrame =
194-
props.platform === "android" &&
195-
status === "connecting" &&
196-
inputState.connected &&
197-
screen !== null;
195+
// Encoder restarts and video reconnects retain the decoded frame and viewer
196+
// while input is still connected and the next keyframe is on its way.
197+
const retainingFrame =
198+
status === "connecting" && hasFrame && inputState.connected && screen !== null;
198199
const showPhone =
199200
props.allowPhoneView &&
200-
(status === "streaming" || retainingAndroidFrame) &&
201+
(status === "streaming" || retainingFrame) &&
201202
props.visible &&
202203
presentation === "phone" &&
203204
!phoneUnavailable &&
204205
!mjpegUrl &&
205206
!props.axOverlay &&
206207
(!isDuo || screen?.supportsHingeAngle === true);
207208
useEffect(() => {
208-
if (!retainingAndroidFrame || !showPhone) return;
209+
if (!retainingFrame || !showPhone) return;
209210
const timeout = window.setTimeout(() => setShowRestartNotice(true), 2_000);
210211
return () => window.clearTimeout(timeout);
211-
}, [retainingAndroidFrame, showPhone]);
212+
}, [retainingFrame, showPhone]);
212213
const controlsInset = props.renderControls && !showPhone ? CONTROLS_RAIL_WIDTH : 0;
213214

214215
// The frame is the largest box at `aspect` that fits the container, so a
@@ -532,14 +533,14 @@ export function DeviceStreamView(props: {
532533
</span>
533534
</div>
534535
) : null}
535-
{retainingAndroidFrame && showPhone && showRestartNotice ? (
536+
{retainingFrame && showPhone && showRestartNotice ? (
536537
<div className="pointer-events-none absolute inset-x-0 bottom-0 flex justify-center p-2">
537538
<span className="rounded-md bg-background/85 px-2 py-1 text-xs text-muted-foreground">
538539
Waiting for device video…
539540
</span>
540541
</div>
541542
) : null}
542-
{status !== "streaming" && !(retainingAndroidFrame && showPhone) ? (
543+
{status !== "streaming" && !(retainingFrame && showPhone) ? (
543544
<div className="absolute inset-0">
544545
<DeviceLoadingView
545546
name={props.deviceName ?? "Device"}
@@ -556,6 +557,7 @@ export function DeviceStreamView(props: {
556557
// An expired ticket surfaces as unauthorized on restart and
557558
// refreshes access through the effect; no need to mint one here.
558559
clientRef.current?.stop();
560+
setPhoneUnavailable(false);
559561
clientRef.current?.start();
560562
}}
561563
>

0 commit comments

Comments
 (0)