Skip to content

Commit 550d9a0

Browse files
fix(devices): refresh backlogged iOS video and require input admission
1 parent ac2cc78 commit 550d9a0

6 files changed

Lines changed: 209 additions & 37 deletions

File tree

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import { expect, it } from "@effect/vitest";
2+
import { NodeWS } from "@effect/platform-node/NodeSocket";
3+
import * as Context from "effect/Context";
4+
import * as Effect from "effect/Effect";
5+
import * as Layer from "effect/Layer";
6+
import { HttpRouter, HttpServer } from "effect/http";
7+
import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer";
8+
import {
9+
AuthOrchestrationOperateScope,
10+
AuthSessionId,
11+
LOCAL_DEVICE_HOST_ID,
12+
} from "@t3tools/contracts";
13+
import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
14+
import * as DeviceService from "./DeviceService.ts";
15+
import * as DeviceHubProxy from "./DeviceHubProxy.ts";
16+
17+
it.effect("preserves binary input and admission frames through the real socket proxy", () =>
18+
Effect.gen(function* () {
19+
const hub = yield* Effect.acquireRelease(
20+
Effect.sync(() => new NodeWS.WebSocketServer({ port: 0 })),
21+
(server) =>
22+
Effect.promise(
23+
() =>
24+
new Promise<void>((resolve) => {
25+
for (const client of server.clients) client.terminate();
26+
server.close(() => resolve());
27+
}),
28+
),
29+
);
30+
if (!hub.address())
31+
yield* Effect.promise(() => new Promise<void>((resolve) => hub.once("listening", resolve)));
32+
const address = hub.address();
33+
if (typeof address !== "object" || !address) throw new Error("Missing hub port");
34+
hub.on("connection", (socket) => {
35+
socket.send(Uint8Array.of(0x83));
36+
socket.on("message", (data, binary) => socket.send(data, { binary }));
37+
});
38+
const services = yield* Layer.build(
39+
HttpRouter.serve(DeviceHubProxy.layer, { disableListenLog: true }).pipe(
40+
Layer.provideMerge(NodeHttpServer.layerTest),
41+
Layer.provide(
42+
Layer.succeed(EnvironmentAuth.EnvironmentAuth, {
43+
authenticateWebSocketUpgrade: () =>
44+
Effect.succeed({
45+
sessionId: AuthSessionId.make("test"),
46+
subject: "test",
47+
method: "bearer-access-token",
48+
scopes: [AuthOrchestrationOperateScope],
49+
}),
50+
} as unknown as EnvironmentAuth.EnvironmentAuth["Service"]),
51+
),
52+
Layer.provide(
53+
Layer.succeed(DeviceService.DeviceService, {
54+
currentReadiness: () =>
55+
Effect.succeed({
56+
hostId: LOCAL_DEVICE_HOST_ID,
57+
hub: { origin: `http://127.0.0.1:${address.port}` },
58+
}),
59+
} as DeviceService.DeviceService["Service"]),
60+
),
61+
),
62+
);
63+
const origin = HttpServer.formatAddress(
64+
Context.get(services, HttpServer.HttpServer).address,
65+
).replace(/^http/, "ws");
66+
const client = yield* Effect.acquireRelease(
67+
Effect.sync(
68+
() =>
69+
new WebSocket(
70+
`${origin}/api/device-hub/vendor/serve-sim/helper/ws?device=test&wsTicket=test`,
71+
),
72+
),
73+
(socket) => Effect.sync(() => socket.close()),
74+
);
75+
client.binaryType = "arraybuffer";
76+
const frames: number[][] = [];
77+
const received = Promise.withResolvers<void>();
78+
client.addEventListener("error", () => received.reject(new Error("Proxy failed")));
79+
client.addEventListener("message", ({ data }) => {
80+
frames.push(Array.from(new Uint8Array(data as ArrayBuffer)));
81+
if (frames.length === 1) {
82+
client.send(Uint8Array.of(0x03, 1));
83+
client.send(Uint8Array.of(0x03, 2));
84+
} else if (frames.length === 3) received.resolve();
85+
});
86+
yield* Effect.promise(() => received.promise);
87+
expect(frames).toEqual([[0x83], [0x03, 1], [0x03, 2]]);
88+
}),
89+
);
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
2-
"agent-device": "0.21.24",
2+
"agent-device": "0.21.25",
33
"expo-device-hub": "0.15.3"
44
}

‎apps/web/src/components/device/DeviceStreamView.test.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ async function setup(h264 = false) {
157157
const packet = new Uint8Array(1 + json.length);
158158
packet[0] = 0x82;
159159
packet.set(json, 1);
160+
sockets[0]?.onmessage?.({ data: Uint8Array.of(0x83).buffer });
160161
sockets[0]?.onmessage?.({ data: packet.buffer });
161162
videoBody?.enqueue(new Uint8Array([0, 0, 0, 5, 1, 1, 0x64, 0, 0x1f]));
162163
},

‎packages/client-runtime/src/device/duoStream.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ it("switches to fixed authenticated feeds while retaining one HID socket, routes
111111
client.start();
112112
await socketConstructed;
113113
const ws = Socket.instances[0]!;
114+
ws.onmessage?.({ data: Uint8Array.of(0x83).buffer });
114115
const config = (
115116
id: number,
116117
orientation: DeviceScreenSize["orientation"] = "portrait",

‎packages/client-runtime/src/device/stream.test.ts‎

Lines changed: 87 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ describe("native device stream transport", () => {
6464
readyState = 1;
6565
binaryType = "";
6666
onopen: (() => void) | null = null;
67+
onmessage: ((event: { data: ArrayBuffer }) => void) | null = null;
6768
onclose: ((event: { code: number; reason: string }) => void) | null = null;
6869
send = vi.fn();
6970
close = vi.fn();
@@ -111,10 +112,12 @@ describe("native device stream transport", () => {
111112
expect(imageUrl.searchParams.get("hostId")).toBe("ssh-host");
112113
expect(fetch).toHaveBeenCalledTimes(1);
113114
socket.onopen?.();
115+
expect(socket.send).not.toHaveBeenCalled();
116+
socket.onmessage?.({ data: Uint8Array.of(0x83).buffer });
114117
client.setOrientation("landscape_right");
115118
client.pressButton("home");
116119
client.sendTouch("begin", 0.25, 0.75);
117-
const messages = socket.send.mock.calls.slice(1).map(([bytes]) => {
120+
const messages = socket.send.mock.calls.map(([bytes]) => {
118121
const packet = bytes as Uint8Array;
119122
return {
120123
tag: packet[0],
@@ -179,6 +182,7 @@ describe("native device stream transport", () => {
179182
client.start();
180183
const replacement = await reopened;
181184
replacement.onopen?.();
185+
if (platform === "ios") replacement.onmessage?.({ data: Uint8Array.of(0x83).buffer });
182186
replacement.send.mockClear();
183187
events.onInputConnected.mockClear();
184188

@@ -336,6 +340,7 @@ describe("iOS input startup", () => {
336340
packet[0] = 0x82;
337341
packet.set(json, 1);
338342
socket.onmessage?.({ data: packet.buffer });
343+
socket.onmessage?.({ data: Uint8Array.of(0x83).buffer });
339344
client.sendTouch("begin", 0.2, 0.7);
340345
const sent = socket.send.mock.calls[0]![0] as Uint8Array;
341346
expect(JSON.parse(new TextDecoder().decode(sent.subarray(1)))).toEqual({
@@ -519,27 +524,100 @@ describe("shared device stream readiness and recovery", () => {
519524
expect(vi.getTimerCount()).toBe(0);
520525
});
521526

522-
it("drops an iOS decode backlog and resumes at a keyframe without switching to MJPEG", async () => {
523-
const { client, videoBodies, decoders, events, decodedFrame } = recoveryFixture("ios", false);
527+
it("requires admission on each iOS socket and does not mistake config for admission", async () => {
528+
const { client, sockets, events, imageReady } = recoveryFixture();
524529
client.start();
525530
await vi.advanceTimersByTimeAsync(0);
531+
imageReady();
532+
const config = (socket: (typeof sockets)[number], unavailable: boolean) => {
533+
const json = new TextEncoder().encode(
534+
JSON.stringify({
535+
width: 400,
536+
height: 800,
537+
orientation: "portrait",
538+
inputUnavailable: unavailable,
539+
supportsHingeAngle: true,
540+
}),
541+
);
542+
socket.onmessage?.({ data: Uint8Array.of(0x82, ...json).buffer });
543+
};
544+
const controls = () => {
545+
client.sendTouch("begin", 0.2, 0.3);
546+
client.pressButton("home");
547+
client.sendKey({ code: "KeyA" } as KeyboardEvent, "down");
548+
client.setOrientation("landscape_left");
549+
client.controlDuo({ control: "angle", value: 90 });
550+
};
551+
const first = sockets[0]!;
552+
config(first, false);
553+
controls();
554+
expect(first.send).not.toHaveBeenCalled();
555+
expect(events.onInputConnected).not.toHaveBeenCalledWith(true, undefined);
556+
config(first, true);
557+
first.onmessage?.({ data: Uint8Array.of(0x83).buffer });
558+
controls();
559+
expect(first.send).not.toHaveBeenCalled();
560+
expect(events.onInputConnected).toHaveBeenLastCalledWith(false);
561+
config(first, false);
562+
client.pressButton("home");
563+
expect(first.send).toHaveBeenCalled();
564+
first.onclose?.({ code: 1013, reason: "retry" });
565+
await vi.advanceTimersByTimeAsync(1_000);
566+
const replacement = sockets[1]!;
567+
config(replacement, false);
568+
controls();
569+
expect(replacement.send).not.toHaveBeenCalled();
570+
first.onmessage?.({ data: Uint8Array.of(0x83).buffer });
571+
expect(events.onInputConnected).toHaveBeenLastCalledWith(false, undefined);
572+
replacement.onmessage?.({ data: Uint8Array.of(0x83).buffer });
573+
client.pressButton("home");
574+
expect(replacement.send).toHaveBeenCalledOnce();
575+
client.stop();
576+
expect(vi.getTimerCount()).toBe(0);
577+
});
578+
579+
it("reopens a backlogged iOS feed for a fresh keyframe while keeping input connected", async () => {
580+
const { client, videoBodies, decoders, events, decodedFrame, signals, sockets, drawImage } =
581+
recoveryFixture("ios", false);
582+
client.start();
583+
await vi.advanceTimersByTimeAsync(0);
584+
const socket = sockets[0]!;
585+
socket.onmessage?.({ data: Uint8Array.of(0x83).buffer });
526586
const body = videoBodies[0]!;
527587
body.enqueue(new Uint8Array([...envelope(1, [1, 0x64, 0, 0x1f]), ...envelope(2, [1])]));
528588
await vi.advanceTimersByTimeAsync(0);
529589
decodedFrame();
530590
const decoder = decoders[0]!;
531-
expect(decoder.decode).toHaveBeenCalledOnce();
532591
decoder.decodeQueueSize = 9;
533592
body.enqueue(new Uint8Array([...envelope(3, [2]), ...envelope(3, [3])]));
534593
await vi.advanceTimersByTimeAsync(0);
535-
expect(decoder.reset).toHaveBeenCalledOnce();
536-
expect(decoder.decode).toHaveBeenCalledOnce();
537-
body.enqueue(new Uint8Array([...envelope(2, [4]), ...envelope(3, [5])]));
594+
expect(signals[1]!.aborted).toBe(true);
595+
expect(decoder.state).toBe("closed");
596+
expect(events.onStatus).toHaveBeenLastCalledWith(
597+
"connecting",
598+
expect.stringContaining("fell behind"),
599+
);
600+
expect(socket.close).not.toHaveBeenCalled();
601+
client.sendTouch("begin", 0.2, 0.3);
602+
expect(socket.send).toHaveBeenCalledOnce();
603+
await vi.advanceTimersByTimeAsync(1_000);
604+
expect(videoBodies).toHaveLength(2);
605+
// The old feed never produces another IDR. Only a fresh subscription can recover.
606+
videoBodies[1]!.enqueue(
607+
new Uint8Array([...envelope(1, [1, 0x64, 0, 0x1f]), ...envelope(2, [4])]),
608+
);
538609
await vi.advanceTimersByTimeAsync(0);
539-
expect(decoder.decode).toHaveBeenCalledTimes(3);
610+
expect(decoders[1]!.decode).toHaveBeenCalledOnce();
611+
drawImage.mockClear();
612+
decodedFrame(0);
613+
expect(drawImage).not.toHaveBeenCalled();
614+
decodedFrame();
615+
expect(drawImage).toHaveBeenCalledOnce();
540616
expect(events.onMjpegFallback).not.toHaveBeenCalled();
541617
expect(events.onStatus).toHaveBeenLastCalledWith("streaming", undefined);
618+
expect(events.onInputConnected).toHaveBeenLastCalledWith(true);
542619
client.stop();
620+
expect(vi.getTimerCount()).toBe(0);
543621
});
544622

545623
it("does not repaint a delayed JPEG seed over newer decoded video", async () => {
@@ -614,7 +692,7 @@ describe("shared device stream readiness and recovery", () => {
614692
expect(sockets[0]!.close).toHaveBeenCalledOnce();
615693
expect(image.src).toBe("");
616694
client.pressButton("home");
617-
expect(sockets[0]!.send).toHaveBeenCalledTimes(1);
695+
expect(sockets[0]!.send).not.toHaveBeenCalled();
618696
expect(vi.getTimerCount()).toBe(0);
619697
});
620698

0 commit comments

Comments
 (0)