Skip to content

Commit ca049dc

Browse files
authored
fix(mobile): restore low-latency decoding for simulator streams (pingdotgg#22733)
1 parent 8f66e67 commit ca049dc

5 files changed

Lines changed: 306 additions & 5 deletions

File tree

‎docs/internals/devices.md‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -105,10 +105,14 @@ binary WebSocket; Android multiplexes SEMU-framed H.264 and JSON gestures over
105105
one WebSocket. [`deviceStream.ts`](../../apps/web/src/components/device/deviceStream.ts)
106106
speaks both so one panel covers both platforms.
107107

108-
Simulators encode H.264 High 5.1. Hardware decoders on some machines and all
109-
headless browsers reject that profile, and WebCodecs is secure-context only, so
110-
the viewer probes `isConfigSupported` and falls back to the MJPEG endpoint on
111-
iOS. Android has no MJPEG; there the panel reports that it cannot decode.
108+
Simulators encode H.264 High 5.1 without frame reordering, but the hub's fallback
109+
encoder can omit the SPS VUI that declares its reorder limits. Chromium's hardware
110+
decoder then retains frames for seconds despite `optimizeForLatency` and an empty
111+
`decodeQueueSize`. The iOS AVCC viewer restores the missing metadata before
112+
configuring the decoder so hardware decoding can remain enabled. WebCodecs is
113+
secure-context only, so the viewer probes `isConfigSupported` and falls back to
114+
MJPEG on iOS when decoding is unavailable. Android has no MJPEG; there the panel
115+
reports that it cannot decode.
112116

113117
serve-sim can also be locked to WebRTC, as on EAS Simulator, and then refuses
114118
AVCC with a 409 naming the transport. The viewer still asks for AVCC first, so
Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
import { describe, expect, it } from "vite-plus/test";
2+
3+
import { simulatorAvcConfig } from "./avcConfig.ts";
4+
5+
// Captured from the hub before and after its encoder falls back from low-latency rate control.
6+
const lowLatency = Uint8Array.from([
7+
1, 100, 0, 51, 255, 225, 0, 22, 39, 100, 0, 51, 172, 19, 20, 60, 4, 192, 20, 158, 106, 154, 129,
8+
1, 1, 3, 194, 1, 8, 248, 1, 0, 4, 40, 238, 60, 176,
9+
]);
10+
const fallback = Uint8Array.from([
11+
1, 100, 0, 51, 255, 225, 0, 12, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 164, 1, 0, 4, 40,
12+
238, 60, 176,
13+
]);
14+
const repaired = Uint8Array.from([
15+
1, 100, 0, 51, 255, 225, 0, 17, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 168, 6, 208, 68, 35,
16+
80, 1, 0, 4, 40, 238, 60, 176,
17+
]);
18+
19+
describe("simulator AVC reorder metadata", () => {
20+
it("adds zero reordering and one reference frame to the captured fallback SPS", () => {
21+
expect(simulatorAvcConfig(fallback)).toEqual(repaired);
22+
expect(simulatorAvcConfig(repaired)).toBe(repaired);
23+
});
24+
25+
it("preserves existing VUI and leaves the original codec description untouched", () => {
26+
const original = fallback.slice();
27+
simulatorAvcConfig(fallback);
28+
expect(fallback).toEqual(original);
29+
expect(simulatorAvcConfig(lowLatency)).toBe(lowLatency);
30+
});
31+
32+
it("repairs multiple SPS entries while preserving PPS and profile extensions", () => {
33+
const extensions = [0xfd, 0xf8, 0xf8, 0];
34+
const description = Uint8Array.from([
35+
...fallback.subarray(0, 5),
36+
0xe2,
37+
...fallback.subarray(6, 20),
38+
...lowLatency.subarray(6, 30),
39+
...fallback.subarray(20),
40+
...extensions,
41+
]);
42+
expect(simulatorAvcConfig(description)).toEqual(
43+
Uint8Array.from([
44+
...description.subarray(0, 6),
45+
...repaired.subarray(6, 25),
46+
...lowLatency.subarray(6, 30),
47+
...fallback.subarray(20),
48+
...extensions,
49+
]),
50+
);
51+
});
52+
53+
it("passes through every truncated prefix of the captured configurations", () => {
54+
for (const config of [fallback, lowLatency]) {
55+
for (let length = 0; length < config.length; length++) {
56+
const truncated = config.subarray(0, length);
57+
expect(simulatorAvcConfig(truncated)).toBe(truncated);
58+
}
59+
}
60+
});
61+
62+
it("does not modify unsupported profiles, scaling lists, or invalid parameter-set lengths", () => {
63+
const unsupportedProfile = fallback.slice();
64+
unsupportedProfile[9] = 110;
65+
const scalingLists = fallback.slice();
66+
scalingLists[12] = scalingLists[12]! | 1;
67+
const invalidSpsLength = fallback.slice();
68+
invalidSpsLength[6] = 0xff;
69+
const invalidPpsLength = fallback.slice();
70+
invalidPpsLength[21] = 0xff;
71+
for (const config of [unsupportedProfile, scalingLists, invalidSpsLength, invalidPpsLength]) {
72+
expect(simulatorAvcConfig(config)).toBe(config);
73+
}
74+
});
75+
76+
it("requires valid SPS trailing bits before appending VUI metadata", () => {
77+
const invalidPadding = fallback.slice();
78+
invalidPadding[19] = 0;
79+
expect(simulatorAvcConfig(invalidPadding)).toBe(invalidPadding);
80+
});
81+
82+
it("passes through invalid NAL headers and unterminated Exp-Golomb values", () => {
83+
const forbiddenBit = fallback.slice();
84+
forbiddenBit[8] = forbiddenBit[8]! | 0x80;
85+
expect(simulatorAvcConfig(forbiddenBit)).toBe(forbiddenBit);
86+
const unterminated = fallback.slice();
87+
unterminated.fill(0, 12, 20);
88+
expect(simulatorAvcConfig(unterminated)).toBe(unterminated);
89+
const overlong = Uint8Array.from([
90+
...fallback.subarray(0, 6),
91+
0,
92+
13,
93+
39,
94+
100,
95+
0,
96+
51,
97+
0,
98+
0,
99+
3,
100+
0,
101+
0,
102+
3,
103+
0,
104+
0,
105+
0,
106+
...fallback.subarray(20),
107+
]);
108+
expect(simulatorAvcConfig(overlong)).toBe(overlong);
109+
});
110+
111+
it("bounds parsing of oversized configurations and SPS data", () => {
112+
const oversizedConfig = new Uint8Array(64 * 1024 + 1);
113+
oversizedConfig.set(fallback);
114+
expect(simulatorAvcConfig(oversizedConfig)).toBe(oversizedConfig);
115+
const oversizedSps = new Uint8Array(6 + 2 + 4097 + 7);
116+
oversizedSps.set(fallback.subarray(0, 6));
117+
oversizedSps.set([0x10, 1], 6);
118+
oversizedSps.set(fallback.subarray(8, 20), 8);
119+
oversizedSps.set(fallback.subarray(20), 8 + 4097);
120+
expect(simulatorAvcConfig(oversizedSps)).toBe(oversizedSps);
121+
});
122+
});
Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
const MAX_CONFIG_BYTES = 64 * 1024;
2+
const MAX_SPS_BYTES = 4096;
3+
4+
const unsignedGolomb = (value: number) => {
5+
const bits = (value + 1).toString(2);
6+
return "0".repeat(bits.length - 1) + bits;
7+
};
8+
9+
function withReorderLimits(sps: Uint8Array): Uint8Array {
10+
if (sps.length < 4 || sps.length > MAX_SPS_BYTES || (sps[0]! & 0x9f) !== 7) return sps;
11+
const rbsp: number[] = [];
12+
let zeros = 0;
13+
for (let index = 1; index < sps.length; index++) {
14+
const byte = sps[index]!;
15+
if (zeros >= 2 && byte === 3) {
16+
if (index + 1 >= sps.length || sps[index + 1]! > 3) return sps;
17+
zeros = 0;
18+
continue;
19+
}
20+
rbsp.push(byte);
21+
zeros = byte === 0 ? zeros + 1 : 0;
22+
}
23+
const bits = rbsp.map((byte) => byte.toString(2).padStart(8, "0")).join("");
24+
let position = 0;
25+
const read = (count: number) => {
26+
if (position + count > bits.length) throw new Error("Truncated H.264 SPS");
27+
const value = Number.parseInt(bits.slice(position, position + count), 2);
28+
position += count;
29+
return value;
30+
};
31+
const readUnsignedGolomb = () => {
32+
let leadingZeros = 0;
33+
while (read(1) === 0) {
34+
if (++leadingZeros > 31) throw new Error("Invalid H.264 Exp-Golomb value");
35+
}
36+
return 2 ** leadingZeros - 1 + (leadingZeros === 0 ? 0 : read(leadingZeros));
37+
};
38+
39+
try {
40+
// Only handle the hub's 8-bit 4:2:0 High profile without scaling lists.
41+
// Unknown encoder configurations pass through rather than guessing their layout.
42+
if (read(8) !== 100) return sps;
43+
read(16);
44+
if (readUnsignedGolomb() > 31 || readUnsignedGolomb() !== 1) return sps;
45+
if (readUnsignedGolomb() !== 0 || readUnsignedGolomb() !== 0) return sps;
46+
read(1);
47+
if (read(1) !== 0) return sps;
48+
if (readUnsignedGolomb() > 12) return sps;
49+
const pictureOrderCountType = readUnsignedGolomb();
50+
if (pictureOrderCountType === 0) {
51+
if (readUnsignedGolomb() > 12) return sps;
52+
} else if (pictureOrderCountType === 1) {
53+
read(1);
54+
readUnsignedGolomb();
55+
readUnsignedGolomb();
56+
const cycleLength = readUnsignedGolomb();
57+
if (cycleLength > 255) return sps;
58+
for (let index = 0; index < cycleLength; index++) readUnsignedGolomb();
59+
} else if (pictureOrderCountType !== 2) return sps;
60+
const maxReferenceFrames = readUnsignedGolomb();
61+
if (maxReferenceFrames > 16) return sps;
62+
read(1);
63+
if (readUnsignedGolomb() > 65535 || readUnsignedGolomb() > 65535) return sps;
64+
if (read(1) === 0) read(1);
65+
read(1);
66+
if (read(1) !== 0) {
67+
for (let index = 0; index < 4; index++) readUnsignedGolomb();
68+
}
69+
const vuiPosition = position;
70+
if (read(1) !== 0) return sps;
71+
// With no VUI, only rbsp_stop_one_bit and byte padding may follow.
72+
if (!/^10{0,7}$/.test(bits.slice(position))) return sps;
73+
74+
// serve-sim disables frame reordering. Its fallback encoder omits the VUI,
75+
// causing Chromium to infer a large reorder queue despite optimizeForLatency.
76+
// Preserve the SPS and add VUI defaults with max_num_reorder_frames = 0.
77+
// The flag bits enable VUI, omit its eight optional sections, and enable
78+
// bitstream restrictions and motion_vectors_over_pic_boundaries_flag.
79+
const restricted =
80+
bits.slice(0, vuiPosition) +
81+
"10000000011" +
82+
[2, 1, 16, 16, 0, maxReferenceFrames].map(unsignedGolomb).join("") +
83+
"1";
84+
const padded = restricted.padEnd(Math.ceil(restricted.length / 8) * 8, "0");
85+
const output = [sps[0]!];
86+
zeros = 0;
87+
for (let index = 0; index < padded.length; index += 8) {
88+
const byte = Number.parseInt(padded.slice(index, index + 8), 2);
89+
if (zeros >= 2 && byte <= 3) {
90+
output.push(3);
91+
zeros = 0;
92+
}
93+
output.push(byte);
94+
zeros = byte === 0 ? zeros + 1 : 0;
95+
}
96+
return Uint8Array.from(output);
97+
} catch {
98+
return sps;
99+
}
100+
}
101+
102+
/** Repair missing reorder metadata only for serve-sim's non-reordering AVCC feed. */
103+
export function simulatorAvcConfig(description: Uint8Array): Uint8Array {
104+
if (description.length < 7 || description.length > MAX_CONFIG_BYTES || description[0] !== 1)
105+
return description;
106+
const parameterSets = description[5]! & 0x1f;
107+
if (parameterSets === 0) return description;
108+
const parts: Uint8Array[] = [description.subarray(0, 6)];
109+
let position = 6;
110+
let changed = false;
111+
for (let index = 0; index < parameterSets; index++) {
112+
if (position + 2 > description.length) return description;
113+
const length = description[position]! * 256 + description[position + 1]!;
114+
position += 2;
115+
if (length === 0 || position + length > description.length) return description;
116+
const sps = description.subarray(position, position + length);
117+
const repaired = withReorderLimits(sps);
118+
changed ||= repaired !== sps;
119+
parts.push(Uint8Array.of(repaired.length >>> 8, repaired.length & 0xff), repaired);
120+
position += length;
121+
}
122+
const suffix = position;
123+
if (position >= description.length) return description;
124+
const pictureParameterSets = description[position++]!;
125+
for (let index = 0; index < pictureParameterSets; index++) {
126+
if (position + 2 > description.length) return description;
127+
const length = description[position]! * 256 + description[position + 1]!;
128+
position += 2;
129+
if (length === 0 || position + length > description.length) return description;
130+
position += length;
131+
}
132+
if (!changed) return description;
133+
parts.push(description.subarray(suffix));
134+
const output = new Uint8Array(parts.reduce((length, part) => length + part.length, 0));
135+
position = 0;
136+
for (const part of parts) {
137+
output.set(part, position);
138+
position += part.length;
139+
}
140+
return output;
141+
}

‎packages/client-runtime/src/device/stream.test.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,36 @@ describe("shared device stream readiness and recovery", () => {
323323
vi.unstubAllGlobals();
324324
});
325325

326+
it("repairs the hub's fallback configuration without disabling hardware decoding", async () => {
327+
const { client, videoBodies, Decoder } = recoveryFixture("ios", false);
328+
try {
329+
client.start();
330+
await vi.advanceTimersByTimeAsync(0);
331+
videoBodies[0]!.enqueue(
332+
Uint8Array.from(
333+
envelope(
334+
1,
335+
[
336+
1, 100, 0, 51, 255, 225, 0, 12, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 164, 1,
337+
0, 4, 40, 238, 60, 176,
338+
],
339+
),
340+
),
341+
);
342+
await vi.advanceTimersByTimeAsync(0);
343+
expect(Decoder.isConfigSupported).toHaveBeenCalledExactlyOnceWith({
344+
codec: "avc1.640033",
345+
description: Uint8Array.from([
346+
1, 100, 0, 51, 255, 225, 0, 17, 39, 100, 0, 51, 172, 86, 128, 76, 1, 73, 230, 168, 6, 208,
347+
68, 35, 80, 1, 0, 4, 40, 238, 60, 176,
348+
]),
349+
optimizeForLatency: true,
350+
});
351+
} finally {
352+
client.stop();
353+
}
354+
});
355+
326356
it("does not repaint a delayed JPEG seed over newer decoded video", async () => {
327357
const { client, videoBodies, drawImage, decodedFrame } = recoveryFixture("ios", false);
328358
let resolveBitmap!: (bitmap: ImageBitmap) => void;

‎packages/client-runtime/src/device/stream.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
import * as Schema from "effect/Schema";
1111
import * as Option from "effect/Option";
1212
import { createCanvasFrameSink, type DeviceFrameSink } from "./frame.ts";
13+
import { simulatorAvcConfig } from "./avcConfig.ts";
1314
import { type DeviceHubAccess, withDeviceHubQuery } from "./hubAccess.ts";
1415
import type { DevicePlatform } from "@t3tools/contracts";
1516

@@ -759,7 +760,10 @@ export function createDeviceStreamClient(
759760
case "description": {
760761
awaitingKeyframe = true;
761762
const configured = await configureDecoder(
762-
{ codec: avcCodecString(chunk.payload), description: chunk.payload },
763+
{
764+
codec: avcCodecString(chunk.payload),
765+
description: simulatorAvcConfig(chunk.payload),
766+
},
763767
isCurrent,
764768
);
765769
if (!isCurrent()) return;

0 commit comments

Comments
 (0)