Skip to content

Commit ca4995c

Browse files
committed
fix(core): preview playback moves every frame between an audio track's time updates
1 parent 841e90f commit ca4995c

5 files changed

Lines changed: 228 additions & 3 deletions

File tree

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
2+
import { TransportClock } from "./clock";
3+
import { syncRuntimeMedia, type RuntimeMediaClip } from "./media";
4+
5+
// Chromium moves an element's currentTime in steps; the element plays at whatever rate it is given.
6+
function steppingAudio(stepMs: number) {
7+
const el = document.createElement("audio");
8+
document.body.appendChild(el);
9+
let position = 0;
10+
let rate = 1;
11+
let rateWrites = 0;
12+
let seeks = 0;
13+
Object.defineProperties(el, {
14+
paused: { configurable: true, get: () => false },
15+
seeking: { configurable: true, get: () => false },
16+
readyState: { configurable: true, get: () => 4 },
17+
duration: { configurable: true, get: () => 60 },
18+
playbackRate: {
19+
configurable: true,
20+
get: () => rate,
21+
set: (v: number) => {
22+
rateWrites += 1;
23+
rate = v;
24+
},
25+
},
26+
currentTime: {
27+
configurable: true,
28+
get: () => (Math.floor((position * 1000) / stepMs) * stepMs) / 1000,
29+
set: (v: number) => {
30+
seeks += 1;
31+
position = v;
32+
},
33+
},
34+
});
35+
el.play = vi.fn(() => Promise.resolve());
36+
el.pause = vi.fn();
37+
return {
38+
el,
39+
advance: (dtMs: number) => (position += (dtMs / 1000) * rate),
40+
rateWrites: () => rateWrites,
41+
seeks: () => seeks,
42+
};
43+
}
44+
45+
/** Plays one audio clip the clock follows for `seconds` of `hz` display frames, syncing media as the runtime does. */
46+
function play(stepMs: number, hz: number, seconds = 10) {
47+
let ms = 0;
48+
const audio = steppingAudio(stepMs);
49+
const clock = new TransportClock({ nowMs: () => ms, duration: 60 });
50+
const clip: RuntimeMediaClip = {
51+
el: audio.el,
52+
start: 0,
53+
mediaStart: 0,
54+
duration: 60,
55+
end: 60,
56+
volume: null,
57+
playbackRate: 1,
58+
loop: false,
59+
sourceDuration: null,
60+
};
61+
clock.play();
62+
const tickMs = 1000 / hz;
63+
for (let tick = 1; tick <= seconds * hz; tick++) {
64+
ms = tick * tickMs;
65+
audio.advance(tickMs);
66+
clock.attachAudioSource({ el: audio.el, compositionStart: 0, mediaStart: 0 });
67+
syncRuntimeMedia({
68+
clips: [clip],
69+
timeSeconds: clock.now(),
70+
playing: true,
71+
playbackRate: 1,
72+
clockAhead: (el) => clock.aheadOf(el),
73+
getCompositionDuration: () => 60,
74+
});
75+
}
76+
return { rateWrites: audio.rateWrites(), seeks: audio.seeks(), filmSpeed: clock.now() / seconds };
77+
}
78+
79+
describe("a clock that runs between the steps of the audio it follows, as Chromium reports it", () => {
80+
beforeEach(() => {
81+
vi.spyOn(navigator, "userAgent", "get").mockReturnValue(
82+
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
83+
);
84+
});
85+
86+
afterEach(() => {
87+
vi.restoreAllMocks();
88+
document.body.innerHTML = "";
89+
});
90+
91+
it("does not steer that audio against its own lead, so the film keeps its speed", () => {
92+
for (const [stepMs, hz] of [
93+
[45, 60],
94+
[50, 144],
95+
[60, 60],
96+
[70, 60],
97+
[90, 120],
98+
]) {
99+
const run = play(stepMs, hz);
100+
expect(run.rateWrites, `${stepMs} ms steps at ${hz} Hz: rate writes`).toBe(0);
101+
expect(run.seeks, `${stepMs} ms steps at ${hz} Hz: seeks`).toBe(0);
102+
expect(
103+
Math.abs(run.filmSpeed - 1),
104+
`${stepMs} ms steps at ${hz} Hz: film speed`,
105+
).toBeLessThan(0.001);
106+
}
107+
});
108+
});

‎packages/core/src/runtime/clock.test.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,63 @@ describe("TransportClock", () => {
284284
expect(clock.getSource()).toBe("audio");
285285
});
286286

287+
/**
288+
* Plays 1/60 s ticks against an element whose time moves in `stepMs` steps at `speed`; frames are 30 fps. Frames
289+
* count from the 7th tick: until the element's second update the clock does not know its step.
290+
*/
291+
function playStepping(ticks: number, stepMs: number, speed = 1, stallAfterMs = Infinity) {
292+
const { clock, advance, getMs } = createClock({ duration: 60 });
293+
const el = createMockAudioEl(0, false);
294+
clock.play();
295+
clock.attachAudioSource({ el, compositionStart: 0, mediaStart: 0 });
296+
let last = 0;
297+
let skipped = 0;
298+
let heldTicks = 0;
299+
let longestHold = 0;
300+
let mostAhead = 0;
301+
let back = 0;
302+
let previous = 0;
303+
for (let tick = 1; tick <= ticks; tick++) {
304+
advance(1000 / 60);
305+
const played = Math.min(getMs(), stallAfterMs) * speed;
306+
el.currentTime = (Math.floor(played / stepMs) * stepMs) / 1000;
307+
const now = clock.now();
308+
if (now < previous) back += 1;
309+
previous = now;
310+
mostAhead = Math.max(mostAhead, now - el.currentTime);
311+
const frame = Math.floor(now * 30 + 1e-6);
312+
if (tick > 6 && frame > last + 1) skipped += 1;
313+
heldTicks = frame === last ? heldTicks + 1 : 0;
314+
if (tick > 6) longestHold = Math.max(longestHold, heldTicks);
315+
last = frame;
316+
}
317+
return { skipped, longestHold, mostAhead, back, end: clock.now() };
318+
}
319+
320+
it("moves on every display frame between the element's time updates, never skipping a film frame", () => {
321+
for (const stepMs of [30, 40, 50]) {
322+
const run = playStepping(240, stepMs);
323+
expect(run.skipped, `${stepMs} ms steps: film frames skipped`).toBe(0);
324+
// A 30 fps frame spans two 60 Hz ticks: held for one extra tick, never two.
325+
expect(run.longestHold, `${stepMs} ms steps: ticks a frame is held`).toBeLessThanOrEqual(1);
326+
expect(run.back, `${stepMs} ms steps: reads that went back`).toBe(0);
327+
expect(run.mostAhead, `${stepMs} ms steps: lead over the element`).toBeLessThanOrEqual(
328+
stepMs / 1000 + 1e-9,
329+
);
330+
}
331+
});
332+
333+
it("stops within one of the element's own updates when the element stops moving", () => {
334+
const run = playStepping(120, 40, 1, 1000);
335+
expect(run.end).toBeLessThanOrEqual(1 + 0.04 + 1e-9);
336+
});
337+
338+
it("stays within one update of an element that plays slightly slower than the wall clock", () => {
339+
const run = playStepping(600, 40, 0.98);
340+
expect(run.mostAhead).toBeLessThanOrEqual(0.04 + 1e-9);
341+
expect(run.back).toBe(0);
342+
});
343+
287344
it("names the clock it follows: an audio element, WebAudio, or neither", () => {
288345
const { clock } = createClock({ duration: 10 });
289346
expect(clock.followedClock()).toBe("monotonic");

‎packages/core/src/runtime/clock.ts‎

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,27 @@ const STALL_ADJUSTED_LAG_MS = 33;
2828
const HAVE_FUTURE_DATA = 3;
2929
const AUDIO_START_WAIT_MS = 1500;
3030
const MEDIA_SYNC_STEER_TOLERANCE_SECONDS = 0.04;
31+
const MAX_MEDIA_STEP_SECONDS = 0.25;
32+
const MEDIA_STEPS_KEPT = 8;
3133

3234
type PendingAudioStart = { sinceMs: number; at: number; from: number | null };
35+
type MediaReading = {
36+
el: HTMLMediaElement;
37+
time: number;
38+
steps: number[];
39+
from: number;
40+
fromMs: number;
41+
};
42+
43+
const furthestAhead = (reading: MediaReading | null) =>
44+
reading ? Math.max(0, ...reading.steps) : 0;
45+
46+
function noteStep(reading: MediaReading, time: number): void {
47+
const step = time - reading.time;
48+
if (step > 0 && step <= MAX_MEDIA_STEP_SECONDS)
49+
reading.steps = [...reading.steps.slice(1 - MEDIA_STEPS_KEPT), step];
50+
reading.time = time;
51+
}
3352

3453
export class TransportClock {
3554
private _baseTime = 0;
@@ -45,6 +64,8 @@ export class TransportClock {
4564
private _lastReadMs: number | null = null;
4665
private _lastNow = 0;
4766
private _audioStart: PendingAudioStart | null = null;
67+
private _reading: MediaReading | null = null;
68+
private _ahead: { el: HTMLMediaElement; seconds: number } | null = null;
4869

4970
constructor(opts?: {
5071
initialTime?: number;
@@ -59,6 +80,7 @@ export class TransportClock {
5980
}
6081

6182
now(): number {
83+
this._ahead = null;
6284
if (this._playStartMs === null) return this._baseTime;
6385

6486
// Audio-master: drift is impossible because audio IS the clock. Clearing
@@ -97,12 +119,43 @@ export class TransportClock {
97119
const spec = rate ?? 1;
98120
const time = timeAtSourceTime(spec, el.currentTime - mediaStart) + compositionStart;
99121
const held = this._heldUntilAudioOutputStarts(el.seeking, time);
100-
if (held !== undefined) return held;
101-
if (time >= this._lastNow) return time;
122+
if (held !== undefined) {
123+
this._reading = null;
124+
return held;
125+
}
126+
const between = this._betweenUpdates(el, time);
127+
if (between !== null) return between;
102128
const wrapped = el.loop && this._lastNow - time > timeAtSourceTime(spec, el.duration) / 2;
103129
return !wrapped && el.readyState < HAVE_FUTURE_DATA ? this._lastNow : null;
104130
}
105131

132+
// Chromium moves currentTime in 10-50 ms steps, so reading it as is holds a frame, then jumps. Run on at the play
133+
// rate, kept within [element time, element time + its largest recent step], never back; null when time is behind
134+
// the playhead by more than that step (a late clip, not a step).
135+
private _betweenUpdates(el: HTMLMediaElement, time: number): number | null {
136+
const nowMs = this._nowMs();
137+
const last = this._reading?.el === el ? this._reading : null;
138+
if (time < this._lastNow - furthestAhead(last)) return null;
139+
if (!last) {
140+
this._reading = { el, time, steps: [], from: time, fromMs: nowMs };
141+
return this._runAhead(el, time, Math.max(time, this._lastNow));
142+
}
143+
noteStep(last, time);
144+
const run = last.from + ((nowMs - last.fromMs) / 1000) * this._rate;
145+
const held = Math.min(Math.max(run, time), time + furthestAhead(last));
146+
if (held !== run) Object.assign(last, { from: held, fromMs: nowMs });
147+
return this._runAhead(el, time, Math.max(held, this._lastNow));
148+
}
149+
150+
private _runAhead(el: HTMLMediaElement, time: number, now: number): number {
151+
this._ahead = { el, seconds: now - time };
152+
return now;
153+
}
154+
155+
aheadOf(el: HTMLMediaElement): number {
156+
return this._ahead?.el === el ? this._ahead.seconds : 0;
157+
}
158+
106159
private _heldUntilAudioOutputStarts(seeking: boolean, time: number): number | null | undefined {
107160
const pending = this._pendingAudioStart();
108161
if (seeking) return pending ? this._lastNow : null;
@@ -139,6 +192,7 @@ export class TransportClock {
139192
this._playStartMs = this._nowMs();
140193
this._audioStart = { sinceMs: this._playStartMs, at: this._baseTime, from: null };
141194
this._lastNow = this._baseTime;
195+
this._reading = null;
142196
// Not a stall: the gap since the clock was last read (possibly a long
143197
// paused idle) says nothing about lost playback time, since none was
144198
// playing. `_applyStallCorrection` treats null as "nothing to compare
@@ -151,6 +205,7 @@ export class TransportClock {
151205
if (this._playStartMs === null) return false;
152206
this._baseTime = this.now();
153207
this._playStartMs = null;
208+
this._reading = null;
154209
return true;
155210
}
156211

@@ -160,6 +215,7 @@ export class TransportClock {
160215
: Math.max(0, timeSeconds);
161216
this._baseTime = clamped;
162217
this._lastNow = clamped;
218+
this._reading = null;
163219
if (this._playStartMs !== null) {
164220
this._playStartMs = this._nowMs();
165221
this._audioStart = { sinceMs: this._playStartMs, at: clamped, from: null };
@@ -175,6 +231,7 @@ export class TransportClock {
175231

176232
setRate(rate: number): void {
177233
const safe = Number.isFinite(rate) && rate > 0 ? Math.max(0.1, Math.min(5, rate)) : 1;
234+
this._reading = null;
178235
if (this._playStartMs !== null) {
179236
this._baseTime = this.now();
180237
this._playStartMs = this._nowMs();
@@ -230,6 +287,7 @@ export class TransportClock {
230287
this._lastReadMs = null;
231288
}
232289
this._audioSource = null;
290+
this._reading = null;
233291
}
234292

235293
/** The element the playhead currently follows, if the source is one. */

‎packages/core/src/runtime/init.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3826,6 +3826,7 @@ export function initSandboxRuntimeModular(): void {
38263826
forceSync,
38273827
keepLateMedia,
38283828
cueAheadSeconds,
3829+
clockAhead: (el) => clock.aheadOf(el),
38293830
clockAdvancing:
38303831
clock.getSource() !== "audio" ||
38313832
lastSyncedMediaTimeSeconds === null ||

‎packages/core/src/runtime/media.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,7 @@ export function syncRuntimeMedia(params: {
499499
* outbound message; further invocations are suppressed by the caller.
500500
*/
501501
onAutoplayBlocked?: () => void;
502+
clockAhead?: (el: HTMLMediaElement) => number;
502503
/** `clipGain` is `data-volume` for a routed clip, played ahead of its FX; 1 otherwise. */
503504
onElementVolume?: (
504505
el: HTMLMediaElement,
@@ -557,7 +558,7 @@ export function syncRuntimeMedia(params: {
557558
!inWindow &&
558559
(params.timeSeconds >= clip.end || sameInstant(params.timeSeconds, clip.end)) &&
559560
isClipVisibleAt(params.timeSeconds, clip.start, clip.end, params.getCompositionDuration());
560-
const relTime = mediaSourceTimeAt(clip, params.timeSeconds);
561+
const relTime = mediaSourceTimeAt(clip, params.timeSeconds - (params.clockAhead?.(el) ?? 0));
561562
const isHeldVideoTail =
562563
isTerminalVideo ||
563564
(isNonLoopVideo && clip.sourceDuration != null && relTime >= clip.sourceDuration && inWindow);

0 commit comments

Comments
 (0)