Skip to content

Commit e0b940a

Browse files
fix(core): a sound effect starts on its time instead of a frame late (#4696)
* fix(core): a sound effect starts on its time instead of a frame late * style(core): format the early-start change * test(core): a clip past its end is never started early * fix(core): a seek stops a clip started early, and the lead never passes 40 ms of real time * test(core): drive the early-start tests the way the runtime receives a seek and a speed * fix(core): the early-start lead never passes 40 ms of film time at fast speeds * fix(core): keep a sped-up sound effect inside strict sync when it starts early * test(core): pin the early-start cap at fast speed
1 parent a0c72ad commit e0b940a

4 files changed

Lines changed: 229 additions & 8 deletions

File tree

‎packages/core/src/runtime/init.test.ts‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3177,6 +3177,102 @@ describe("initSandboxRuntimeModular", () => {
31773177
expect(childTimeline.time()).toBeCloseTo(1, 1);
31783178
});
31793179

3180+
const mountLateSfx = () => {
3181+
const raf = createManualRaf();
3182+
vi.spyOn(performance, "now").mockImplementation(() => raf.now());
3183+
window.requestAnimationFrame = raf.requestAnimationFrame as typeof window.requestAnimationFrame;
3184+
window.cancelAnimationFrame = raf.cancelAnimationFrame as typeof window.cancelAnimationFrame;
3185+
const root = document.createElement("div");
3186+
root.setAttribute("data-composition-id", "main");
3187+
root.setAttribute("data-root", "true");
3188+
root.setAttribute("data-start", "0");
3189+
root.setAttribute("data-duration", "4");
3190+
root.setAttribute("data-width", "1920");
3191+
root.setAttribute("data-height", "1080");
3192+
document.body.appendChild(root);
3193+
const sfx = document.createElement("audio");
3194+
sfx.setAttribute("data-start", "1");
3195+
sfx.setAttribute("data-duration", "1");
3196+
sfx.setAttribute("src", "/assets/click.mp3");
3197+
Object.defineProperty(sfx, "paused", { value: true, writable: true, configurable: true });
3198+
Object.defineProperty(sfx, "currentTime", { value: 0, writable: true, configurable: true });
3199+
Object.defineProperty(sfx, "readyState", { value: 4, configurable: true });
3200+
const startedAt: number[] = [];
3201+
sfx.play = vi.fn(() => {
3202+
startedAt.push(window.__player!.getTime());
3203+
Object.assign(sfx, { paused: false });
3204+
return Promise.resolve();
3205+
});
3206+
sfx.pause = vi.fn(() => Object.assign(sfx, { paused: true }));
3207+
root.appendChild(sfx);
3208+
window.__timelines = { main: createMockTimeline(4) };
3209+
initSandboxRuntimeModular();
3210+
return { raf, sfx, startedAt };
3211+
};
3212+
3213+
it("starts an audio clip on the tick before its time, not the tick after", () => {
3214+
const { raf, startedAt } = mountLateSfx();
3215+
window.__player?.play();
3216+
for (let frame = 0; frame < 75; frame++) raf.step(16);
3217+
3218+
expect(startedAt.length).toBeGreaterThan(0);
3219+
expect(startedAt[0]).toBeLessThan(1);
3220+
expect(startedAt[0]).toBeGreaterThanOrEqual(1 - 0.02);
3221+
});
3222+
3223+
const setRate = (playbackRate: number) =>
3224+
window.dispatchEvent(
3225+
new MessageEvent("message", {
3226+
source: window.parent,
3227+
data: { source: "hf-parent", type: "control", action: "set-playback-rate", playbackRate },
3228+
}),
3229+
);
3230+
3231+
it.each([1, 4])(
3232+
"stops a clip it started early when a seek jumps back before it, at %sx",
3233+
(rate) => {
3234+
const { raf, sfx, startedAt } = mountLateSfx();
3235+
setRate(rate);
3236+
window.__player?.play();
3237+
for (let frame = 0; frame < 120 && startedAt.length === 0; frame++) raf.step(13);
3238+
expect(startedAt[0]).toBeLessThan(1);
3239+
3240+
window.__player?.seek(0, { keepPlaying: true });
3241+
raf.step(16);
3242+
3243+
expect(sfx.pause).toHaveBeenCalled();
3244+
},
3245+
);
3246+
3247+
it("does not start a clip early on the tick a seek lands just before it", () => {
3248+
const { raf, startedAt } = mountLateSfx();
3249+
window.__player?.play();
3250+
raf.step(16);
3251+
window.__player?.seek(0.99, { keepPlaying: true }); // lands on the frame at 0.967
3252+
3253+
expect(startedAt).toEqual([]);
3254+
for (let frame = 0; frame < 10 && startedAt.length === 0; frame++) raf.step(16);
3255+
expect(startedAt[0]).toBeGreaterThanOrEqual(1 - 0.02);
3256+
});
3257+
3258+
it("starts a clip at most 40 ms of real time early at a slow speed", () => {
3259+
const { raf, startedAt } = mountLateSfx();
3260+
setRate(0.25);
3261+
window.__player?.play();
3262+
for (let frame = 0; frame < 60 && startedAt.length === 0; frame++) raf.step(76);
3263+
3264+
expect(startedAt[0]).toBeGreaterThanOrEqual(1 - 0.04 * 0.25);
3265+
});
3266+
3267+
it("starts a clip at most 40 ms of film time early at a fast speed", () => {
3268+
const { raf, startedAt } = mountLateSfx();
3269+
setRate(4);
3270+
window.__player?.play();
3271+
for (let frame = 0; frame < 30 && startedAt.length === 0; frame++) raf.step(18);
3272+
3273+
expect(startedAt[0]).toBeGreaterThanOrEqual(1 - 0.04);
3274+
});
3275+
31803276
it.each([24, 30, 60, 30_000 / 1_001])(
31813277
"preserves public playback state across keepPlaying seeks at %s fps",
31823278
(fps) => {

‎packages/core/src/runtime/init.ts‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import {
3030
import { sourceTimeAt } from "../speedRamp";
3131
import { createWaapiAdapter } from "./adapters/waapi";
3232
import {
33+
MEDIA_SYNC_TOLERANCE_SECONDS,
3334
isUnplayable,
3435
readElementPlaybackRate,
3536
readElementPlaybackStart,
@@ -2843,8 +2844,8 @@ export function initSandboxRuntimeModular(): void {
28432844
* The media elements this pass must visit, and the argument that the rest can
28442845
* be skipped.
28452846
*
2846-
* `isActive` requires `isInClipWindow`, so a clip whose window excludes the
2847-
* new time is inactive there no matter what its element state is. A clip that
2847+
* `isActive` requires `isInClipWindow` (or, for audio, a start within the early-start
2848+
* margin), so a clip outside that is inactive there whatever its element state. A clip that
28482849
* is in neither the previous in-window set nor the set of windows whose
28492850
* endpoint the transport just crossed was therefore out of window BEFORE and
28502851
* is out of window NOW — the pass would only have evicted sync state that is
@@ -2858,6 +2859,7 @@ export function initSandboxRuntimeModular(): void {
28582859
const collectMediaElementsToVisit = (
28592860
index: MediaClipIndex,
28602861
toSeconds: number,
2862+
cueAheadSeconds: number,
28612863
): Array<HTMLVideoElement | HTMLAudioElement> => {
28622864
const fromSeconds = lastSyncedMediaTimeSeconds;
28632865
if (fromSeconds === null) return index.clips.map((clip) => clip.el);
@@ -2867,7 +2869,12 @@ export function initSandboxRuntimeModular(): void {
28672869
const hi = Math.max(fromSeconds, toSeconds) + sameInstantMargin;
28682870
const visiting = new Set<HTMLVideoElement | HTMLAudioElement>();
28692871
for (const clip of mediaClipsInWindow) visiting.add(clip.el);
2870-
for (const clip of clipsWithEndpointBetween(index.byStart, (c) => c.start, lo, hi)) {
2872+
for (const clip of clipsWithEndpointBetween(
2873+
index.byStart,
2874+
(c) => c.start,
2875+
lo,
2876+
hi + cueAheadSeconds,
2877+
)) {
28712878
visiting.add(clip.el);
28722879
}
28732880
for (const clip of clipsWithEndpointBetween(index.byEnd, (c) => c.end, lo, hi)) {
@@ -2971,15 +2978,23 @@ export function initSandboxRuntimeModular(): void {
29712978
// stale one cannot be recovered, so it visits all of them exactly as before.
29722979
// Same reasoning, and the same latch, as the duration floors above.
29732980
const indexed = !renderCaptureSeekStarted;
2981+
const cueAheadSeconds =
2982+
state.isPlaying && !state.mediaForceSyncNextTick && lastSyncedMediaTimeSeconds !== null
2983+
? Math.min(
2984+
Math.max(0, state.currentTime - lastSyncedMediaTimeSeconds),
2985+
MEDIA_SYNC_TOLERANCE_SECONDS * Math.min(1, state.playbackRate),
2986+
)
2987+
: 0;
29742988
const mediaClips = withTimingResolver(() => {
29752989
if (!indexed) return buildRuntimeMediaCache().mediaClips;
29762990
const index = resolveMediaClipIndex();
29772991
// Windows come from the index, but every field handed to `syncRuntimeMedia`
29782992
// is re-read here: `el.duration` is reset by any `el.load()` the retry path
29792993
// made last pass, and a cached copy of it would be exactly the stale
29802994
// duration the two-scope rule exists to prevent.
2981-
return buildRuntimeMediaCache(collectMediaElementsToVisit(index, state.currentTime))
2982-
.mediaClips;
2995+
return buildRuntimeMediaCache(
2996+
collectMediaElementsToVisit(index, state.currentTime, cueAheadSeconds),
2997+
).mediaClips;
29832998
});
29842999
// Attach probed volume keyframes to clips so syncRuntimeMedia can use the
29853000
// same envelope the renderer uses instead of tracking GSAP-change diffs.
@@ -3008,6 +3023,7 @@ export function initSandboxRuntimeModular(): void {
30083023
userMuted: state.bridgeMuted,
30093024
userVolume: state.bridgeVolume,
30103025
forceSync,
3026+
cueAheadSeconds,
30113027
getCompositionDuration: () =>
30123028
getSafeTimelineDurationSeconds(state.capturedTimeline, 0, timingRevision),
30133029
onElementVolume: (el, _effectiveVolume, authorVolume) =>
@@ -3025,7 +3041,7 @@ export function initSandboxRuntimeModular(): void {
30253041
// Every clip not visited was out of window at both ends of this seek, so
30263042
// the visited ones are the only possible members.
30273043
mediaClipsInWindow = mediaClips.filter((clip) =>
3028-
isInClipWindow(state.currentTime, clip.start, clip.end),
3044+
isInClipWindow(state.currentTime, clip.start - MEDIA_SYNC_TOLERANCE_SECONDS, clip.end),
30293045
);
30303046
} else {
30313047
lastSyncedMediaTimeSeconds = null;

‎packages/core/src/runtime/media.test.ts‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -874,6 +874,100 @@ describe("syncRuntimeMedia", () => {
874874
expect(clip.el.play).not.toHaveBeenCalled();
875875
});
876876

877+
it("starts an audio clip due within the next tick, from its first sample", () => {
878+
const clip = createMockClip({ start: 5, end: 6, mediaStart: 2 }, "audio");
879+
syncRuntimeMedia({
880+
clips: [clip],
881+
timeSeconds: 4.99,
882+
playing: true,
883+
playbackRate: 1,
884+
cueAheadSeconds: 0.017,
885+
});
886+
expect(clip.el.play).toHaveBeenCalledTimes(1);
887+
expect(clip.el.currentTime).toBe(2);
888+
});
889+
890+
it("does not start an audio clip further away than the next tick", () => {
891+
const clip = createMockClip({ start: 5, end: 6 }, "audio");
892+
syncRuntimeMedia({
893+
clips: [clip],
894+
timeSeconds: 4.95,
895+
playing: true,
896+
playbackRate: 1,
897+
cueAheadSeconds: 0.017,
898+
});
899+
expect(clip.el.play).not.toHaveBeenCalled();
900+
});
901+
902+
it("does not start a sped-up clip so early that its window opens it past strict sync", () => {
903+
const clip = createMockClip({ start: 5, end: 6, rate: 3 }, "audio");
904+
syncRuntimeMedia({
905+
clips: [clip],
906+
timeSeconds: 4.98,
907+
playing: true,
908+
playbackRate: 1,
909+
cueAheadSeconds: 0.034,
910+
});
911+
expect(clip.el.play).not.toHaveBeenCalled();
912+
});
913+
914+
it("keeps a clip it started early playing when the next tick is shorter", () => {
915+
const clip = createMockClip({ start: 5, end: 6 }, "audio");
916+
syncRuntimeMedia({
917+
clips: [clip],
918+
timeSeconds: 4.97,
919+
playing: true,
920+
playbackRate: 1,
921+
cueAheadSeconds: 0.034,
922+
});
923+
Object.defineProperty(clip.el, "paused", {
924+
value: false,
925+
writable: true,
926+
configurable: true,
927+
});
928+
syncRuntimeMedia({
929+
clips: [clip],
930+
timeSeconds: 4.98,
931+
playing: true,
932+
playbackRate: 1,
933+
cueAheadSeconds: 0.01,
934+
});
935+
expect(clip.el.pause).not.toHaveBeenCalled();
936+
});
937+
938+
it("does not start a clip again once the playhead is past it", () => {
939+
const clip = createMockClip({ start: 5, end: 6 }, "audio");
940+
syncRuntimeMedia({
941+
clips: [clip],
942+
timeSeconds: 6.01,
943+
playing: true,
944+
playbackRate: 1,
945+
cueAheadSeconds: 0.017,
946+
});
947+
expect(clip.el.play).not.toHaveBeenCalled();
948+
});
949+
950+
it("starts nothing early while paused, and never a video", () => {
951+
const audio = createMockClip({ start: 5, end: 6 }, "audio");
952+
syncRuntimeMedia({
953+
clips: [audio],
954+
timeSeconds: 4.99,
955+
playing: false,
956+
playbackRate: 1,
957+
cueAheadSeconds: 0.017,
958+
});
959+
const video = createMockClip({ start: 5, end: 6 }, "video");
960+
syncRuntimeMedia({
961+
clips: [video],
962+
timeSeconds: 4.99,
963+
playing: true,
964+
playbackRate: 1,
965+
cueAheadSeconds: 0.017,
966+
});
967+
expect(audio.el.play).not.toHaveBeenCalled();
968+
expect(video.el.play).not.toHaveBeenCalled();
969+
});
970+
877971
it("does not play() an element whose networkState is NO_SOURCE", () => {
878972
const clip = createMockClip({ start: 0, end: 10 });
879973
Object.defineProperty(clip.el, "networkState", { value: 3, configurable: true });

‎packages/core/src/runtime/media.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,7 @@ const seekLoadRetried = new WeakSet<HTMLMediaElement>();
195195
// AbortError / NotAllowedError that should surface. Cleared on the `playing`
196196
// event (actual playback started) or on `pause`/`error` (state ended).
197197
const playRequested = new WeakSet<HTMLMediaElement>();
198+
const startedEarly = new WeakSet<HTMLMediaElement>();
198199
function markPlayRequested(el: HTMLMediaElement): void {
199200
if (playRequested.has(el)) return;
200201
playRequested.add(el);
@@ -253,7 +254,7 @@ export function hasMediaSyncStateForTest(el: HTMLMediaElement): boolean {
253254
export const MEDIA_HARD_SYNC_SECONDS = 0.5;
254255

255256
/** Drift a playing audio element may carry before sync pulls it back onto the playhead. */
256-
const MEDIA_SYNC_TOLERANCE_SECONDS = 0.04;
257+
export const MEDIA_SYNC_TOLERANCE_SECONDS = 0.04;
257258

258259
// A playing video is steered back by rate, not seeked (a seek resets its decoder).
259260
// Its rate is written only when steering starts or stops: every write costs a frame.
@@ -311,6 +312,8 @@ export function syncRuntimeMedia(params: {
311312
* unity; do not mistake that transport write for an authored volume edit. */
312313
isWebAudioRouted?: (el: HTMLMediaElement) => boolean;
313314
forceSync?: boolean;
315+
/** How far the next tick will move the playhead: an audio clip due within it starts now. */
316+
cueAheadSeconds?: number;
314317
/** Lets a video clip that runs to the composition end hold its last frame at the terminal time.
315318
* A thunk, because deriving the duration is only worth it for a clip past its own end. */
316319
getCompositionDuration: () => number;
@@ -322,6 +325,18 @@ export function syncRuntimeMedia(params: {
322325
const clipRate = clip.rate ?? clip.playbackRate;
323326
const isNonLoopVideo = el.tagName === "VIDEO" && !clip.loop;
324327
const inWindow = isInClipWindow(params.timeSeconds, clip.start, clip.end);
328+
const dueIn = clip.start - params.timeSeconds;
329+
const startsEarly =
330+
el.tagName === "AUDIO" &&
331+
!inWindow &&
332+
dueIn > 0 &&
333+
dueIn * Math.max(1, rateAt(clipRate, 0)) <=
334+
Math.max(
335+
params.cueAheadSeconds ?? 0,
336+
startedEarly.has(el) ? MEDIA_SYNC_TOLERANCE_SECONDS : 0,
337+
);
338+
if (startsEarly) startedEarly.add(el);
339+
else startedEarly.delete(el);
325340
// A video that runs to the composition end stays the visible frame at and past it, so it
326341
// is held on the frame it shows at its own end rather than left on a stale one.
327342
const isTerminalVideo =
@@ -355,7 +370,7 @@ export function syncRuntimeMedia(params: {
355370
// video additionally remains an active visual through
356371
// its authored window, with tail seeks clamped to the final frame.
357372
const isActive =
358-
(inWindow || isTerminalVideo) &&
373+
(inWindow || isTerminalVideo || startsEarly) &&
359374
relTime >= 0 &&
360375
(!el.ended || clip.loop || isHeldVideoTail || canSeekEndedMediaBackward);
361376
if (isActive) {

0 commit comments

Comments
 (0)