Skip to content

Commit b3c5207

Browse files
fix(core): music keeps playing smoothly across video cuts in preview (#4939)
* fix(core): music keeps playing smoothly across video cuts in preview While no Web Audio buffer plays, the preview playhead follows an audio element. It followed the first one in page order, so a sub-composition whose shots each carry a short sound clip took the playhead away from the music bed. At every cut the next shot's sound starts late, the playhead waited for it, the music played on, and media sync then pulled the music back: a jump at each cut. The followed clip still keeps the playhead; otherwise the in-window clip that runs longest takes it. * refactor(core): check the followed clip first and drop the clock pick's flag * fix(core): an ended clip never leads the playhead, and the followed clip keeps it A bed whose file ends before its authored length no longer wins the pick and leaves the playhead on wall time. Pins the rule that a longer clip starting late cannot take the playhead from the followed one.
1 parent 5632356 commit b3c5207

2 files changed

Lines changed: 127 additions & 30 deletions

File tree

‎packages/core/src/runtime/audioClockSource.test.ts‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,100 @@ describe("the audio the playhead follows", () => {
167167
expect(lastTime).toBeGreaterThan(1.4);
168168
});
169169

170+
it("follows the music across cuts between short clips earlier in the page", async () => {
171+
mount(
172+
`<audio id="shot1" data-start="0" data-duration="1" src="/assets/a.mp4"></audio>` +
173+
`<audio id="shot2" data-start="1" data-duration="1" src="/assets/b.mp4"></audio>` +
174+
`<audio id="shot3" data-start="2" data-duration="1" src="/assets/c.mp4"></audio>` +
175+
`<audio id="music" data-start="0" data-duration="10" src="/assets/music.wav"></audio>`,
176+
);
177+
const shots = ["shot1", "shot2", "shot3"].map(
178+
(id) => document.getElementById(id) as HTMLAudioElement,
179+
);
180+
const music = document.getElementById("music") as HTMLAudioElement;
181+
let musicTime = 0;
182+
const musicSeeks: number[] = [];
183+
Object.defineProperty(music, "currentTime", {
184+
configurable: true,
185+
get: () => musicTime,
186+
set: (value: number) => {
187+
musicSeeks.push(value);
188+
musicTime = value;
189+
},
190+
});
191+
initSandboxRuntimeModular();
192+
await flush();
193+
window.__player?.play();
194+
await flush();
195+
const coldStartLagSeconds = 0.25; // how late each shot's sound starts, as a cold clip does at a cut
196+
const playedAt = nowMs;
197+
musicSeeks.length = 0; // Play itself lands every clip on the playhead
198+
Object.assign(music, { paused: false });
199+
for (let frame = 1; frame <= 170; frame++) {
200+
const t = (nowMs + 1000 / 60 - playedAt) / 1000;
201+
musicTime = t;
202+
shots.forEach((shot, i) => {
203+
if (t < i) return;
204+
Object.assign(shot, {
205+
paused: t >= i + 1,
206+
currentTime: Math.max(0, t - i - coldStartLagSeconds),
207+
});
208+
});
209+
stepFrames(1);
210+
expect(window.__player!.getTime()).toBeCloseTo(t, 3);
211+
}
212+
expect(musicSeeks).toEqual([]);
213+
});
214+
215+
it("keeps following a playing voice when a longer bed starts late", async () => {
216+
mount(
217+
`<audio id="vo" data-start="0" data-duration="5" src="/assets/vo.mp3"></audio>` +
218+
`<audio id="bed" data-start="1" data-duration="60" src="/assets/bed.wav"></audio>`,
219+
);
220+
const vo = document.getElementById("vo") as HTMLAudioElement;
221+
const bed = document.getElementById("bed") as HTMLAudioElement;
222+
initSandboxRuntimeModular();
223+
await flush();
224+
window.__player?.play();
225+
await flush();
226+
const playedAt = nowMs;
227+
Object.assign(vo, { paused: false });
228+
for (let frame = 0; frame < 90; frame++) {
229+
stepFrames(1);
230+
const t = (nowMs - playedAt) / 1000;
231+
vo.currentTime = t;
232+
if (t >= 1) Object.assign(bed, { paused: false, currentTime: 0 }); // started, not moving yet
233+
}
234+
expect(window.__player!.getTime()).toBeGreaterThan(1.4);
235+
});
236+
237+
it("follows a playing clip when the longest one ran out of source early", async () => {
238+
mount(
239+
`<audio id="shot" data-start="0" data-duration="3" src="/assets/shot.mp4"></audio>` +
240+
`<audio id="bed" data-start="0" data-duration="10" src="/assets/bed.wav"></audio>`,
241+
);
242+
const shot = document.getElementById("shot") as HTMLAudioElement;
243+
const bed = document.getElementById("bed") as HTMLAudioElement;
244+
initSandboxRuntimeModular();
245+
await flush();
246+
window.__player?.play();
247+
await flush();
248+
const playedAt = nowMs;
249+
Object.assign(shot, { paused: false });
250+
Object.assign(bed, { paused: false });
251+
for (let frame = 0; frame < 150; frame++) {
252+
stepFrames(1);
253+
const t = (nowMs - playedAt) / 1000;
254+
shot.currentTime = t + 0.2; // ahead of the wall clock, so only the shot explains it
255+
bed.currentTime = Math.min(t, 1);
256+
if (t >= 1) {
257+
Object.assign(bed, { paused: true });
258+
Object.defineProperty(bed, "ended", { value: true, configurable: true });
259+
}
260+
}
261+
expect(window.__player!.getTime()).toBeGreaterThan((nowMs - playedAt) / 1000 + 0.1);
262+
});
263+
170264
it("follows the next clip when one earlier in the page failed to load", async () => {
171265
mount(
172266
`<audio id="broken" data-start="0" data-duration="10" src="/assets/missing.mp3"></audio>` +

‎packages/core/src/runtime/init.ts‎

Lines changed: 33 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -4534,6 +4534,26 @@ export function initSandboxRuntimeModular(): void {
45344534
state.transportRafId = window.requestAnimationFrame(transportTick);
45354535
};
45364536

4537+
const followedOrLongestRunningAudio = (
4538+
followed: HTMLMediaElement | null,
4539+
): { el: HTMLMediaElement; start: number } | null => {
4540+
const audioEls = document.querySelectorAll("audio[data-start]");
4541+
let longest: { el: HTMLMediaElement; start: number; runsUntil: number } | null = null;
4542+
for (const el of followed ? [followed, ...audioEls] : audioEls) {
4543+
if (!isMediaElement(el) || !el.isConnected) continue;
4544+
if (isSilencedByHidden(el) || isUnplayable(el) || (el.ended && !el.loop)) continue;
4545+
if (!el.hasAttribute("src") && !el.querySelector("source[src]")) continue;
4546+
const start = resolveAbsoluteMediaStartSeconds(el);
4547+
const durAttr = parseStrictFiniteTimingNumber(el.dataset.duration);
4548+
const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity;
4549+
if (!Number.isFinite(start) || !isInClipWindow(state.currentTime, start, end)) continue;
4550+
if (el === followed) return { el, start };
4551+
const runsUntil = start + (resolveMediaElementDurationSeconds(el) ?? Infinity);
4552+
if (!longest || runsUntil > longest.runsUntil) longest = { el, start, runsUntil };
4553+
}
4554+
return longest;
4555+
};
4556+
45374557
const transportTick = () => {
45384558
if (state.tornDown || inTransportTick) return;
45394559
inTransportTick = true;
@@ -4640,36 +4660,19 @@ export function initSandboxRuntimeModular(): void {
46404660
clock.attachAudioSource({ currentTimeSeconds: webAudioTime });
46414661
}
46424662
} else {
4643-
const audioEls = document.querySelectorAll("audio[data-start]");
4644-
const followed = clock.audioElement();
4645-
let foundActive = false;
4646-
for (const rawEl of followed ? [followed, ...audioEls] : audioEls) {
4647-
if (!isMediaElement(rawEl) || !rawEl.isConnected) continue;
4648-
if (isSilencedByHidden(rawEl) || isUnplayable(rawEl)) continue;
4649-
if (!rawEl.hasAttribute("src") && !rawEl.querySelector("source[src]")) continue;
4650-
const start = resolveAbsoluteMediaStartSeconds(rawEl);
4651-
const durAttr = parseStrictFiniteTimingNumber(rawEl.dataset.duration);
4652-
const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity;
4653-
const mediaStart = readElementPlaybackStart(rawEl);
4654-
if (Number.isFinite(start) && isInClipWindow(state.currentTime, start, end)) {
4655-
if (!rawEl.paused) {
4656-
clock.attachAudioSource({
4657-
el: rawEl,
4658-
compositionStart: start,
4659-
mediaStart,
4660-
rate: readElementRateSpec(rawEl),
4661-
});
4662-
foundActive = true;
4663-
} else if (rawEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {
4664-
// Audio is buffering — freeze visuals at last known position
4665-
// instead of falling through to monotonic (which runs ahead).
4666-
clock.attachAudioSource({ currentTimeSeconds: state.currentTime });
4667-
foundActive = true;
4668-
}
4669-
break;
4670-
}
4671-
}
4672-
if (!foundActive && clock.hasAudioSource()) {
4663+
const source = followedOrLongestRunningAudio(clock.audioElement());
4664+
if (source && !source.el.paused) {
4665+
clock.attachAudioSource({
4666+
el: source.el,
4667+
compositionStart: source.start,
4668+
mediaStart: readElementPlaybackStart(source.el),
4669+
rate: readElementRateSpec(source.el),
4670+
});
4671+
} else if (source && source.el.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {
4672+
// Audio is buffering — freeze visuals at last known position
4673+
// instead of falling through to monotonic (which runs ahead).
4674+
clock.attachAudioSource({ currentTimeSeconds: state.currentTime });
4675+
} else if (clock.hasAudioSource()) {
46734676
clock.detachAudioSource();
46744677
}
46754678
}

0 commit comments

Comments
 (0)