Skip to content

Commit 8519f28

Browse files
fix(core): a preview video short of data holds the film instead of looping snippets (#6974)
* fix(core): a preview video short of data holds the film instead of looping snippets * fix(core): keep the comment share of the runtime files where it was
1 parent 7f48a68 commit 8519f28

3 files changed

Lines changed: 173 additions & 24 deletions

File tree

‎packages/core/src/runtime/init.buffering.test.ts‎

Lines changed: 129 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { afterEach, expect, it, vi } from "vitest";
22
import { initSandboxRuntimeModular } from "./init";
33
import { heldPosters } from "./proxySrc";
4+
import { STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
45
import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
56

67
function audio(id: string, duration: number, readyState: { value: number; bufferedTo?: number }) {
@@ -27,12 +28,13 @@ function audio(id: string, duration: number, readyState: { value: number; buffer
2728
return el;
2829
}
2930

30-
function playPodcast(voiceStart = 0, voiceTag = "audio") {
31+
function playPodcast(voiceStart = 0, voiceTag = "audio", extraHtml = "") {
3132
resetRuntimeFixtureDom();
3233
document.body.innerHTML =
3334
'<div data-composition-id="main" data-root="true" data-duration="10">' +
3435
'<audio id="music" data-start="0" data-duration="10" src="/music.m4a"></audio>' +
35-
`<${voiceTag} id="voice" data-start="${voiceStart}" data-duration="4" src="/voice.m4a"></${voiceTag}>` +
36+
`<${voiceTag} id="voice" data-track-index="0" data-start="${voiceStart}" data-duration="4" src="/voice.m4a"></${voiceTag}>` +
37+
extraHtml +
3638
"</div>";
3739
const voiceReady: { value: number; bufferedTo?: number } = { value: 1 };
3840
const music = audio("music", 10, { value: 4 });
@@ -112,6 +114,9 @@ it("holds on the frame a slow tick skips into a clip with no data, after asking
112114
});
113115

114116
it("resumes for a held clip with data buffered ahead that reports only its current frame, as WebKit's do", () => {
117+
vi.spyOn(navigator, "userAgent", "get").mockReturnValue(
118+
"AppleWebKit/605.1.15 Version/17.4 Safari/605.1.15",
119+
);
115120
const { voiceReady, states, playFor } = playPodcast();
116121

117122
playFor(0.5);
@@ -125,6 +130,128 @@ it("resumes for a held clip with data buffered ahead that reports only its curre
125130
expect(states.at(-1)).toEqual({ isPlaying: true, buffering: false });
126131
});
127132

133+
it("waits for real data ahead: a starved clip reporting a little future data does not resume", () => {
134+
const { voiceReady, states, playFor } = playPodcast();
135+
136+
playFor(0.5);
137+
voiceReady.value = 3;
138+
voiceReady.bufferedTo = 0.2;
139+
playFor(0.5);
140+
expect(states.at(-1)).toEqual({ isPlaying: true, buffering: true });
141+
142+
voiceReady.bufferedTo = 2;
143+
playFor(0.5);
144+
expect(states.at(-1)).toEqual({ isPlaying: true, buffering: false });
145+
});
146+
147+
it("a video a little behind the film is not sought when it starves or when it resumes", () => {
148+
const { voice, voiceReady, playFor } = playPodcast(0, "video");
149+
voiceReady.value = 4;
150+
playFor(1);
151+
const seeks = vi.fn();
152+
let position = window.__player!.getTime() - 0.07;
153+
Object.defineProperty(voice, "currentTime", {
154+
get: () => position,
155+
set: (to: number) => {
156+
seeks(to);
157+
position = to;
158+
},
159+
configurable: true,
160+
});
161+
162+
voiceReady.value = 2;
163+
voiceReady.bufferedTo = position + 0.1;
164+
playFor(0.5);
165+
voiceReady.value = 4;
166+
playFor(0.1);
167+
168+
expect(voice.paused).toBe(false);
169+
expect(seeks).not.toHaveBeenCalled();
170+
});
171+
172+
it("the film waits for a playing video stuck waiting, even with the bytes buffered", () => {
173+
vi.spyOn(navigator, "userAgent", "get").mockReturnValue("Chrome/140.0 AppleWebKit/537.36");
174+
const { voice, voiceReady, states, playFor } = playPodcast(0, "video");
175+
voiceReady.value = 4;
176+
playFor(0.5);
177+
expect(voice.paused).toBe(false);
178+
179+
voiceReady.value = 2;
180+
voiceReady.bufferedTo = 5;
181+
playFor(0.1);
182+
183+
expect(states.at(-1)).toEqual({ isPlaying: true, buffering: true });
184+
});
185+
186+
it("a hold for data does not reload a later clip", () => {
187+
const load = vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => {});
188+
const mark = document.createElement("meta");
189+
mark.name = STUDIO_PREVIEW_MARK_META;
190+
document.head.append(mark);
191+
const { voiceReady, playFor } = playPodcast(
192+
0,
193+
"video",
194+
'<video id="later" data-track-index="0" data-start="9" data-duration="1" src="/later.mp4"></video>',
195+
);
196+
const later = document.getElementById("later") as HTMLVideoElement;
197+
load.mockClear();
198+
voiceReady.value = 4;
199+
playFor(0.5);
200+
const armed = later.preload;
201+
const before = load.mock.contexts.filter((el) => el === later).length;
202+
203+
voiceReady.value = 1;
204+
playFor(0.5);
205+
206+
mark.remove();
207+
expect(armed).toBe("auto");
208+
expect(later.preload).toBe(armed);
209+
expect(load.mock.contexts.filter((el) => el === later).length).toBe(before);
210+
});
211+
212+
it("a film held for data reads as choppy, so the app can lower it", () => {
213+
const reports: Array<{ choppy: boolean; waitingShare: number }> = [];
214+
const { voiceReady, playFor } = playPodcast(0, "video");
215+
voiceReady.value = 4;
216+
playFor(0.5);
217+
const post = vi.mocked(window.parent.postMessage);
218+
post.mockClear();
219+
voiceReady.value = 1;
220+
playFor(6);
221+
for (const [message] of post.mock.calls) {
222+
const m = message as {
223+
type?: string;
224+
health: { choppy: boolean; waitingShare: number } | null;
225+
};
226+
if (m?.type === "preview-health" && m.health) reports.push(m.health);
227+
}
228+
expect(reports.at(-1)?.choppy).toBe(true);
229+
expect(reports.at(-1)?.waitingShare).toBeGreaterThan(0.9);
230+
});
231+
232+
it("a video far behind the film lands on it when it starves", () => {
233+
const { voice, voiceReady, playFor } = playPodcast(0, "video");
234+
voiceReady.value = 4;
235+
playFor(2);
236+
const seeks = vi.fn();
237+
let position = 0.5;
238+
Object.defineProperty(voice, "currentTime", {
239+
get: () => position,
240+
set: (to: number) => {
241+
seeks(to);
242+
position = to;
243+
},
244+
configurable: true,
245+
});
246+
247+
voiceReady.value = 2;
248+
voiceReady.bufferedTo = 0.6;
249+
playFor(0.1);
250+
251+
expect(seeks).toHaveBeenCalled();
252+
expect(position).toBeCloseTo(window.__player!.getTime(), 1);
253+
});
254+
128255
it("a pause while buffering stays paused once the track is ready", () => {
129256
const { music, voiceReady, states, playFor } = playPodcast();
130257

‎packages/core/src/runtime/init.ts‎

Lines changed: 41 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ import { sourceTimeAt } from "../speedRamp";
8484
import { createWaapiAdapter } from "./adapters/waapi";
8585
import {
8686
MEDIA_SYNC_TOLERANCE_SECONDS,
87+
MEDIA_HARD_SYNC_SECONDS,
8788
isUnplayable,
8889
mediaSourceTimeAt,
8990
primeMediaForLaterStarts,
@@ -3102,14 +3103,17 @@ export function initSandboxRuntimeModular(): void {
31023103
startPrefetch(next.el);
31033104
}
31043105
};
3106+
let bufferingFor: HTMLMediaElement[] = [];
3107+
let keepLateMedia = false;
31053108
const preloadNearPlayhead = (
31063109
el: HTMLMediaElement,
31073110
visible: boolean,
31083111
upcoming: boolean,
31093112
time: number,
31103113
forLoop: boolean,
31113114
) => {
3112-
const playing = clock.isPlaying();
3115+
// A hold for data is still playing: fetching other clips now would take bandwidth from the starved one.
3116+
const playing = clock.isPlaying() || bufferingFor.length > 0;
31133117
if (!(visible && playing)) armedFrom.delete(el);
31143118
else if (!armedFrom.has(el)) {
31153119
armedFrom.add(el);
@@ -3440,7 +3444,6 @@ export function initSandboxRuntimeModular(): void {
34403444
let lastSyncedMediaTimeSeconds: number | null = null;
34413445
let mediaClipsInWindow: RuntimeMediaClip[] = [];
34423446
// Clips due now that could not play: every track holds until they can, so none runs on alone.
3443-
let bufferingFor: HTMLMediaElement[] = [];
34443447

34453448
// The one definition of "media the transport drives": timed itself, or hosted
34463449
// by a composition whose timing it inherits. Both the media cache and the
@@ -3724,6 +3727,7 @@ export function initSandboxRuntimeModular(): void {
37243727
userMuted: state.bridgeMuted,
37253728
userVolume: state.bridgeVolume,
37263729
forceSync,
3730+
keepLateMedia,
37273731
cueAheadSeconds,
37283732
clockAdvancing:
37293733
clock.getSource() !== "audio" ||
@@ -4749,18 +4753,20 @@ export function initSandboxRuntimeModular(): void {
47494753
postState(true);
47504754
};
47514755

4752-
const haltPlayback = () => {
4756+
const haltPlayback = (starved = false) => {
47534757
webAudio.stopAll();
47544758
clock.detachAudioSource();
47554759
clock.pause();
47564760
state.isPlaying = false;
47574761
state.currentTime = clock.now();
4758-
state.mediaForceSyncNextTick = true;
4759-
hardSyncAllMedia(state.currentTime);
4762+
state.mediaForceSyncNextTick = !starved;
4763+
keepLateMedia = starved;
4764+
hardSyncAllMedia(state.currentTime, starved ? MEDIA_HARD_SYNC_SECONDS : undefined);
47604765
const tl = state.capturedTimeline;
47614766
pauseTimelineIfPossible(tl);
47624767
runAdapters("pause");
47634768
syncMediaForCurrentState();
4769+
keepLateMedia = false;
47644770
colorGrading.redraw();
47654771
// Paused: back to plain DOM so Studio can select inside the composition.
47664772
void paintCaptureMotionBlur(state.currentTime);
@@ -4770,6 +4776,7 @@ export function initSandboxRuntimeModular(): void {
47704776
const transport: RuntimePlayerTransport = {
47714777
play: () => {
47724778
flushHeldSeek();
4779+
const resumingStarved = bufferingFor.length > 0;
47734780
bufferingFor = [];
47744781
const tl = state.capturedTimeline;
47754782
if (clock.isPlaying()) return;
@@ -4786,8 +4793,9 @@ export function initSandboxRuntimeModular(): void {
47864793
pauseTimelineIfPossible(tl);
47874794
if (!clock.play()) return;
47884795
state.isPlaying = true;
4789-
state.mediaForceSyncNextTick = true;
4790-
hardSyncAllMedia(clock.now());
4796+
state.mediaForceSyncNextTick = !resumingStarved;
4797+
keepLateMedia = resumingStarved;
4798+
hardSyncAllMedia(clock.now(), resumingStarved ? MEDIA_HARD_SYNC_SECONDS : undefined);
47914799
// Schedule audio through WebAudio for sample-accurate timing.
47924800
// Falls back to HTMLMediaElement playback if WebAudio isn't ready
47934801
// or decoding fails (the syncRuntimeMedia path handles that).
@@ -4796,6 +4804,7 @@ export function initSandboxRuntimeModular(): void {
47964804
}
47974805
runAdapters("play");
47984806
syncMediaForCurrentState();
4807+
keepLateMedia = false;
47994808
primeMediaForLaterStarts(resolveMediaClipIndex().clips.map((clip) => clip.el));
48004809
colorGrading.redraw();
48014810
postState(true);
@@ -5674,25 +5683,31 @@ export function initSandboxRuntimeModular(): void {
56745683
}
56755684

56765685
const READY_AHEAD_SECONDS = 1;
5677-
// WebKit reports a paused element as HAVE_CURRENT_DATA however much it has buffered, so read the ranges too.
5678-
const hasDataAhead = (el: HTMLMediaElement): boolean => {
5679-
if (el.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA) return true;
5680-
if (el.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) return false;
5686+
// WebKit reports a paused element as HAVE_CURRENT_DATA however much it has buffered, so its ranges count too.
5687+
const hasDataAhead = (el: HTMLMediaElement): boolean =>
5688+
el.readyState >= HTMLMediaElement.HAVE_FUTURE_DATA || (isWebKit() && canResumeFrom(el));
5689+
// HAVE_FUTURE_DATA can mean one frame, so a held film resumes only on the stricter reading.
5690+
const canResumeFrom = (el: HTMLMediaElement): boolean => {
5691+
if (el.readyState >= HTMLMediaElement.HAVE_ENOUGH_DATA) return true;
5692+
const least = isWebKit()
5693+
? HTMLMediaElement.HAVE_CURRENT_DATA
5694+
: HTMLMediaElement.HAVE_FUTURE_DATA;
5695+
if (el.readyState < least) return false;
56815696
const until = Math.min(el.currentTime + READY_AHEAD_SECONDS, el.duration) - 0.01;
56825697
for (let i = 0; i < el.buffered.length; i++) {
56835698
if (el.buffered.start(i) <= el.currentTime && el.buffered.end(i) >= until) return true;
56845699
}
56855700
return false;
56865701
};
56875702

5688-
const mediaNotReadyAt = (t: number): HTMLMediaElement[] =>
5703+
const mediaNotReadyAt = (t: number, ready = hasDataAhead): HTMLMediaElement[] =>
56895704
state.nativeMediaSyncDisabled
56905705
? []
56915706
: mediaClipsInWindow
56925707
.filter(
56935708
({ el, start, end }) =>
56945709
isInClipWindow(t, start, end) &&
5695-
!hasDataAhead(el) &&
5710+
!ready(el) &&
56965711
isPlayableTrack(el) &&
56975712
!stillStandsIn(el) &&
56985713
!webAudio.ownsElement(el),
@@ -5796,11 +5811,13 @@ export function initSandboxRuntimeModular(): void {
57965811
filmTime: t,
57975812
fps: state.canonicalFps,
57985813
rate: state.playbackRate,
5799-
playing,
5814+
playing: playing || bufferingFor.length > 0,
58005815
hidden: document.visibilityState === "hidden",
5801-
waiting: mediaClipsInWindow.some(
5802-
({ el }) => el.tagName === "VIDEO" && !el.paused && el.readyState < 3,
5803-
),
5816+
waiting:
5817+
bufferingFor.length > 0 ||
5818+
mediaClipsInWindow.some(
5819+
({ el }) => el.tagName === "VIDEO" && !el.paused && el.readyState < 3,
5820+
),
58045821
frozen: hasFrozenPicture(mediaClipsInWindow, t),
58055822
clock: clock.followedClock(),
58065823
layoutKey: `${window.innerWidth}x${window.innerHeight}@${window.devicePixelRatio}`,
@@ -5873,8 +5890,11 @@ export function initSandboxRuntimeModular(): void {
58735890
const notReady = mediaNotReadyAt(state.currentTime);
58745891
if (clock.isPlaying() && notReady.length > 0) {
58755892
bufferingFor = notReady;
5876-
haltPlayback();
5877-
} else if (bufferingFor.length > 0 && !notReady.some((el) => bufferingFor.includes(el))) {
5893+
haltPlayback(true);
5894+
} else if (
5895+
bufferingFor.length > 0 &&
5896+
!mediaNotReadyAt(state.currentTime, canResumeFrom).some((el) => bufferingFor.includes(el))
5897+
) {
58785898
transport.play();
58795899
}
58805900
postState(false);
@@ -5887,14 +5907,14 @@ export function initSandboxRuntimeModular(): void {
58875907
}
58885908
};
58895909

5890-
const hardSyncAllMedia = (timeSeconds: number) => {
5910+
const hardSyncAllMedia = (timeSeconds: number, tolerance = 0.001) => {
58915911
// Same clips and source time as the media sync that follows, so it has nothing left to seek.
58925912
const clips = withTimingResolver(() => buildRuntimeMediaCache().mediaClips);
58935913
for (const clip of transportMovableClips(clips) ?? []) {
58945914
if (!clip.el.isConnected || !isInClipWindow(timeSeconds, clip.start, clip.end)) continue;
58955915
const target = mediaSourceTimeAt(clip, timeSeconds);
58965916
// A seek flushes the decoder even when it lands where the video already is.
5897-
if (Math.abs(clip.el.currentTime - target) <= 0.001) continue;
5917+
if (Math.abs(clip.el.currentTime - target) <= tolerance) continue;
58985918
try {
58995919
clip.el.currentTime = mediaElementTime(target);
59005920
} catch {

‎packages/core/src/runtime/media.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import {
2222
MEDIA_SYNC_TOLERANCE_SECONDS,
2323
steerMedia,
2424
} from "./mediaSteer.js";
25-
export { MEDIA_SYNC_TOLERANCE_SECONDS } from "./mediaSteer.js";
25+
export { MEDIA_HARD_SYNC_SECONDS, MEDIA_SYNC_TOLERANCE_SECONDS } from "./mediaSteer.js";
2626
export type { RuntimeMediaClip } from "./types.js";
2727
export {
2828
readElementPlaybackRate,
@@ -444,6 +444,7 @@ export function syncRuntimeMedia(params: {
444444
* unity; do not mistake that transport write for an authored volume edit. */
445445
isWebAudioRouted?: (el: HTMLMediaElement) => boolean;
446446
forceSync?: boolean;
447+
keepLateMedia?: boolean;
447448
/** False while the playhead is not moving (audio output has not started): a paused video waits for it. */
448449
clockAdvancing?: boolean;
449450
/** How far the next tick will move the playhead: an audio clip due within it starts now. */
@@ -697,6 +698,7 @@ export function syncRuntimeMedia(params: {
697698
let strictSync = false;
698699
if (
699700
!isPlayingMedia &&
701+
!params.keepLateMedia &&
700702
!holdingAhead &&
701703
!hardSync &&
702704
!firstTickOfClip &&

0 commit comments

Comments
 (0)