Skip to content

Commit aca914d

Browse files
committed
fix(studio): release scrub decoders before native playback
1 parent e220e96 commit aca914d

13 files changed

Lines changed: 382 additions & 57 deletions

‎docs/packages/studio.mdx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,9 @@ advancing the composition's animations. Completed frames appear during a drag,
111111
with newer pointer positions coalesced into the next decode. Canvas and GPU
112112
effects consume the same staged pixels. Continuous playback uses native video;
113113
Studio keeps the paused picture visible while those videos catch up before Play.
114+
A slow native handoff stays buffered instead of revealing an unready video; Pause
115+
or a new seek cancels it. Studio releases paused decoding sessions before native
116+
playback and reopens them when paused scrubbing resumes.
114117
Scrubbing and paused frame steps decode the original video at source resolution
115118
throughout. Automatic playback quality reductions do not reduce these frames,
116119
and the frame decoder does not substitute re-encoded seek copies. Playback

‎packages/core/src/runtime/init.previewSeek.test.ts‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@ function decoder(video: HTMLVideoElement, initiallyReady: boolean) {
3939
return {
4040
video,
4141
seeks,
42+
metadataAt: (position: number) => {
43+
time = position;
44+
ready = 1;
45+
seeking = false;
46+
},
4247
land: () => {
4348
ready = 4;
4449
seeking = false;
@@ -359,6 +364,128 @@ it.each(["stop-media", "set-native-media-sync-disabled"])(
359364
},
360365
);
361366

367+
it("keeps the complete picture and buffers beyond the handoff cap until native frames are ready", async () => {
368+
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
369+
const { main, old, next } = film();
370+
const { pending } = frameProvider([...old, ...next]);
371+
const landed = window.__player!.seek(1);
372+
pending.shift()!();
373+
await landed;
374+
for (const video of old) video.metadataAt(1);
375+
window.__player!.play();
376+
expect(old.every(({ video }) => video.preload === "auto")).toBe(true);
377+
await vi.advanceTimersByTimeAsync(5100);
378+
expect(window.__player!.isPlaying()).toBe(false);
379+
expect(main.time?.()).toBe(1);
380+
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
381+
window.__player!.seek(1);
382+
for (const video of old) video.land();
383+
await vi.advanceTimersByTimeAsync(0);
384+
expect(window.__player!.isPlaying()).toBe(true);
385+
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
386+
});
387+
388+
it("does not wait for a video frame from an audio-only video element", async () => {
389+
const { main, next } = film();
390+
for (const { video } of next)
391+
Object.defineProperties(video, {
392+
readyState: { value: 1, configurable: true },
393+
videoWidth: { value: 0, configurable: true },
394+
error: { value: null, configurable: true },
395+
});
396+
expect(window.__player!.seek(6)).toBeUndefined();
397+
expect(main.time?.()).toBe(6);
398+
window.__player!.play();
399+
expect(window.__player!.isPlaying()).toBe(true);
400+
});
401+
402+
it("waits for paused decoder suspension before preparing native playback", async () => {
403+
const { old, next } = film();
404+
const { provider, pending } = frameProvider([...old, ...next]);
405+
let releaseDecoders = () => {};
406+
provider.suspend = vi.fn(
407+
() =>
408+
new Promise<void>((resolve) => {
409+
releaseDecoders = resolve;
410+
}),
411+
);
412+
const initial = window.__player!.seek(1);
413+
pending.shift()!();
414+
await initial;
415+
const seeks = old.map((v) => v.seeks.length);
416+
window.__player!.play();
417+
expect(provider.suspend).toHaveBeenCalledOnce();
418+
expect(old.map((v) => v.seeks.length)).toEqual(seeks);
419+
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
420+
releaseDecoders();
421+
await Promise.resolve();
422+
for (const video of old) video.land();
423+
await new Promise((resolve) => setTimeout(resolve, 0));
424+
expect(window.__player!.isPlaying()).toBe(true);
425+
});
426+
427+
it("restores paused ownership when Pause cancels an unfinished decoder suspension", async () => {
428+
const { old, next } = film();
429+
const { provider, pending } = frameProvider([...old, ...next]);
430+
let releaseDecoders = () => {};
431+
provider.suspend = () =>
432+
new Promise<void>((resolve) => {
433+
releaseDecoders = resolve;
434+
});
435+
const initial = window.__player!.seek(1);
436+
pending.shift()!();
437+
await initial;
438+
window.__player!.play();
439+
window.__player!.pause();
440+
expect(pending).toHaveLength(1);
441+
pending.shift()!();
442+
releaseDecoders();
443+
await new Promise((resolve) => setTimeout(resolve, 0));
444+
expect(window.__player!.isPlaying()).toBe(false);
445+
expect(
446+
old.every(({ video }) => Object.hasOwn(video, "currentTime") && video.currentTime === 1),
447+
).toBe(true);
448+
});
449+
450+
it("warms the restart destination before Play when paused at the end", async () => {
451+
const { main, old, next } = film();
452+
const { pending } = frameProvider([...old, ...next]);
453+
const landed = window.__player!.seek(10, { settled: true });
454+
pending.shift()!();
455+
await landed;
456+
expect(main.time?.()).toBe(10);
457+
for (const video of old) video.metadataAt(0);
458+
const seeks = old.map((v) => v.seeks.length);
459+
window.__player!.play();
460+
expect(old.map((v) => v.seeks.length)).toEqual(seeks.map((n) => n + 1));
461+
expect(window.__player!.isPlaying()).toBe(false);
462+
for (const video of old) video.land();
463+
await new Promise((resolve) => setTimeout(resolve, 0));
464+
expect(window.__player!.isPlaying()).toBe(true);
465+
expect(window.__player!.getTime()).toBeLessThan(1);
466+
expect(old.every(({ video }) => video.style.visibility === "visible")).toBe(true);
467+
});
468+
469+
it("retains a complete paused picture while replacing or removing its frame provider", async () => {
470+
const { main, old, next } = film();
471+
const { provider, pending } = frameProvider([...old, ...next]);
472+
const initial = window.__player!.seek(1);
473+
pending.shift()!();
474+
await initial;
475+
const replacement = { decode: provider.decode, dispose: vi.fn() };
476+
window.__hf!.setPausedVideoFrameProvider!(replacement);
477+
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
478+
pending.shift()!();
479+
await Promise.resolve();
480+
await Promise.resolve();
481+
expect(main.time?.()).toBe(1);
482+
window.__hf!.setPausedVideoFrameProvider!(null);
483+
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
484+
for (const video of old) video.land();
485+
await Promise.resolve();
486+
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
487+
});
488+
362489
it("retains an in-flight destination when the host replaces its frame provider", async () => {
363490
const { main, old, next } = film();
364491
const { provider, pending } = frameProvider([...old, ...next]);

‎packages/core/src/runtime/init.ts‎

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -535,6 +535,7 @@ export function initSandboxRuntimeModular(): void {
535535
let previewSeek: ReturnType<typeof createPreviewSeek> | undefined;
536536
const pausedFrames = createPausedVideoFrames();
537537
let nativePlayGeneration = 0;
538+
let nativePlayTime: number | undefined;
538539
let nativeWarmTimer: ReturnType<typeof wallSetTimeout> | undefined;
539540
const stampsForStudio = window.parent !== window;
540541
// Studio's discovery stamps time a clip for the timeline only; nothing hides it at their end.
@@ -748,6 +749,7 @@ export function initSandboxRuntimeModular(): void {
748749
};
749750
registerRuntimeCleanup(() => {
750751
nativePlayGeneration++;
752+
nativePlayTime = undefined;
751753
wallClearTimeout(nativeWarmTimer);
752754
pausedFrames.destroy();
753755
delete window.__hf?.setPausedVideoFrameProvider;
@@ -3709,12 +3711,22 @@ export function initSandboxRuntimeModular(): void {
37093711
window.__hf.leasePausedMedia = leasePausedMedia;
37103712
window.__hf.releasePausedMedia = releasePausedMedia;
37113713
window.__hf.setPausedVideoFrameProvider = (provider) => {
3712-
const destination = previewSeek?.time;
3714+
const resumePlay = nativePlayTime !== undefined;
3715+
const destination = previewSeek?.time ?? (pausedFrames.active ? state.currentTime : undefined);
37133716
nativePlayGeneration++;
3717+
nativePlayTime = undefined;
37143718
wallClearTimeout(nativeWarmTimer);
37153719
previewSeek?.cancel();
37163720
pausedFrames.setProvider(provider);
3717-
if (destination !== undefined) void seekWhenReady(destination);
3721+
if (!provider) pausedFrames.prepareNative();
3722+
if (destination !== undefined) {
3723+
const landed = seekWhenReady(destination);
3724+
const generation = nativePlayGeneration;
3725+
if (resumePlay)
3726+
void Promise.resolve(landed).then(() => {
3727+
if (!state.tornDown && generation === nativePlayGeneration) transport.play();
3728+
});
3729+
}
37183730
};
37193731
// Clips the transport may move now (not a paused lease); null while the host drives media.
37203732
const transportMovableClips = (clips: RuntimeMediaClip[]): RuntimeMediaClip[] | null => {
@@ -4997,6 +5009,7 @@ export function initSandboxRuntimeModular(): void {
49975009
return (transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []).filter(
49985010
(clip) =>
49995011
isVideoElement(clip.el) &&
5012+
!(clip.el.readyState >= 1 && clip.el.videoWidth === 0) &&
50005013
!isSilencedByHidden(clip.el) &&
50015014
(clip.el.hasAttribute("src") || clip.el.querySelector("source[src]") !== null) &&
50025015
isClipVisibleAt(time, clip.start, clip.end, duration) &&
@@ -5007,6 +5020,7 @@ export function initSandboxRuntimeModular(): void {
50075020
previewSeek = createPreviewSeek({
50085021
frames: {
50095022
available: pausedFrames.available,
5023+
active: () => pausedFrames.active,
50105024
prepare: (clips, time, signal, settled) => {
50115025
moveSceneWindow(time);
50125026
for (const { el } of clips) {
@@ -5028,6 +5042,7 @@ export function initSandboxRuntimeModular(): void {
50285042
);
50295043
return pausedFrames.prepare(requests, signal);
50305044
},
5045+
suspend: pausedFrames.suspendProvider,
50315046
prepareNative: pausedFrames.prepareNative,
50325047
clear: pausedFrames.clear,
50335048
},
@@ -5042,11 +5057,17 @@ export function initSandboxRuntimeModular(): void {
50425057
({ name, start, end }) => start <= time && time <= end && parkedNames.has(name),
50435058
) &&
50445059
previewVideoClipsAt(time).every(
5045-
({ el }) => !isVideoElement(el) || el.videoWidth > 0 || el.error !== null,
5060+
({ el }) => !isVideoElement(el) || el.videoWidth > 0 || isUnplayable(el),
50465061
),
50475062
clipsAt: previewVideoClipsAt,
50485063
prepare: (clips, time) => {
50495064
moveSceneWindow(time);
5065+
for (const clip of clips) {
5066+
const { el } = clip;
5067+
if (el.preload !== "auto") el.preload = "auto";
5068+
if (el.readyState === 1 && !el.seeking)
5069+
el.currentTime = mediaSourceTimeAt(clip, time) + 1e-5;
5070+
}
50505071
syncRuntimeMedia({
50515072
clips,
50525073
timeSeconds: time,
@@ -5076,7 +5097,9 @@ export function initSandboxRuntimeModular(): void {
50765097
});
50775098

50785099
const seekWhenReady = (time: number, options?: { keepPlaying?: boolean; settled?: boolean }) => {
5100+
if (nativePlayTime !== undefined && Math.abs(time - nativePlayTime) < 1e-5) return;
50795101
nativePlayGeneration++;
5102+
nativePlayTime = undefined;
50805103
wallClearTimeout(nativeWarmTimer);
50815104
if (
50825105
!mediaDeferred ||
@@ -5116,7 +5139,11 @@ export function initSandboxRuntimeModular(): void {
51165139
) {
51175140
wallClearTimeout(nativeWarmTimer);
51185141
const generation = ++nativePlayGeneration;
5119-
const landed = previewSeek?.handoff(previewSeek.time ?? state.currentTime);
5142+
const duration = getSafeTimelineDurationSeconds(state.capturedTimeline, 0);
5143+
if (duration > 0) clock.setDuration(duration);
5144+
nativePlayTime =
5145+
previewSeek?.time ?? (clock.reachedEnd() ? clock.getPlayStart() : state.currentTime);
5146+
const landed = previewSeek?.handoff(nativePlayTime);
51205147
if (landed) {
51215148
bufferingFor = withTimingResolver(() =>
51225149
(transportMovableClips(buildRuntimeMediaCache().mediaClips) ?? []).map(({ el }) => el),
@@ -5125,10 +5152,12 @@ export function initSandboxRuntimeModular(): void {
51255152
void landed.then(() => {
51265153
if (state.tornDown || generation !== nativePlayGeneration) return;
51275154
bufferingFor = [];
5155+
nativePlayTime = undefined;
51285156
transport.play();
51295157
});
51305158
return;
51315159
}
5160+
nativePlayTime = undefined;
51325161
}
51335162
const destination = previewSeek?.time;
51345163
previewSeek?.cancel();
@@ -5167,8 +5196,9 @@ export function initSandboxRuntimeModular(): void {
51675196
postState(true);
51685197
},
51695198
pause: () => {
5199+
const destination = previewSeek?.time ?? nativePlayTime;
51705200
nativePlayGeneration++;
5171-
const destination = previewSeek?.time;
5201+
nativePlayTime = undefined;
51725202
previewSeek?.cancel();
51735203
const wasBuffering = bufferingFor.length > 0;
51745204
bufferingFor = [];
@@ -5185,12 +5215,14 @@ export function initSandboxRuntimeModular(): void {
51855215
else if (wasBuffering) postState(true);
51865216
},
51875217
seek: (timeSeconds, options) => {
5188-
nativePlayGeneration++;
5189-
wallClearTimeout(nativeWarmTimer);
51905218
const quantized = quantizeTimeToFrame(
51915219
Math.max(0, Number(timeSeconds) || 0),
51925220
state.canonicalFps,
51935221
);
5222+
if (nativePlayTime !== undefined && Math.abs(quantized - nativePlayTime) < 1e-5) return;
5223+
nativePlayGeneration++;
5224+
nativePlayTime = undefined;
5225+
wallClearTimeout(nativeWarmTimer);
51945226
heldSeek = null;
51955227
const undecoded = clock.isPlaying() ? [] : undecodedImagesShownAt(quantized);
51965228
if (undecoded.length === 0) {
@@ -5222,6 +5254,7 @@ export function initSandboxRuntimeModular(): void {
52225254
},
52235255
renderSeek: (timeSeconds, options) => {
52245256
nativePlayGeneration++;
5257+
nativePlayTime = undefined;
52255258
wallClearTimeout(nativeWarmTimer);
52265259
pausedFrames.clear();
52275260
heldSeek = null;
@@ -6500,6 +6533,7 @@ export function initSandboxRuntimeModular(): void {
65006533
},
65016534
onStopMedia: () => {
65026535
nativePlayGeneration++;
6536+
nativePlayTime = undefined;
65036537
wallClearTimeout(nativeWarmTimer);
65046538
previewSeek?.cancel();
65056539
bufferingFor = [];
@@ -6556,6 +6590,7 @@ export function initSandboxRuntimeModular(): void {
65566590
onSetNativeMediaSyncDisabled: (disabled) => {
65576591
if (disabled) {
65586592
nativePlayGeneration++;
6593+
nativePlayTime = undefined;
65596594
wallClearTimeout(nativeWarmTimer);
65606595
previewSeek?.cancel();
65616596
pausedFrames.clear();

‎packages/core/src/runtime/media.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -663,17 +663,17 @@ export function syncRuntimeMedia(params: {
663663
// Mute only when force-muted or the transport owns this element; an unclaimed
664664
// track stays audible via the HTMLMedia fallback.
665665
if ((forceMuteAll || params.isWebAudioOwned?.(el)) && !el.muted) el.muted = true;
666-
if (!params.playing && ownsPausedVideoTime(el)) {
667-
if (!el.paused) el.pause();
668-
continue;
669-
}
670666
// Ensure full preload for every active media element. Streaming
671667
// formats (MP3) may arrive with preload="metadata", which only
672668
// buffers the first few seconds and causes seeks to silently fail
673669
// past the buffered range. Setting this on every tick is cheap
674670
// (no-op when already "auto") and catches elements whose preload
675671
// was overridden after init.ts set it.
676672
if (el.preload !== "auto" && !isRenderFrameVideo(el)) el.preload = "auto";
673+
if (!params.playing && ownsPausedVideoTime(el)) {
674+
if (!el.paused) el.pause();
675+
continue;
676+
}
677677
// Per-element rate × global transport rate
678678
const baseRate = rateAt(clipRate, params.timeSeconds - origin) * params.playbackRate;
679679
// Too slow for the browser to play: kept paused and stepped onto the playhead by seeks.

‎packages/core/src/runtime/pausedVideoFrames.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,18 @@ it("releases a late decode when its request is aborted or its source changes", a
9191
expect(getPausedVideoFrame(video)).toBeNull();
9292
});
9393

94+
it("does not surrender decoded presentation for the runtime's native seek padding", async () => {
95+
const { session, video, request } = fixture();
96+
const batch = await session.prepare(request, new AbortController().signal);
97+
batch!.present();
98+
batch!.refresh();
99+
const canvas = getPausedVideoFrame(video);
100+
video.currentTime = 3.00001;
101+
expect(ownsPausedVideoTime(video)).toBe(true);
102+
expect(getPausedVideoFrame(video)).toBe(canvas);
103+
expect(video.currentTime).toBe(3);
104+
});
105+
94106
it("gives an authored seek back to native media and preserves an existing currentTime override", async () => {
95107
const { session, video, request } = fixture();
96108
const batch = await session.prepare(request, new AbortController().signal);

0 commit comments

Comments
 (0)