Skip to content

Commit 3f1dd2f

Browse files
committed
fix(studio): bound Play handoffs and drain transparent preview samples
1 parent a1f2a76 commit 3f1dd2f

10 files changed

Lines changed: 486 additions & 129 deletions

File tree

‎docs/packages/studio.mdx‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -111,12 +111,15 @@ advancing the composition's animations. Completed frames appear during a drag,
111111
with newer pointer positions coalesced into the next decode. Canvas and GPU
112112
effects consume the same staged pixels. Continuous playback uses native video;
113113
Studio keeps the paused picture visible while those videos catch up before Play.
114-
A slow native handoff stays buffered instead of revealing an unready video; Pause
115-
or a new seek cancels it. If the browser stops loading a clip, the rest can play
116-
while its last decoded picture remains visible until native frames recover.
117-
A paused fallback that outlasts its wait still publishes when the video becomes
118-
ready. Studio releases paused decoding sessions before native
119-
playback and reopens them when paused scrubbing resumes.
114+
A native handoff waits for ready frames for up to ten seconds; Pause or a new
115+
seek cancels it. At the deadline, the rest plays while each unready video keeps
116+
its last full-quality picture. Recovery seeks to the current playback time
117+
before revealing native video. A skipped source keeps its picture on subsequent
118+
Play requests until it recovers.
119+
A paused fallback that outlasts its wait retains the presented clock and picture,
120+
while paused sync continues. It can still publish when the video becomes ready;
121+
a new gesture cancels that recovery. Studio requests paused-decoder release
122+
before preparing native playback and reopens readers when paused scrubbing resumes.
120123
Scrubbing and paused frame steps decode the original video at source resolution
121124
throughout. Automatic playback quality reductions do not reduce these frames,
122125
and the frame decoder does not substitute re-encoded seek copies. Playback

‎packages/core/src/runtime/init.previewSeek.test.ts‎

Lines changed: 151 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ import { afterEach, expect, it, vi } from "vitest";
22
import { initSandboxRuntimeModular } from "./init";
33
import { hideTimedClipsUntilFirstPass } from "./timedClipHide";
44
import { resetSeekDispatchState } from "./adapters/seek-dispatch";
5-
import { notePlaybackIntent } from "./media";
65
import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
76
import { PREVIEW_SEEK_HOLD_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
87
import type {
@@ -414,29 +413,88 @@ it("settles a stalled paused fallback while retaining the last complete picture"
414413
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
415414
});
416415

417-
it("plays the rest after native loading stops at metadata, retaining the stalled picture until recovery", async () => {
416+
it.each([
417+
["already metadata-only, idle", false, 1],
418+
["ready, then metadata-only idle", true, 1],
419+
["ready, then loading forever", true, 2],
420+
["metadata-only, loading forever", false, 2],
421+
] as const)(
422+
"bounds Play for %s and releases its picture only at the current transport time",
423+
async (_, initiallyReady, networkState) => {
424+
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
425+
const { main, old, next } = film();
426+
const { pending } = frameProvider([...old, ...next]);
427+
const initial = window.__player!.seek(3);
428+
pending.shift()!();
429+
await initial;
430+
const stalled = old[0]!;
431+
if (initiallyReady) stalled.land();
432+
else stalled.metadataAt(3);
433+
Object.defineProperty(stalled.video, "networkState", {
434+
value: networkState,
435+
configurable: true,
436+
});
437+
window.__player!.play();
438+
old[1]!.land();
439+
await vi.advanceTimersByTimeAsync(5100);
440+
if (initiallyReady || networkState === 2) expect(window.__player!.isPlaying()).toBe(false);
441+
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(
442+
window.__player!.isPlaying() ? 1 : 2,
443+
);
444+
stalled.video.dispatchEvent(new Event("progress"));
445+
await vi.advanceTimersByTimeAsync(4900);
446+
expect(window.__player!.isPlaying()).toBe(true);
447+
expect(main.time?.()).toBeGreaterThanOrEqual(3);
448+
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(1);
449+
const held = document.querySelector("[data-hf-preview-video-canvas]")!;
450+
expect([held.getAttribute("width"), held.getAttribute("height")]).toEqual(["64", "36"]);
451+
},
452+
);
453+
454+
it("keeps every unready video picture at the hard deadline", async () => {
418455
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
419-
const { main, old, next } = film();
456+
const { old, next } = film();
420457
const { pending } = frameProvider([...old, ...next]);
421-
const initial = window.__player!.seek(1);
458+
const initial = window.__player!.seek(3);
459+
pending.shift()!();
460+
await initial;
461+
const pictures = [...document.querySelectorAll("[data-hf-preview-video-canvas]")];
462+
window.__player!.play();
463+
await vi.advanceTimersByTimeAsync(10000);
464+
expect(window.__player!.isPlaying()).toBe(true);
465+
expect([...document.querySelectorAll("[data-hf-preview-video-canvas]")]).toEqual(pictures);
466+
window.__player!.pause();
467+
pending.shift()!();
468+
await Promise.resolve();
469+
await Promise.resolve();
470+
window.__player!.play();
471+
await Promise.resolve();
472+
await Promise.resolve();
473+
expect(window.__player!.isPlaying()).toBe(true);
474+
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
475+
});
476+
477+
it("seeks late native recovery to the moving transport before removing its picture", async () => {
478+
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
479+
const { old, next } = film();
480+
const { pending } = frameProvider([...old, ...next]);
481+
const initial = window.__player!.seek(3);
422482
pending.shift()!();
423483
await initial;
424484
const stalled = old[0]!;
425-
stalled.land();
426-
notePlaybackIntent(stalled.video);
427-
stalled.metadataAt(1);
428-
Object.defineProperty(stalled.video, "networkState", { value: 1, configurable: true });
485+
stalled.metadataAt(3);
486+
let error: { code: number } | null = { code: 4 };
487+
Object.defineProperty(stalled.video, "error", { configurable: true, get: () => error });
429488
window.__player!.play();
430489
old[1]!.land();
431-
await vi.advanceTimersByTimeAsync(5100);
490+
await vi.advanceTimersByTimeAsync(500);
432491
expect(window.__player!.isPlaying()).toBe(true);
433-
expect(main.time?.()).toBeGreaterThanOrEqual(1);
434-
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(1);
435-
expect(stalled.video.previousElementSibling?.hasAttribute("data-hf-preview-video-canvas")).toBe(
436-
true,
437-
);
492+
const picture = stalled.video.previousElementSibling;
493+
error = null;
494+
stalled.land();
495+
expect(stalled.video.previousElementSibling).toBe(picture);
496+
expect(stalled.video.currentTime).toBeCloseTo(3.5, 2);
438497
stalled.land();
439-
await Promise.resolve();
440498
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
441499
});
442500

@@ -454,6 +512,12 @@ it("publishes a late native recovery after a paused fallback wait has settled",
454512
await stalled;
455513
expect(main.time?.()).toBe(1);
456514
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
515+
expect(window.__player!.getTime()).toBe(1);
516+
const priorSeeks = old[0]!.seeks.length;
517+
const destinationSeeks = next.map((video) => video.seeks.length);
518+
old[0]!.reload();
519+
expect(old[0]!.seeks.length).toBeGreaterThan(priorSeeks);
520+
expect(next.map((video) => video.seeks.length)).toEqual(destinationSeeks);
457521
for (const video of next) video.land();
458522
await Promise.resolve();
459523
expect(main.time?.()).toBe(6);
@@ -470,11 +534,11 @@ it("keeps a hold-only snapshot visible while Play awaits decoder suspension", as
470534
suspended = resolve;
471535
});
472536
const seek = window.__player!.seek(6);
473-
expect(document.querySelectorAll("[data-hf-seek-video]")).toHaveLength(2);
537+
const pictures = [...document.querySelectorAll("canvas")];
538+
expect(pictures).toHaveLength(2);
474539
window.__player!.play();
475540
await seek;
476-
expect(document.querySelectorAll("[data-hf-seek-video]")).toHaveLength(2);
477-
expect(document.querySelectorAll("canvas")).toHaveLength(2);
541+
expect([...document.querySelectorAll("canvas")]).toEqual(pictures);
478542
suspended();
479543
await Promise.resolve();
480544
for (const video of next) video.land();
@@ -499,6 +563,75 @@ it("resumes a pending Play after its frame provider is replaced", async () => {
499563
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
500564
});
501565

566+
it("cancels the hard Play deadline when Pause restores provider ownership", async () => {
567+
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
568+
const { old, next } = film();
569+
const { pending } = frameProvider([...old, ...next]);
570+
const initial = window.__player!.seek(3);
571+
pending.shift()!();
572+
await initial;
573+
window.__player!.play();
574+
await vi.advanceTimersByTimeAsync(9000);
575+
window.__player!.pause();
576+
pending.shift()!();
577+
await vi.advanceTimersByTimeAsync(20000);
578+
expect(window.__player!.isPlaying()).toBe(false);
579+
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
580+
});
581+
582+
it("bounds a deferred decoder release and ignores its late completion", async () => {
583+
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
584+
const { old, next } = film();
585+
const { provider, pending } = frameProvider([...old, ...next]);
586+
let released = () => {};
587+
provider.suspend = () =>
588+
new Promise<void>((resolve) => {
589+
released = resolve;
590+
});
591+
const initial = window.__player!.seek(3);
592+
pending.shift()!();
593+
await initial;
594+
window.__player!.play();
595+
await vi.advanceTimersByTimeAsync(10000);
596+
expect(window.__player!.isPlaying()).toBe(true);
597+
const pictures = [...document.querySelectorAll("[data-hf-preview-video-canvas]")];
598+
expect(pictures).toHaveLength(2);
599+
released();
600+
await Promise.resolve();
601+
await Promise.resolve();
602+
expect(window.__player!.isPlaying()).toBe(true);
603+
expect([...document.querySelectorAll("[data-hf-preview-video-canvas]")]).toEqual(pictures);
604+
});
605+
606+
it("ignores decoder release after Pause cancels a pending Play", async () => {
607+
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
608+
const { old, next } = film();
609+
const { provider, pending } = frameProvider([...old, ...next]);
610+
let released = () => {};
611+
provider.suspend = () =>
612+
new Promise<void>((resolve) => {
613+
released = resolve;
614+
});
615+
const initial = window.__player!.seek(3);
616+
pending.shift()!();
617+
await initial;
618+
window.__player!.play();
619+
await vi.advanceTimersByTimeAsync(9000);
620+
window.__player!.pause();
621+
pending.shift()!();
622+
await Promise.resolve();
623+
await Promise.resolve();
624+
const pictures = [...document.querySelectorAll("[data-hf-preview-video-canvas]")];
625+
const seeks = old.map((video) => video.seeks.length);
626+
released();
627+
await Promise.resolve();
628+
await Promise.resolve();
629+
expect(old.map((video) => video.seeks.length)).toEqual(seeks);
630+
await vi.advanceTimersByTimeAsync(1000);
631+
expect(window.__player!.isPlaying()).toBe(false);
632+
expect([...document.querySelectorAll("[data-hf-preview-video-canvas]")]).toEqual(pictures);
633+
});
634+
502635
it("does not wait for a video frame from an audio-only video element", async () => {
503636
const { main, next } = film();
504637
for (const { video } of next)

‎packages/core/src/runtime/init.ts‎

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3712,7 +3712,8 @@ export function initSandboxRuntimeModular(): void {
37123712
window.__hf.releasePausedMedia = releasePausedMedia;
37133713
window.__hf.setPausedVideoFrameProvider = (provider) => {
37143714
const resumePlay = nativePlayTime !== undefined;
3715-
const destination = previewSeek?.time ?? (pausedFrames.active ? state.currentTime : undefined);
3715+
const destination =
3716+
previewSeek?.targetTime ?? (pausedFrames.active ? state.currentTime : undefined);
37163717
nativePlayGeneration++;
37173718
nativePlayTime = undefined;
37183719
wallClearTimeout(nativeWarmTimer);
@@ -3731,7 +3732,11 @@ export function initSandboxRuntimeModular(): void {
37313732
// Clips the transport may move now (not a paused lease); null while the host drives media.
37323733
const transportMovableClips = (clips: RuntimeMediaClip[]): RuntimeMediaClip[] | null => {
37333734
if (state.nativeMediaSyncDisabled) return null;
3734-
return state.isPlaying ? clips : clips.filter((clip) => !pausedMediaLeases.has(clip.el));
3735+
return state.isPlaying
3736+
? clips
3737+
: clips.filter(
3738+
(clip) => !pausedMediaLeases.has(clip.el) && !previewSeek?.ownsRecovery(clip.el),
3739+
);
37353740
};
37363741
window.__hf.setLoopStart = (seconds) => {
37373742
loopStartSeconds = seconds != null && Number.isFinite(seconds) && seconds >= 0 ? seconds : null;
@@ -4525,7 +4530,7 @@ export function initSandboxRuntimeModular(): void {
45254530
// Redraw the current frame as a seek does; a playing film cut to end at or before it stops there, as at its end.
45264531
const cutShort = clock.isPlaying() && duration > 0 && state.currentTime >= duration;
45274532
if (joining && clock.isPlaying() && !cutShort) joinPlayback(swappedHosts);
4528-
else if (previewSeek?.time !== undefined) previewSeek.refresh();
4533+
else if (previewSeek?.targetTime !== undefined) previewSeek.refresh();
45294534
else transport.seek(cutShort ? duration : state.currentTime, { keepPlaying: !cutShort });
45304535
if (previewSeek?.time === undefined) syncTimedElementVisibility(state.currentTime);
45314536
postTimeline();
@@ -4835,7 +4840,7 @@ export function initSandboxRuntimeModular(): void {
48354840
.sort((a, b) => Math.abs(a.start - time) - Math.abs(b.start - time))[0];
48364841
const toMount = landed ?? neighbour;
48374842
const toRetire =
4838-
toMount || previewSeek?.time !== undefined
4843+
toMount || previewSeek?.targetTime !== undefined
48394844
? []
48404845
: windowScenes.filter(({ name }) => !near.has(name) && !parked(name));
48414846
if (!toMount && toRetire.length === 0) {
@@ -4844,9 +4849,11 @@ export function initSandboxRuntimeModular(): void {
48444849
}
48454850
const wantedNow = (name: string) =>
48464851
heldScenes.has(name) ||
4847-
scenesWanted(windowScenes, previewSeek?.time ?? state.currentTime, clock.isPlaying()).has(
4848-
name,
4849-
);
4852+
scenesWanted(
4853+
windowScenes,
4854+
previewSeek?.targetTime ?? state.currentTime,
4855+
clock.isPlaying(),
4856+
).has(name);
48504857
const parkedBeforeStep = parkedNames.size;
48514858
windowWork = (async () => {
48524859
if (landed) await mountScene(landed.name);
@@ -4859,7 +4866,7 @@ export function initSandboxRuntimeModular(): void {
48594866
windowWork = null;
48604867
const stuckStepKeepsItsPlan = parkedNames.size === parkedBeforeStep;
48614868
windowNear = stuckStepKeepsItsPlan ? key : "";
4862-
moveSceneWindow(previewSeek?.time ?? state.currentTime);
4869+
moveSceneWindow(previewSeek?.targetTime ?? state.currentTime);
48634870
});
48644871
// A seek is not done until the scene it lands in is mounted.
48654872
if (landed) registerSeekCompletion(windowWork, "scene-window");
@@ -5021,6 +5028,10 @@ export function initSandboxRuntimeModular(): void {
50215028
frames: {
50225029
available: pausedFrames.available,
50235030
active: () => pausedFrames.active,
5031+
skipped: pausedFrames.skipped,
5032+
hold: (videos) => {
5033+
if (pausedFrames.available()) pausedFrames.holdNative(videos);
5034+
},
50245035
prepare: (clips, time, signal, settled) => {
50255036
moveSceneWindow(time);
50265037
for (const { el } of clips) {
@@ -5044,7 +5055,16 @@ export function initSandboxRuntimeModular(): void {
50445055
},
50455056
suspend: pausedFrames.suspendProvider,
50465057
prepareNative: pausedFrames.prepareNative,
5047-
clear: pausedFrames.clear,
5058+
clear: (keep) =>
5059+
pausedFrames.clear(keep, (video) => {
5060+
const time = clock.now();
5061+
const clip = previewVideoClipsAt(time).find(({ el }) => el === video);
5062+
if (!clip) return true;
5063+
const expected = mediaSourceTimeAt(clip, time);
5064+
if (Math.abs(video.currentTime - expected) < 0.02) return true;
5065+
video.currentTime = mediaElementTime(expected);
5066+
return false;
5067+
}),
50485068
},
50495069
videosToHold: () =>
50505070
withTimingResolver(() =>
@@ -5142,7 +5162,8 @@ export function initSandboxRuntimeModular(): void {
51425162
const duration = getSafeTimelineDurationSeconds(state.capturedTimeline, 0);
51435163
if (duration > 0) clock.setDuration(duration);
51445164
nativePlayTime =
5145-
previewSeek?.time ?? (clock.reachedEnd() ? clock.getPlayStart() : state.currentTime);
5165+
previewSeek?.targetTime ??
5166+
(clock.reachedEnd() ? clock.getPlayStart() : state.currentTime);
51465167
const landed = previewSeek?.handoff(nativePlayTime);
51475168
if (landed) {
51485169
bufferingFor = withTimingResolver(() =>
@@ -5196,7 +5217,7 @@ export function initSandboxRuntimeModular(): void {
51965217
postState(true);
51975218
},
51985219
pause: () => {
5199-
const destination = previewSeek?.time ?? nativePlayTime;
5220+
const destination = previewSeek?.targetTime ?? nativePlayTime;
52005221
nativePlayGeneration++;
52015222
nativePlayTime = undefined;
52025223
previewSeek?.cancel();
@@ -6113,6 +6134,7 @@ export function initSandboxRuntimeModular(): void {
61136134
!ready(el) &&
61146135
isPlayableTrack(el) &&
61156136
!stillStandsIn(el) &&
6137+
!pausedFrames.retained(el) &&
61166138
!webAudio.ownsElement(el),
61176139
)
61186140
.map(({ el }) => el);

‎packages/core/src/runtime/media.ts‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -256,11 +256,6 @@ function noteFirstPlayAsk(el: HTMLMediaElement): void {
256256
}
257257
}
258258

259-
export function notePlaybackIntent(el: HTMLMediaElement): void {
260-
if (el.readyState <= MEDIA_HAVE_METADATA) firstPlayAskedAt.delete(el);
261-
noteFirstPlayAsk(el);
262-
}
263-
264259
type Stall = "after its header" | "never started";
265260

266261
// Safari can leave a video "loading" with no request ever sent; a slow fetch still delivers bytes (progress).

‎packages/core/src/runtime/pausedVideoFrames.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,3 +191,18 @@ it("keeps the displayed surface in sync with paused editing without releasing de
191191
expect(canvas.style.width).toBe("80px");
192192
expect(video.currentTime).toBe(3);
193193
});
194+
195+
it("holds a full-size native snapshot through original-source provider replacement", async () => {
196+
const { session, video, request } = fixture();
197+
Object.defineProperty(video, "readyState", { value: 4 });
198+
session.holdNative([video]);
199+
const snapshot = getPausedVideoFrame(video)!;
200+
expect([snapshot.width, snapshot.height]).toEqual([64, 36]);
201+
expect(snapshot.hasAttribute("data-hyperframes-picker-ignore")).toBe(true);
202+
expect(ownsPausedVideoTime(video)).toBe(false);
203+
const decoded = await session.prepare(request, new AbortController().signal);
204+
expect(decoded!.present()).toBe(true);
205+
expect(getPausedVideoFrame(video)).toBe(snapshot);
206+
expect(ownsPausedVideoTime(video)).toBe(true);
207+
expect(video.currentTime).toBe(3);
208+
});

0 commit comments

Comments
 (0)