Skip to content

Commit 17b19ca

Browse files
committed
fix(preview): recover retained video without chasing playback clock
1 parent 0fd3404 commit 17b19ca

8 files changed

Lines changed: 254 additions & 19 deletions

File tree

‎docs/packages/studio.mdx‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,9 +113,10 @@ effects consume the same staged pixels. Continuous playback uses native video;
113113
Studio keeps the paused picture visible while those videos catch up before Play.
114114
A native handoff waits for ready frames for up to ten seconds; Pause or a new
115115
seek cancels it. At the deadline, the rest plays while each unready video keeps
116-
its last full-quality picture. Recovery seeks to the current playback time
117-
before revealing native video. A skipped source keeps its picture on subsequent
118-
Play requests until it recovers.
116+
its last full-quality picture. During playback, native frame presentation or
117+
playback progress releases that picture; normal transport sync corrects drift.
118+
Paused recovery still waits for its requested frame. A skipped source keeps its
119+
picture on subsequent Play requests until it recovers.
119120
A paused fallback that outlasts its wait retains the presented clock and picture,
120121
while paused sync continues. It can still publish when the video becomes ready;
121122
a new gesture cancels that recovery. Studio requests paused-decoder release

‎packages/core/src/runtime/init.previewSeek.test.ts‎

Lines changed: 62 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,12 @@ function decoder(video: HTMLVideoElement, initiallyReady: boolean) {
4545
ready = 1;
4646
seeking = false;
4747
},
48+
presentAt: (position: number) => {
49+
time = position;
50+
ready = 4;
51+
seeking = false;
52+
video.dispatchEvent(new Event("timeupdate"));
53+
},
4854
land: () => {
4955
ready = 4;
5056
seeking = false;
@@ -419,7 +425,7 @@ it.each([
419425
["ready, then loading forever", true, 2],
420426
["metadata-only, loading forever", false, 2],
421427
] as const)(
422-
"bounds Play for %s and releases its picture only at the current transport time",
428+
"bounds Play for %s while retaining its full-quality picture",
423429
async (_, initiallyReady, networkState) => {
424430
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
425431
const { main, old, next } = film();
@@ -474,7 +480,35 @@ it("keeps every unready video picture at the hard deadline", async () => {
474480
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(2);
475481
});
476482

477-
it("seeks late native recovery to the moving transport before removing its picture", async () => {
483+
it("reveals a delayed playing frame without chasing the moving clock with more seeks", async () => {
484+
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
485+
const { old, next } = film();
486+
const { pending } = frameProvider([...old, ...next]);
487+
const initial = window.__player!.seek(3);
488+
pending.shift()!();
489+
await initial;
490+
const stalled = old[0]!;
491+
stalled.metadataAt(3);
492+
let error: { code: number } | null = { code: 4 };
493+
Object.defineProperty(stalled.video, "error", { configurable: true, get: () => error });
494+
window.__player!.play();
495+
old[1]!.land();
496+
await vi.advanceTimersByTimeAsync(0);
497+
error = null;
498+
const seeks = stalled.seeks.length;
499+
Object.defineProperty(stalled.video, "paused", { configurable: true, value: false });
500+
for (let i = 0; i < 3; i++) {
501+
await vi.advanceTimersByTimeAsync(50);
502+
stalled.land();
503+
if (i === 0) stalled.video.dispatchEvent(new Event("playing"));
504+
stalled.presentAt(3 + (i + 1) / 30);
505+
}
506+
expect(stalled.seeks).toHaveLength(seeks);
507+
expect(document.querySelectorAll("[data-hf-preview-video-canvas]")).toHaveLength(0);
508+
expect(window.__player!.isPlaying()).toBe(true);
509+
});
510+
511+
it("keeps a paused recovery covered until its static target seek lands", async () => {
478512
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] });
479513
const { old, next } = film();
480514
const { pending } = frameProvider([...old, ...next]);
@@ -489,8 +523,12 @@ it("seeks late native recovery to the moving transport before removing its pictu
489523
old[1]!.land();
490524
await vi.advanceTimersByTimeAsync(500);
491525
expect(window.__player!.isPlaying()).toBe(true);
492-
const picture = stalled.video.previousElementSibling;
526+
window.__hf!.setPausedVideoFrameProvider!(null);
527+
window.__player!.pause();
528+
const picture = document.querySelector("[data-hf-preview-video-canvas]");
529+
expect(picture).not.toBeNull();
493530
error = null;
531+
stalled.metadataAt(3);
494532
stalled.land();
495533
expect(stalled.video.previousElementSibling).toBe(picture);
496534
expect(stalled.video.currentTime).toBeCloseTo(3.5, 2);
@@ -737,8 +775,11 @@ it("retains an in-flight destination when the host replaces its frame provider",
737775
const { main, old, next } = film();
738776
const { provider, pending } = frameProvider([...old, ...next]);
739777
const first = window.__player!.seek(6);
778+
const picture = document.querySelector("[data-hf-preview-video-canvas]");
779+
expect(picture).not.toBeNull();
740780
const replacement = { decode: provider.decode, dispose: vi.fn() };
741781
window.__hf!.setPausedVideoFrameProvider!(replacement);
782+
expect(document.querySelector("[data-hf-preview-video-canvas]")).toBe(picture);
742783
expect(provider.dispose).toHaveBeenCalledTimes(1);
743784
expect(window.__player!.getTime()).toBe(6);
744785
pending.shift()!();
@@ -842,3 +883,21 @@ afterEach(() => {
842883
vi.useRealTimers();
843884
resetRuntimeFixtureDom();
844885
});
886+
887+
it("releases a native hold when stop-media cancels its first provider seek", async () => {
888+
const { old, next } = film();
889+
const { pending } = frameProvider([...old, ...next]);
890+
const seek = window.__player!.seek(3);
891+
expect(document.querySelectorAll("canvas[data-hf-preview-video-canvas]")).toHaveLength(2);
892+
window.dispatchEvent(
893+
new MessageEvent("message", {
894+
source: window,
895+
data: { source: "hf-parent", type: "control", action: "stop-media" },
896+
}),
897+
);
898+
await seek;
899+
expect(document.querySelectorAll("canvas[data-hf-preview-video-canvas]")).toHaveLength(0);
900+
pending.shift()!();
901+
await Promise.resolve();
902+
expect(document.querySelectorAll("canvas[data-hf-preview-video-canvas]")).toHaveLength(0);
903+
});

‎packages/core/src/runtime/init.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3717,7 +3717,7 @@ export function initSandboxRuntimeModular(): void {
37173717
nativePlayGeneration++;
37183718
nativePlayTime = undefined;
37193719
wallClearTimeout(nativeWarmTimer);
3720-
previewSeek?.cancel();
3720+
previewSeek?.cancel(true);
37213721
pausedFrames.setProvider(provider);
37223722
if (!provider) pausedFrames.prepareNative();
37233723
if (destination !== undefined) {
@@ -5031,7 +5031,7 @@ export function initSandboxRuntimeModular(): void {
50315031
active: () => pausedFrames.active,
50325032
skipped: pausedFrames.skipped,
50335033
hold: (videos) => {
5034-
if (pausedFrames.available()) pausedFrames.holdNative(videos);
5034+
if (pausedFrames.available()) return pausedFrames.holdNative(videos);
50355035
},
50365036
prepare: (clips, time, signal, settled) => {
50375037
moveSceneWindow(time);
@@ -5057,7 +5057,8 @@ export function initSandboxRuntimeModular(): void {
50575057
suspend: pausedFrames.suspendProvider,
50585058
prepareNative: pausedFrames.prepareNative,
50595059
clear: (keep) =>
5060-
pausedFrames.clear(keep, (video) => {
5060+
pausedFrames.clear(keep, (video, presented) => {
5061+
if (clock.isPlaying()) return presented;
50615062
const time = clock.now();
50625063
const clip = previewVideoClipsAt(time).find(({ el }) => el === video);
50635064
if (!clip) return true;
@@ -5221,7 +5222,7 @@ export function initSandboxRuntimeModular(): void {
52215222
const destination = previewSeek?.targetTime ?? nativePlayTime;
52225223
nativePlayGeneration++;
52235224
nativePlayTime = undefined;
5224-
previewSeek?.cancel();
5225+
previewSeek?.cancel(true);
52255226
const wasBuffering = bufferingFor.length > 0;
52265227
bufferingFor = [];
52275228
if (clock.isPlaying()) {

‎packages/core/src/runtime/pausedVideoCallbacks.test.ts‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { afterEach, beforeEach, expect, it, vi } from "vitest";
22
import {
33
flushPausedVideoCallbacks,
44
installPausedVideoCallbacks,
5+
observeNativeVideoFrames,
56
releasePausedVideoCallbacks,
67
stagePausedVideoCallbacks,
78
} from "./pausedVideoCallbacks";
@@ -263,3 +264,66 @@ it("invalidates the actual vendored Three VideoTexture before redraw and continu
263264
texture.dispose();
264265
expect(pending.size).toBe(0);
265266
});
267+
268+
it("observes native frames independently of staged authored callbacks", () => {
269+
const video = document.createElement("video");
270+
const authored = vi.fn();
271+
video.requestVideoFrameCallback(authored);
272+
stagePausedVideoCallbacks(video, frame(3));
273+
const recovered = vi.fn(() => true);
274+
const stop = observeNativeVideoFrames(video, recovered);
275+
deliverNative(102, metadata(3.05, 2));
276+
expect(recovered).toHaveBeenCalledOnce();
277+
expect(authored).not.toHaveBeenCalled();
278+
flushPausedVideoCallbacks(video);
279+
expect(authored).toHaveBeenCalledOnce();
280+
expect(authored.mock.lastCall?.[1].mediaTime).toBe(3);
281+
stop();
282+
});
283+
284+
it("cancels a re-armed native observer and ignores an already queued callback", () => {
285+
const video = document.createElement("video");
286+
const recovered = vi.fn(() => false);
287+
const stop = observeNativeVideoFrames(video, recovered);
288+
deliverNative(101, metadata(3, 1));
289+
const queued = pending.get(102)!;
290+
stop();
291+
expect(pending.has(102)).toBe(false);
292+
queued.callback(130, metadata(3.05, 2));
293+
expect(recovered).toHaveBeenCalledOnce();
294+
expect(pending.size).toBe(0);
295+
});
296+
297+
it("uses playing and progressing time when opacity defers a native callback", () => {
298+
const video = document.createElement("video");
299+
let paused = true;
300+
let time = 3;
301+
Object.defineProperties(video, {
302+
paused: { get: () => paused },
303+
currentTime: { get: () => time },
304+
});
305+
const recovered = vi.fn(() => true);
306+
const stop = observeNativeVideoFrames(video, recovered);
307+
video.dispatchEvent(new Event("timeupdate"));
308+
expect(recovered).not.toHaveBeenCalled();
309+
paused = false;
310+
video.dispatchEvent(new Event("playing"));
311+
video.dispatchEvent(new Event("timeupdate"));
312+
expect(recovered).not.toHaveBeenCalled();
313+
time += 1 / 30;
314+
video.dispatchEvent(new Event("timeupdate"));
315+
expect(recovered).toHaveBeenCalledOnce();
316+
expect(pending.size).toBe(0);
317+
stop();
318+
});
319+
320+
it("keeps waiting when no native frame or playback progress arrives", () => {
321+
const video = document.createElement("video");
322+
const recovered = vi.fn(() => true);
323+
const stop = observeNativeVideoFrames(video, recovered);
324+
for (let i = 0; i < 3; i++) video.dispatchEvent(new Event("timeupdate"));
325+
expect(recovered).not.toHaveBeenCalled();
326+
expect(pending.size).toBe(1);
327+
stop();
328+
expect(pending.size).toBe(0);
329+
});

‎packages/core/src/runtime/pausedVideoCallbacks.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,3 +129,43 @@ export function releasePausedVideoCallbacks(video: HTMLVideoElement): void {
129129
pending.native = requestNative(owner, video, state, pending, () => id);
130130
}
131131
}
132+
133+
/** Recovery must observe native frames even while authored callbacks still see staged pixels. */
134+
export function observeNativeVideoFrames(
135+
video: HTMLVideoElement,
136+
onFrame: () => boolean,
137+
): () => void {
138+
const owner = registry();
139+
const request = owner?.request ?? video.requestVideoFrameCallback;
140+
const cancel = owner?.cancel ?? video.cancelVideoFrameCallback;
141+
let handle: number | null = null;
142+
let stopped = false;
143+
let playingAt: number | null = video.paused ? null : video.currentTime;
144+
const stop = () => {
145+
if (stopped) return;
146+
stopped = true;
147+
if (handle !== null && typeof cancel === "function") cancel.call(video, handle);
148+
video.removeEventListener("playing", playing);
149+
video.removeEventListener("timeupdate", progress);
150+
};
151+
const arm = () => {
152+
if (!stopped && typeof request === "function") handle = request.call(video, presented);
153+
};
154+
const presented = () => {
155+
handle = null;
156+
if (stopped) return;
157+
if (onFrame()) stop();
158+
else arm();
159+
};
160+
const playing = () => {
161+
playingAt = video.currentTime;
162+
};
163+
const progress = () => {
164+
if (stopped || playingAt === null || video.paused || video.currentTime === playingAt) return;
165+
if (onFrame()) stop();
166+
};
167+
video.addEventListener("playing", playing);
168+
video.addEventListener("timeupdate", progress);
169+
arm();
170+
return stop;
171+
}

‎packages/core/src/runtime/pausedVideoFrames.test.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,3 +206,51 @@ it("holds a full-size native snapshot through original-source provider replaceme
206206
expect(ownsPausedVideoTime(video)).toBe(true);
207207
expect(video.currentTime).toBe(3);
208208
});
209+
210+
it("cancels a retained native observer when a new paused picture replaces it", async () => {
211+
const proto = HTMLVideoElement.prototype;
212+
const original = Object.getOwnPropertyDescriptor(proto, "requestVideoFrameCallback");
213+
const originalCancel = Object.getOwnPropertyDescriptor(proto, "cancelVideoFrameCallback");
214+
let queued: VideoFrameRequestCallback = () => {};
215+
const cancel = vi.fn();
216+
Object.defineProperty(proto, "requestVideoFrameCallback", {
217+
configurable: true,
218+
writable: true,
219+
value: (callback: VideoFrameRequestCallback) => {
220+
queued = callback;
221+
return 123;
222+
},
223+
});
224+
Object.defineProperty(proto, "cancelVideoFrameCallback", {
225+
configurable: true,
226+
writable: true,
227+
value: cancel,
228+
});
229+
try {
230+
const { session, video, request } = fixture();
231+
const first = await session.prepare(request, new AbortController().signal);
232+
first!.present();
233+
session.prepareNative();
234+
session.clear(new Set([video]), () => false);
235+
const replacement = await session.prepare(request, new AbortController().signal);
236+
replacement!.present();
237+
const picture = getPausedVideoFrame(video);
238+
expect(cancel).toHaveBeenCalledWith(123);
239+
queued(0, {
240+
mediaTime: 3,
241+
width: 64,
242+
height: 36,
243+
presentedFrames: 1,
244+
presentationTime: 0,
245+
expectedDisplayTime: 0,
246+
processingDuration: 0,
247+
});
248+
expect(getPausedVideoFrame(video)).toBe(picture);
249+
expect(ownsPausedVideoTime(video)).toBe(true);
250+
} finally {
251+
if (original) Object.defineProperty(proto, "requestVideoFrameCallback", original);
252+
else Reflect.deleteProperty(proto, "requestVideoFrameCallback");
253+
if (originalCancel) Object.defineProperty(proto, "cancelVideoFrameCallback", originalCancel);
254+
else Reflect.deleteProperty(proto, "cancelVideoFrameCallback");
255+
}
256+
});

0 commit comments

Comments
 (0)