Skip to content

Commit 9d5b242

Browse files
committed
fix(core): retain complete preview pictures during native seeks
1 parent b418bb2 commit 9d5b242

9 files changed

Lines changed: 606 additions & 18 deletions

File tree

‎docs/packages/player.mdx‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,10 @@ console.log(player.duration, player.paused, player.ready);
100100
console.log(player.compositionWidth, player.compositionHeight);
101101
```
102102

103+
In Studio previews, scrubbing keeps the last complete picture visible while video
104+
layers decode at the requested time. Rapid seeks keep only the latest queued
105+
destination, and the captions and video layers appear together once it is ready.
106+
103107
## Events
104108

105109
```js

‎packages/core/src/runtime/colorGrading.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// fallow-ignore-file code-duplication
22
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
33
import { mediaElementTime } from "./mediaClock";
4+
import { PREVIEW_SEEK_HOLD_ATTR } from "../studioPreviewMark";
45
import {
56
HF_COLOR_GRADING_ATTR,
67
HF_COLOR_GRADING_EFFECT_KEYS,
@@ -352,6 +353,26 @@ describe("createColorGradingRuntime", () => {
352353
expect(texImage2DCalls.length).toBeGreaterThan(drawsBefore);
353354
});
354355

356+
it("keeps graded layers on the committed picture until a preview seek lands", () => {
357+
const { video } = startRuntimeWithVideo();
358+
const drawsBefore = texImage2DCalls.length;
359+
const canvas = document.querySelector("[data-hf-color-grading-canvas]");
360+
document.documentElement.setAttribute(PREVIEW_SEEK_HOLD_ATTR, "");
361+
try {
362+
video.dispatchEvent(new Event("seeked"));
363+
video.style.setProperty("--hf-color-grading-exposure", "1");
364+
runtime?.refresh();
365+
expect(runtime?.redraw()).toBe(0);
366+
expect(texImage2DCalls).toHaveLength(drawsBefore);
367+
expect(document.querySelector("[data-hf-color-grading-canvas]")).toBe(canvas);
368+
expect(canvas?.isConnected).toBe(true);
369+
} finally {
370+
document.documentElement.removeAttribute(PREVIEW_SEEK_HOLD_ATTR);
371+
}
372+
expect(runtime?.redraw()).toBe(1);
373+
expect(texImage2DCalls.length).toBeGreaterThan(drawsBefore);
374+
});
375+
355376
it("does not redraw-loop on its own hide writes (opacity/visibility only)", async () => {
356377
const { video } = startRuntimeWithVideo();
357378
await new Promise((resolve) => requestAnimationFrame(() => resolve(null)));

‎packages/core/src/runtime/colorGrading.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { wallClearTimeout, wallSetTimeout } from "./wallClock";
2+
import { PREVIEW_SEEK_HOLD_ATTR } from "../studioPreviewMark";
23
import {
34
HF_COLOR_GRADING_ADJUST_KEYS,
45
HF_COLOR_GRADING_ANIMATABLE_PROPERTIES,
@@ -3419,6 +3420,7 @@ type DrawableFrame = {
34193420
};
34203421

34213422
function drawableFrame(entry: ColorGradingEntry): DrawableFrame | null {
3423+
if (document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)) return null;
34223424
const source = getDrawableSource(entry.element);
34233425
const pixels = source && readablePixels(source, entry);
34243426
if (!source || !pixels) {

‎packages/core/src/runtime/entry.test.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,34 @@ describe("runtime entry", () => {
316316
await window.__hfWaitForSeekCompletion?.();
317317
});
318318

319+
it("resolves a mixed image/video seek only after both have decoded", async () => {
320+
servePreview();
321+
const root = mountRoot();
322+
const current = timed(root, "div", "0");
323+
const later = timed(root, "div", "5");
324+
const image = later.appendChild(document.createElement("img"));
325+
let decodeImage = () => {};
326+
image.decode = () => new Promise<void>((resolve) => (decodeImage = resolve));
327+
const video = timed(later, "video", "5");
328+
video.src = "/movie.mp4";
329+
let ready = 1;
330+
Object.defineProperty(video, "readyState", { configurable: true, get: () => ready });
331+
Object.defineProperty(video, "networkState", { configurable: true, value: 2 });
332+
await evaluateRuntime();
333+
let settled = false;
334+
const landed = Promise.resolve(window.__player!.seek(5.5)).then(() => {
335+
settled = true;
336+
});
337+
decodeImage();
338+
await new Promise((resolve) => setTimeout(resolve, 0));
339+
expect(settled).toBe(false);
340+
expect(visibility(current, later)).toEqual(["visible", "hidden"]);
341+
ready = 4;
342+
video.dispatchEvent(new Event("seeked"));
343+
await landed;
344+
expect(visibility(current, later)).toEqual(["hidden", "visible"]);
345+
});
346+
319347
it("applies a held jump after the cap when an image never decodes", async () => {
320348
servePreview();
321349
const root = mountRoot();
@@ -495,6 +523,9 @@ describe("runtime entry", () => {
495523
]);
496524

497525
window.__player?.seek(3.5);
526+
const video = second![0]!;
527+
Object.defineProperty(video, "readyState", { value: 4, configurable: true });
528+
video.dispatchEvent(new Event("seeked"));
498529
expect([first, second, third].map((take) => armed(...take))).toEqual([
499530
[false, false],
500531
[true, true],
Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
import { afterEach, expect, it, vi } from "vitest";
2+
import { initSandboxRuntimeModular } from "./init";
3+
import { createMockTimeline, resetRuntimeFixtureDom } from "./runtimeSeekFixture.test-helpers";
4+
import { PREVIEW_SEEK_HOLD_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
5+
6+
function decoder(video: HTMLVideoElement, initiallyReady: boolean) {
7+
let time = 0;
8+
let ready = initiallyReady ? 4 : 0;
9+
let seeking = false;
10+
const seeks: number[] = [];
11+
Object.defineProperties(video, {
12+
duration: { value: 10, configurable: true },
13+
videoWidth: { value: 64, configurable: true },
14+
videoHeight: { value: 36, configurable: true },
15+
readyState: { get: () => ready, configurable: true },
16+
networkState: { value: 2, configurable: true },
17+
paused: { value: true, configurable: true },
18+
seeking: { get: () => seeking, configurable: true },
19+
currentTime: {
20+
get: () => time,
21+
set: (next: number) => {
22+
time = next;
23+
seeks.push(next);
24+
seeking = true;
25+
ready = 1;
26+
},
27+
configurable: true,
28+
},
29+
});
30+
video.pause = vi.fn();
31+
video.play = vi.fn(() => Promise.resolve());
32+
video.load = vi.fn();
33+
return {
34+
video,
35+
seeks,
36+
land: () => {
37+
ready = 4;
38+
seeking = false;
39+
video.dispatchEvent(new Event("seeked"));
40+
},
41+
reload: () => {
42+
time = 0;
43+
ready = 1;
44+
seeking = false;
45+
video.dispatchEvent(new Event("loadedmetadata"));
46+
},
47+
};
48+
}
49+
50+
function film() {
51+
resetRuntimeFixtureDom();
52+
const meta = document.createElement("meta");
53+
meta.name = STUDIO_PREVIEW_MARK_META;
54+
document.head.append(meta);
55+
vi.spyOn(window, "requestAnimationFrame").mockReturnValue(0);
56+
const drawImage = vi.fn();
57+
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
58+
drawImage,
59+
} as unknown as CanvasRenderingContext2D);
60+
document.body.innerHTML =
61+
'<div data-composition-id="main" data-root="true" data-duration="10">' +
62+
[0, 5]
63+
.flatMap((start) =>
64+
["plate", "matte"].map(
65+
(name) =>
66+
`<video id="${name}-${start}" style="width:64px;height:36px" data-start="${start}" data-duration="5" src="/${name}.mp4" muted></video>`,
67+
),
68+
)
69+
.join("") +
70+
"</div>";
71+
const videos = [...document.querySelectorAll("video")].map((v) =>
72+
decoder(v, v.dataset.start === "0"),
73+
);
74+
const main = createMockTimeline(10);
75+
window.__timelines = { main };
76+
initSandboxRuntimeModular();
77+
for (const video of videos.slice(0, 2)) video.land();
78+
return { main, old: videos.slice(0, 2), next: videos.slice(2), drawImage };
79+
}
80+
81+
it("keeps the old picture and page until both destination videos have decoded", async () => {
82+
const { main, old, next, drawImage } = film();
83+
const landed = window.__player!.seek(5.5);
84+
expect(window.__player!.getTime()).toBe(5.5);
85+
expect(main.time?.()).toBe(0);
86+
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
87+
expect(drawImage).toHaveBeenCalledTimes(2);
88+
expect(document.querySelectorAll("canvas")).toHaveLength(2);
89+
expect(old.every(({ video }) => getComputedStyle(video).display === "none")).toBe(true);
90+
next[0]!.land();
91+
expect(main.time?.()).toBe(0);
92+
next[1]!.land();
93+
await landed;
94+
expect(main.time?.()).toBe(5.5);
95+
expect(document.querySelectorAll("canvas")).toHaveLength(0);
96+
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
97+
expect(old.every(({ video }) => video.style.visibility === "hidden")).toBe(true);
98+
expect(next.every(({ video }) => video.style.visibility === "visible")).toBe(true);
99+
});
100+
101+
it("coalesces requests without restarting the in-flight decoders or painting stale targets", async () => {
102+
const { main, next } = film();
103+
const landed = window.__player!.seek(5.5);
104+
const initialSeeks = next.map((v) => v.seeks.length);
105+
window.__player!.seek(6);
106+
window.__player!.seek(7.5);
107+
expect(next.map((v) => v.seeks.length)).toEqual(initialSeeks);
108+
for (const video of next) video.land();
109+
expect(main.time?.()).toBe(0);
110+
expect(next.every((v) => Math.abs(v.video.currentTime - 2.5) < 0.001)).toBe(true);
111+
for (const video of next) video.land();
112+
await landed;
113+
expect(main.time?.()).toBe(7.5);
114+
});
115+
116+
it("holds overlapping same-clip seeks too, and repositions a reloaded source at the requested time", async () => {
117+
const { main, old } = film();
118+
const landed = window.__player!.seek(2);
119+
old[0]!.reload();
120+
expect(old[0]!.video.currentTime).toBeCloseTo(2, 3);
121+
old[0]!.land();
122+
expect(main.time?.()).toBe(0);
123+
old[1]!.land();
124+
await landed;
125+
expect(main.time?.()).toBe(2);
126+
});
127+
128+
it("holds a paused keepPlaying seek and normal seeks after a non-export renderSeek", async () => {
129+
const { main, old, next } = film();
130+
window.__player!.renderSeek(0.2);
131+
for (const video of old) video.land();
132+
const landed = window.__player!.seek(6, { keepPlaying: true });
133+
expect(main.time?.()).toBe(0.2);
134+
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
135+
for (const video of next) video.land();
136+
await landed;
137+
expect(main.time?.()).toBe(6);
138+
});
139+
140+
it("does not pause, seek or cover a video leased by another preview control", async () => {
141+
const { old, next } = film();
142+
const leased = old[0]!;
143+
window.__hf!.leasePausedMedia!(leased.video);
144+
Object.defineProperty(leased.video, "paused", { value: false, configurable: true });
145+
const seeks = leased.seeks.length;
146+
const landed = window.__player!.seek(6);
147+
expect(leased.video.pause).not.toHaveBeenCalled();
148+
expect(leased.seeks).toHaveLength(seeks);
149+
expect(leased.video.hasAttribute("data-hf-seek-video")).toBe(false);
150+
for (const video of next) video.land();
151+
await landed;
152+
expect(leased.video.pause).not.toHaveBeenCalled();
153+
});
154+
155+
it("leaves host-controlled native media to the host", () => {
156+
const { old } = film();
157+
window.dispatchEvent(
158+
new MessageEvent("message", {
159+
source: window,
160+
data: {
161+
source: "hf-parent",
162+
type: "control",
163+
action: "set-native-media-sync-disabled",
164+
disabled: true,
165+
},
166+
}),
167+
);
168+
Object.defineProperty(old[0]!.video, "paused", { value: false, configurable: true });
169+
const seeks = old[0]!.seeks.length;
170+
window.__player!.seek(2);
171+
expect(old[0]!.seeks).toHaveLength(seeks);
172+
expect(old[0]!.video.pause).not.toHaveBeenCalled();
173+
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
174+
});
175+
176+
it("leaves a grading canvas's hidden source in its layout during the hold", async () => {
177+
const { old, next, drawImage } = film();
178+
old[0]!.video.setAttribute("data-hf-color-grading-source-hidden", "true");
179+
old[0]!.video.style.setProperty("opacity", "0", "important");
180+
const landed = window.__player!.seek(6);
181+
expect(old[0]!.video.hasAttribute("data-hf-seek-video")).toBe(false);
182+
expect(drawImage).toHaveBeenCalledTimes(1);
183+
for (const video of next) video.land();
184+
await landed;
185+
});
186+
187+
it.each(["play", "render", "teardown"])(
188+
"cleans up a pending seek on %s and ignores late completions",
189+
async (action) => {
190+
const { main, next } = film();
191+
const player = window.__player!;
192+
const landed = player.seek(6);
193+
if (action === "play") player.play();
194+
else if (action === "render") player.renderSeek(2);
195+
else window.__hfRuntimeTeardown?.();
196+
const time = main.time?.();
197+
for (const video of next) video.land();
198+
await landed;
199+
expect(main.time?.()).toBe(time);
200+
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
201+
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
202+
if (action === "play") expect(player.getTime()).toBeGreaterThanOrEqual(6);
203+
if (action === "render") expect(time).toBe(2);
204+
},
205+
);
206+
207+
it("caps a stalled decoder without leaving the picture hidden", async () => {
208+
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
209+
const { main } = film();
210+
const landed = window.__player!.seek(6);
211+
await vi.advanceTimersByTimeAsync(5000);
212+
await landed;
213+
expect(main.time?.()).toBe(6);
214+
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
215+
expect(document.querySelectorAll("canvas, [data-hf-seek-video]")).toHaveLength(0);
216+
});
217+
218+
afterEach(() => {
219+
window.__hfRuntimeTeardown?.();
220+
document.querySelector(`meta[name="${STUDIO_PREVIEW_MARK_META}"]`)?.remove();
221+
vi.restoreAllMocks();
222+
vi.useRealTimers();
223+
resetRuntimeFixtureDom();
224+
});

‎packages/core/src/runtime/init.swapScenes.test.ts‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import { probeAndCacheElementVolume } from "./mediaVolumeEnvelope.js";
99
import { wrapScopedCompositionScript } from "../compiler/compositionScoping";
1010
import { AFTER_FONTS_SCRIPT_TYPE } from "../compiler/scriptRuns";
1111
import { parkFarScenesAtBoot } from "./sceneWindow";
12+
import { PREVIEW_SEEK_HOLD_ATTR, STUDIO_PREVIEW_MARK_META } from "../studioPreviewMark";
1213

1314
vi.mock("./mediaVolumeEnvelope.js", async (importOriginal) => {
1415
const actual = await importOriginal<typeof import("./mediaVolumeEnvelope.js")>();
@@ -1767,9 +1768,13 @@ describe("the preview's scene window", () => {
17671768
el.getAttribute("data-hf-scene"),
17681769
);
17691770
// As the runtime's entry does: park before any scene script runs, then run the rest.
1770-
function bootWindowed(scenes: Scene[], root: Tl, withManifest = true) {
1771+
function bootWindowed(scenes: Scene[], root: Tl, withManifest = true, studioPreview = false) {
17711772
const { head, body } = preview(scenes);
17721773
document.head.innerHTML = withManifest ? head : head.replace(/<meta[^>]*>/, "");
1774+
if (studioPreview)
1775+
document.head.append(
1776+
Object.assign(document.createElement("meta"), { name: STUDIO_PREVIEW_MARK_META }),
1777+
);
17731778
document.body.innerHTML = body;
17741779
window.__timelines = { main: root };
17751780
parkFarScenesAtBoot();
@@ -1868,6 +1873,38 @@ describe("the preview's scene window", () => {
18681873
expect(parked()).toEqual(["s2"]);
18691874
});
18701875

1876+
it("holds a parked destination until its videos mount and decode without retiring the old picture", async () => {
1877+
const ready = new WeakSet<HTMLMediaElement>();
1878+
vi.spyOn(HTMLMediaElement.prototype, "readyState", "get").mockImplementation(function () {
1879+
return ready.has(this) ? 4 : 1;
1880+
});
1881+
vi.spyOn(HTMLMediaElement.prototype, "networkState", "get").mockReturnValue(2);
1882+
vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation(() => {});
1883+
vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(() => {});
1884+
const { root } = trackingRoot();
1885+
bootWindowed(
1886+
[S0, S1, { ...S2, body: '<video src="/late.mp4" data-start="0" data-duration="2"></video>' }],
1887+
root,
1888+
true,
1889+
true,
1890+
);
1891+
await tick();
1892+
const landed = window.__player!.seek(5);
1893+
await vi.waitFor(() => {
1894+
expect(sceneHost("s2").querySelector("video")?.currentTime).toBeCloseTo(1, 3);
1895+
});
1896+
expect(root.time?.()).toBe(0);
1897+
expect(window.__player!.getTime()).toBe(5);
1898+
expect(parked()).not.toContain("s0");
1899+
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(true);
1900+
const video = sceneHost("s2").querySelector("video")!;
1901+
ready.add(video);
1902+
video.dispatchEvent(new Event("seeked"));
1903+
await landed;
1904+
expect(root.time?.()).toBe(5);
1905+
expect(document.documentElement.hasAttribute(PREVIEW_SEEK_HOLD_ATTR)).toBe(false);
1906+
});
1907+
18711908
it("waits for the scene a seek lands in while its mount as a neighbour is still running", async () => {
18721909
bootWindowed([S0, S1, S2], trackingRoot().root);
18731910
await tick();

0 commit comments

Comments
 (0)