Skip to content

Commit faa7546

Browse files
committed
feat(studio): scrub full-quality video with decoded preview frames
1 parent 9d5b242 commit faa7546

49 files changed

Lines changed: 3184 additions & 104 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎bun.lock‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/packages/studio.mdx‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,18 @@ depends on the browser playing in real time, while render captures frames one at
106106
time. A heavy project can therefore stutter during preview and still render every
107107
frame.
108108

109+
Paused video scrubbing decodes and stages the visible video layers before
110+
advancing the composition's animations. Completed frames appear during a drag,
111+
with newer pointer positions coalesced into the next decode. Canvas and GPU
112+
effects consume the same staged pixels. Continuous playback uses native video;
113+
Studio keeps the paused picture visible while those videos catch up before Play.
114+
Scrubbing and paused frame steps decode the original video at source resolution
115+
throughout. Automatic playback quality reductions do not reduce these frames,
116+
and the frame decoder does not substitute re-encoded seek copies. Playback
117+
quality settings do not reduce these paused frames. Browsers or
118+
media that cannot use the frame decoder fall back to native seeking. Long-GOP
119+
sources can take longer to seek, especially backward or across distant cuts.
120+
109121
## Tailwind preset
110122

111123
```typescript

‎packages/core/src/index.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,11 @@ export {
308308
type PreviewQuality,
309309
type PreviewSourceFacts,
310310
} from "./previewProxyBox";
311+
export type {
312+
PausedVideoFrameProvider,
313+
PausedVideoFrameRequest,
314+
PausedVideoFrame,
315+
} from "./runtime/pausedVideoFrames";
311316

312317
// Inline scripts
313318
export {

‎packages/core/src/previewProxyBox.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ export type PreviewSourceFacts = {
5555
bitDepth: number;
5656
bitrate: number;
5757
codec?: string;
58+
frameRate?: number;
59+
variableFrameRate?: boolean;
5860
};
5961

6062
export const HEAVY_PIXEL_SHARE = 0.5;
Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
import { afterEach, expect, it, vi } from "vitest";
2+
import { createPausedVideoFrames, getPausedVideoFrame } from "../pausedVideoFrames";
3+
import {
4+
patchCanvasVideoFrameCompat,
5+
patchExternalVideoTextureCompat,
6+
patchVideoTextureCompat,
7+
patchWebGLVideoTextureCompat,
8+
} from "./video-texture-compat";
9+
10+
const sessions: ReturnType<typeof createPausedVideoFrames>[] = [];
11+
afterEach(() => {
12+
for (const session of sessions.splice(0)) session.destroy();
13+
document.body.innerHTML = "";
14+
vi.restoreAllMocks();
15+
vi.unstubAllGlobals();
16+
});
17+
18+
function surfaces() {
19+
const draw = vi.fn();
20+
const offscreenDraw = vi.fn();
21+
const image = vi.fn();
22+
const subImage = vi.fn();
23+
const copy = vi.fn();
24+
const imported = vi.fn((_descriptor: unknown) => ({ texture: true }));
25+
const constructed = vi.fn();
26+
class Context2D {
27+
drawImage(...args: unknown[]) {
28+
draw.apply(this, args);
29+
}
30+
}
31+
class Offscreen2D {
32+
drawImage(...args: unknown[]) {
33+
offscreenDraw.apply(this, args);
34+
}
35+
}
36+
class GL {
37+
texImage2D(...args: unknown[]) {
38+
image.apply(this, args);
39+
}
40+
texSubImage2D(...args: unknown[]) {
41+
subImage.apply(this, args);
42+
}
43+
}
44+
class GL2 extends GL {}
45+
class Queue {
46+
copyExternalImageToTexture(...args: unknown[]) {
47+
copy.apply(this, args);
48+
}
49+
}
50+
class Device {
51+
importExternalTexture(descriptor: unknown) {
52+
return imported.call(this, descriptor);
53+
}
54+
}
55+
class Frame {
56+
close = vi.fn();
57+
constructor(source: unknown, options: unknown) {
58+
constructed(source, options);
59+
}
60+
}
61+
vi.stubGlobal("CanvasRenderingContext2D", Context2D);
62+
vi.stubGlobal("OffscreenCanvasRenderingContext2D", Offscreen2D);
63+
vi.stubGlobal("WebGLRenderingContext", GL);
64+
vi.stubGlobal("WebGL2RenderingContext", GL2);
65+
vi.stubGlobal("GPUQueue", Queue);
66+
vi.stubGlobal("GPUDevice", Device);
67+
vi.stubGlobal("VideoFrame", Frame);
68+
vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
69+
clearRect: vi.fn(),
70+
drawImage: vi.fn(),
71+
} as unknown as CanvasRenderingContext2D);
72+
return {
73+
Context2D,
74+
Offscreen2D,
75+
GL,
76+
GL2,
77+
Queue,
78+
Device,
79+
draw,
80+
offscreenDraw,
81+
image,
82+
subImage,
83+
copy,
84+
imported,
85+
constructed,
86+
};
87+
}
88+
89+
async function presented() {
90+
const session = createPausedVideoFrames();
91+
sessions.push(session);
92+
const video = document.createElement("video");
93+
video.src = "/original.mp4";
94+
document.body.append(video);
95+
Object.defineProperties(video, {
96+
videoWidth: { value: 1920 },
97+
videoHeight: { value: 1080 },
98+
});
99+
const source = document.createElement("canvas");
100+
source.width = 320;
101+
source.height = 180;
102+
session.setProvider({
103+
decode: async (requests) =>
104+
requests.map(({ time }) => ({
105+
source,
106+
timestamp: time,
107+
duration: 1 / 30,
108+
width: 1920,
109+
height: 1080,
110+
release: vi.fn(),
111+
})),
112+
dispose: vi.fn(),
113+
});
114+
const show = async (time: number) => {
115+
const batch = await session.prepare(
116+
[{ video, source: video.src, time }],
117+
new AbortController().signal,
118+
);
119+
expect(batch?.present()).toBe(true);
120+
batch!.refresh();
121+
};
122+
await show(3);
123+
const canvas = getPausedVideoFrame(video)!;
124+
expect([canvas.width, canvas.height]).toEqual([video.videoWidth, video.videoHeight]);
125+
return { session, video, canvas, show };
126+
}
127+
128+
it.each([
129+
["three arguments", [11, 17]],
130+
["five arguments", [11, 17, 640, 360]],
131+
["nine arguments", [120, 60, 960, 540, 11, 17, 640, 360]],
132+
])("preserves %s and source coordinates for both canvas surfaces", async (_name, coordinates) => {
133+
const { Context2D, Offscreen2D, draw, offscreenDraw } = surfaces();
134+
const { session, video, canvas } = await presented();
135+
patchCanvasVideoFrameCompat();
136+
patchCanvasVideoFrameCompat();
137+
const context = new Context2D();
138+
const offscreen = new Offscreen2D();
139+
context.drawImage(video, ...coordinates);
140+
offscreen.drawImage(video, ...coordinates);
141+
expect(draw).toHaveBeenCalledExactlyOnceWith(canvas, ...coordinates);
142+
expect(offscreenDraw).toHaveBeenCalledExactlyOnceWith(canvas, ...coordinates);
143+
expect(draw.mock.contexts[0]).toBe(context);
144+
expect(offscreenDraw.mock.contexts[0]).toBe(offscreen);
145+
session.clear();
146+
context.drawImage(video, ...coordinates);
147+
expect(draw).toHaveBeenLastCalledWith(video, ...coordinates);
148+
const image = document.createElement("img");
149+
offscreen.drawImage(image, ...coordinates);
150+
expect(offscreenDraw).toHaveBeenLastCalledWith(image, ...coordinates);
151+
});
152+
153+
it("uploads paused pixels through both WebGL versions without changing crop or upload parameters", async () => {
154+
const { GL, GL2, image, subImage } = surfaces();
155+
const { session, video, canvas } = await presented();
156+
patchWebGLVideoTextureCompat();
157+
for (const Context of [GL, GL2]) {
158+
const gl = new Context();
159+
const imageArgs = [0x0de1, 2, 0x1908, 0x1908, 0x1401];
160+
const subArgs = [0x0de1, 2, 13, 29, 0x1908, 0x1401];
161+
gl.texImage2D(...imageArgs, video);
162+
gl.texSubImage2D(...subArgs, video);
163+
expect(image).toHaveBeenLastCalledWith(...imageArgs, canvas);
164+
expect(subImage).toHaveBeenLastCalledWith(...subArgs, canvas);
165+
expect(image.mock.contexts.at(-1)).toBe(gl);
166+
const pixels = new Uint8Array([10, 20, 30, 255]);
167+
gl.texImage2D(0x0de1, 0, 0x1908, 1, 1, 0, 0x1908, 0x1401, pixels);
168+
expect(image).toHaveBeenLastCalledWith(0x0de1, 0, 0x1908, 1, 1, 0, 0x1908, 0x1401, pixels);
169+
}
170+
session.clear();
171+
new GL().texImage2D(0x0de1, 0, 0x1908, 0x1908, 0x1401, video);
172+
expect(image.mock.lastCall?.at(-1)).toBe(video);
173+
});
174+
175+
it("preserves WebGPU crop, orientation, destination and full intrinsic copy extent", async () => {
176+
const { Queue, copy } = surfaces();
177+
const { session, video, canvas } = await presented();
178+
patchVideoTextureCompat();
179+
const queue = new Queue();
180+
const source = { source: video, origin: { x: 37, y: 41 }, flipY: true };
181+
const destination = { texture: {}, origin: [5, 7], mipLevel: 1, premultipliedAlpha: true };
182+
const extent = {
183+
width: video.videoWidth - 37,
184+
height: video.videoHeight - 41,
185+
depthOrArrayLayers: 1,
186+
};
187+
queue.copyExternalImageToTexture(source, destination, extent);
188+
expect(copy).toHaveBeenCalledExactlyOnceWith({ ...source, source: canvas }, destination, extent);
189+
expect(copy.mock.contexts[0]).toBe(queue);
190+
expect(source.source).toBe(video);
191+
expect(copy.mock.calls[0]?.[1]).toBe(destination);
192+
expect(copy.mock.calls[0]?.[2]).toBe(extent);
193+
session.clear();
194+
queue.copyExternalImageToTexture(source, destination, extent);
195+
expect(copy).toHaveBeenLastCalledWith(source, destination, extent);
196+
expect(copy.mock.lastCall?.[0]).toBe(source);
197+
});
198+
199+
it("reuses an external VideoFrame until presentation changes, then closes it on replacement and clear", async () => {
200+
const { Device, imported, constructed } = surfaces();
201+
const { session, video, canvas, show } = await presented();
202+
patchExternalVideoTextureCompat();
203+
const once = Device.prototype.importExternalTexture;
204+
patchExternalVideoTextureCompat();
205+
expect(Device.prototype.importExternalTexture).toBe(once);
206+
const device = new Device();
207+
const descriptor = { source: video, colorSpace: "srgb", label: "authored texture" };
208+
const texture = device.importExternalTexture(descriptor);
209+
const first = imported.mock.lastCall?.[0] as {
210+
source: { close: ReturnType<typeof vi.fn> };
211+
colorSpace: string;
212+
label: string;
213+
};
214+
expect(texture).toEqual({ texture: true });
215+
expect(constructed).toHaveBeenCalledExactlyOnceWith(canvas, { timestamp: 3_000_000 });
216+
expect(first).toEqual({ ...descriptor, source: first.source });
217+
expect(first.source).not.toBe(video);
218+
expect(imported.mock.contexts[0]).toBe(device);
219+
expect(descriptor.source).toBe(video);
220+
device.importExternalTexture(descriptor);
221+
expect(imported.mock.lastCall?.[0]).toEqual(first);
222+
expect(constructed).toHaveBeenCalledTimes(1);
223+
expect(first.source.close).not.toHaveBeenCalled();
224+
await show(4);
225+
expect(first.source.close).toHaveBeenCalledTimes(1);
226+
device.importExternalTexture(descriptor);
227+
const second = imported.mock.lastCall?.[0] as typeof first;
228+
expect(second.source).not.toBe(first.source);
229+
expect(constructed).toHaveBeenLastCalledWith(getPausedVideoFrame(video), {
230+
timestamp: 4_000_000,
231+
});
232+
session.clear();
233+
expect(second.source.close).toHaveBeenCalledTimes(1);
234+
device.importExternalTexture(descriptor);
235+
expect(imported).toHaveBeenLastCalledWith(descriptor);
236+
expect(imported.mock.lastCall?.[0]).toBe(descriptor);
237+
expect(constructed).toHaveBeenCalledTimes(2);
238+
});

‎packages/core/src/runtime/adapters/video-texture-compat.ts‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import { findInjectedRenderFrame } from "../renderFrameSibling.js";
1717
import { isImageElement, isVideoElement } from "../domRealm";
18+
import { getPausedExternalVideoFrame, getPausedVideoFrame } from "../pausedVideoFrames";
1819

1920
/**
2021
* Resolve the decoded render-frame `<img>` for a source `<video>`, if the
@@ -26,7 +27,9 @@ import { isImageElement, isVideoElement } from "../domRealm";
2627
* first (cheap) and fall back to an id lookup in case a node was inserted
2728
* between them.
2829
*/
29-
function resolveRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | null {
30+
function resolveRenderFrameImage(
31+
video: HTMLVideoElement,
32+
): HTMLImageElement | HTMLCanvasElement | null {
3033
const sibling = video.nextElementSibling;
3134
if (
3235
isImageElement(sibling) &&
@@ -40,7 +43,48 @@ function resolveRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | nu
4043
if (byId && byId.complete && byId.naturalWidth > 0) {
4144
return byId;
4245
}
43-
return null;
46+
return getPausedVideoFrame(video);
47+
}
48+
49+
export function patchCanvasVideoFrameCompat(): void {
50+
for (const ctor of [
51+
globalThis.CanvasRenderingContext2D,
52+
globalThis.OffscreenCanvasRenderingContext2D,
53+
]) {
54+
if (!ctor?.prototype?.drawImage) continue;
55+
const original = ctor.prototype.drawImage as ((...args: unknown[]) => void) & {
56+
__hfVideoPatched?: boolean;
57+
};
58+
if (original.__hfVideoPatched) continue;
59+
const patched = function (this: unknown, ...args: unknown[]) {
60+
if (isVideoElement(args[0])) args[0] = getPausedVideoFrame(args[0]) ?? args[0];
61+
original.apply(this, args);
62+
};
63+
patched.__hfVideoPatched = true;
64+
ctor.prototype.drawImage = patched;
65+
}
66+
}
67+
68+
export function patchExternalVideoTextureCompat(): void {
69+
const ctor = (globalThis as Record<string, unknown>).GPUDevice as
70+
| {
71+
prototype: {
72+
importExternalTexture?: ((source: Record<string, unknown>) => unknown) & {
73+
__hfVideoPatched?: boolean;
74+
};
75+
};
76+
}
77+
| undefined;
78+
const original = ctor?.prototype.importExternalTexture;
79+
if (!ctor || typeof original !== "function" || original.__hfVideoPatched) return;
80+
const patched = function (this: unknown, descriptor: Record<string, unknown>) {
81+
const frame = isVideoElement(descriptor.source)
82+
? getPausedExternalVideoFrame(descriptor.source)
83+
: null;
84+
return original.call(this, frame ? { ...descriptor, source: frame } : descriptor);
85+
};
86+
patched.__hfVideoPatched = true;
87+
ctor.prototype.importExternalTexture = patched;
4488
}
4589

4690
export function patchVideoTextureCompat(): void {

‎packages/core/src/runtime/colorGrading.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ import { mediaElementTime } from "./mediaClock";
4949
import { swallow } from "./diagnostics";
5050
import { createRecordFilter } from "./mutationRecords";
5151
import { findInjectedRenderFrame } from "./renderFrameSibling";
52+
import { getPausedVideoFrame } from "./pausedVideoFrames";
5253
import {
5354
isCanvasElement,
5455
isElementNode,
@@ -2807,6 +2808,8 @@ function getDrawableSource(element: ColorGradingMediaElement): TexImageSource |
28072808
if (isVideoElement(element)) {
28082809
const renderFrame = findRenderFrameImage(element);
28092810
if (renderFrame) return renderFrame;
2811+
const previewFrame = getPausedVideoFrame(element);
2812+
if (previewFrame) return previewFrame;
28102813
}
28112814
return isDrawableSource(element) ? element : null;
28122815
}
@@ -3542,7 +3545,7 @@ function placeCanvas(
35423545
layout: { width: number; height: number };
35433546
uv: ReturnType<typeof calculateObjectFitUv>;
35443547
} | null {
3545-
const styleSource = isHtmlElement(source) ? source : entry.element;
3548+
const styleSource = isRenderFrameImage(source) ? source : entry.element;
35463549
const layout = updateCanvasLayout(entry, styleSource);
35473550
if (!layout) return null;
35483551
const style = window.getComputedStyle(styleSource);

0 commit comments

Comments
 (0)