Skip to content

Commit 3b89bfe

Browse files
fix(engine): motion blur works over video by holding each video frame across samples (#5150)
* fix(engine): motion blur works over video by holding each video frame across samples Each output frame's injected video frame is now held for every sub-frame sample and probe of that frame, so footage stays as shot and only the layers over it blur. The refusal for sessions with a before-capture hook is removed. Fixes #5144 * fix(engine): trim held-video comments, pin the new test-classification hash * fix(engine): a blurred frame at a sub-composition cut keeps its video Each sample now decides which videos show at its own time, like every other layer, and only a video on screen at the output frame's time keeps that frame's picture. Before, the visible set was taken at the frame's time while sub-composition hosts were hidden at the sample's time, so the samples just before a cut showed no video and the cut frame dipped toward black. The before-capture hook takes the held time as an optional third argument, and the early injection is gone: the GPU re-render now always runs at the page's own time. * style(engine): one doc comment for the before-capture hook * fix(engine): a video animating its own style stays in step when blurred The injector copies a video's own style only when its frame changes, so the first injection of each blurred frame now runs at the frame's own time before the samples. A video sliding by its own transform no longer sits a quarter frame behind with blur on. * test(producer): a two-line header for the motion blur over video test * test(producer): reach the cut fixture by path, not a URL fallow reads as an import
1 parent 2d98d5c commit 3b89bfe

8 files changed

Lines changed: 384 additions & 21 deletions

File tree

‎packages/engine/src/services/frameCapture-gpuCompletion.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,9 @@ describe("WebGPU frame completion", () => {
3333

3434
it("runs the completion wait after video injection and before screenshot capture", () => {
3535
const source = readFileSync(new URL("./frameCapture.ts", import.meta.url), "utf8");
36-
const injection = source.indexOf("await session.onBeforeCapture(page, quantizedTime)");
36+
const injection = source.indexOf(
37+
"await session.onBeforeCapture(page, quantizedTime, heldVideoTime)",
38+
);
3739
const completion = source.indexOf("await waitForPendingSeekCompletion(page)");
3840
const screenshot = source.indexOf("async function captureFrameCore");
3941
expect(injection).toBeGreaterThan(-1);

‎packages/engine/src/services/frameCapture-motionBlur.test.ts‎

Lines changed: 53 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -317,10 +317,59 @@ describe("resolveSessionMotionBlur rejects what accumulation cannot render", ()
317317
expect(() => resolveSessionMotionBlur(session)).toThrow(/format "png"/);
318318
});
319319

320-
it("rejects injected video frames, which cannot follow a sub-frame seek", () => {
321-
expect(() =>
322-
resolveSessionMotionBlur(withOptions({ onBeforeCapture: async () => {} })),
323-
).toThrow(/video/);
320+
it("accepts a session that injects video frames (#5144)", () => {
321+
const plan = resolveSessionMotionBlur(withOptions({ onBeforeCapture: async () => {} }));
322+
expect(plan).not.toBeUndefined();
323+
});
324+
});
325+
326+
describe("injected video is held for the whole shutter window (#5144)", () => {
327+
function recordInjections() {
328+
const calls: Array<{ time: number; held: number | undefined; seeksBefore: number }> = [];
329+
const hook: CaptureSession["onBeforeCapture"] = async (_page, time, held) => {
330+
calls.push({ time, held, seeksBefore: seeks.length });
331+
};
332+
return { calls, hook };
333+
}
334+
const sampleSeekTimes = () =>
335+
seeks.filter((s) => s.subFrameDivisions !== undefined).map((s) => s.time);
336+
337+
it("injects at the frame time first, then each sample at its own time holding the frame", async () => {
338+
const { calls, hook } = recordInjections();
339+
340+
await captureFrameToBuffer(makeSession({ onBeforeCapture: hook }), 10, 10 / 30);
341+
342+
// The first call comes right after the eventful seek, before any sample seek.
343+
expect(calls[0]).toEqual({ time: 10 / 30, held: 10 / 30, seeksBefore: 1 });
344+
const sampleCalls = calls.slice(1);
345+
expect(sampleCalls.map((call) => call.time)).toEqual(sampleSeekTimes());
346+
expect(new Set(sampleCalls.map((call) => call.time)).size).toBe(16);
347+
expect(sampleCalls.map((call) => call.held)).toEqual(Array(16).fill(10 / 30));
348+
});
349+
350+
it("holds the frame time for the adaptive probes too", async () => {
351+
const { calls, hook } = recordInjections();
352+
353+
await captureFrameToBuffer(
354+
makeSession({ onBeforeCapture: hook, motionBlur: resolveMotionBlurPlan({}) ?? undefined }),
355+
10,
356+
10 / 30,
357+
);
358+
359+
expect(calls.map((call) => call.time)).toEqual([10 / 30, ...sampleSeekTimes()]);
360+
expect(calls.map((call) => call.held)).toEqual(Array(calls.length).fill(10 / 30));
361+
});
362+
363+
it("holds nothing on a frame without blur", async () => {
364+
const { calls, hook } = recordInjections();
365+
366+
await captureFrameToBuffer(
367+
makeSession({ onBeforeCapture: hook, motionBlur: undefined }),
368+
10,
369+
10 / 30,
370+
);
371+
372+
expect(calls).toEqual([{ time: 10 / 30, held: undefined, seeksBefore: 1 }]);
324373
});
325374
});
326375

‎packages/engine/src/services/frameCapture.ts‎

Lines changed: 27 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,10 @@ export { isMemoryExhaustionError, isTransientBrowserError } from "./captureFailu
102102

103103
export type { CaptureOptions, CaptureResult, CaptureBufferResult, CapturePerfSummary };
104104

105-
/** Called after seeking, before screenshot. Use for video frame injection or other pre-capture work. */
106-
export type BeforeCaptureHook = (page: Page, time: number) => Promise<void>;
105+
/** Called after seeking, before screenshot. Use for video frame injection or other pre-capture work.
106+
* `heldVideoTime` (motion-blur samples only) is the output frame's time: a video on screen at
107+
* both times shows its frame for `heldVideoTime`, so footage does not smear. */
108+
export type BeforeCaptureHook = (page: Page, time: number, heldVideoTime?: number) => Promise<void>;
107109

108110
export interface CaptureSession {
109111
browser: Browser;
@@ -2805,6 +2807,7 @@ async function prepareFrameForCapture(
28052807
frameIndex: number,
28062808
time: number,
28072809
seekOptions?: HfSeekOptions,
2810+
heldVideoTime?: number,
28082811
): Promise<{
28092812
quantizedTime: number;
28102813
seekMs: number;
@@ -2835,7 +2838,7 @@ async function prepareFrameForCapture(
28352838
// replacements for <video> elements.
28362839
const beforeCaptureStart = Date.now();
28372840
if (session.onBeforeCapture) {
2838-
await session.onBeforeCapture(page, quantizedTime);
2841+
await session.onBeforeCapture(page, quantizedTime, heldVideoTime);
28392842
}
28402843
await waitForPendingSeekCompletion(page);
28412844
await page.evaluate(async () => {
@@ -3879,9 +3882,7 @@ export async function withFrameDeadline<T>(
38793882
*
38803883
* Called once initialization has settled the capture mode. `format: "png"` is required
38813884
* because samples are averaged pixel by pixel: JPEG samples would be averaged after
3882-
* lossy quantization and the blended frame is re-encoded as PNG. `<video>` content is
3883-
* out of scope because it is supplied by the before-capture frame-injection hook rather
3884-
* than by the timeline seek, so it cannot follow a sub-frame time.
3885+
* lossy quantization and the blended frame is re-encoded as PNG.
38853886
*/
38863887
export function resolveSessionMotionBlur(session: CaptureSession): MotionBlurPlan | undefined {
38873888
const plan = resolveMotionBlurPlan(session.options.motionBlur);
@@ -3896,11 +3897,6 @@ export function resolveSessionMotionBlur(session: CaptureSession): MotionBlurPla
38963897
`[MotionBlur] sub-frame motion blur requires format "png", got "${session.options.format ?? "jpeg"}"`,
38973898
);
38983899
}
3899-
if (session.onBeforeCapture) {
3900-
throw new Error(
3901-
"[MotionBlur] sub-frame motion blur cannot run with injected video frames: video content is extracted per output frame and does not follow a sub-frame seek",
3902-
);
3903-
}
39043900
return plan;
39053901
}
39063902

@@ -3919,13 +3915,15 @@ async function captureFrameSurface(
39193915
frameIndex: number,
39203916
time: number,
39213917
seekOptions?: HfSeekOptions,
3918+
heldVideoTime?: number,
39223919
): Promise<CapturedSurface> {
39233920
const { page, options } = session;
39243921
const { quantizedTime, seekMs, beforeCaptureMs } = await prepareFrameForCapture(
39253922
session,
39263923
frameIndex,
39273924
time,
39283925
seekOptions,
3926+
heldVideoTime,
39293927
);
39303928

39313929
const screenshotStart = Date.now();
@@ -4112,6 +4110,7 @@ async function resolveAdaptiveSampleCount(
41124110
plan: MotionBlurPlan,
41134111
fps: number,
41144112
sampleSeek: HfSeekOptions,
4113+
frameTime: number,
41154114
): Promise<{
41164115
samplesPerFrame: number;
41174116
seekMs: number;
@@ -4127,8 +4126,8 @@ async function resolveAdaptiveSampleCount(
41274126
};
41284127
}
41294128
const { windowStart, windowEnd } = motionBlurProbeTimes(plan, absFrameIndex, fps);
4130-
const probeA = await captureFrameSurface(session, frameIndex, windowStart, sampleSeek);
4131-
const probeB = await captureFrameSurface(session, frameIndex, windowEnd, sampleSeek);
4129+
const probeA = await captureFrameSurface(session, frameIndex, windowStart, sampleSeek, frameTime);
4130+
const probeB = await captureFrameSurface(session, frameIndex, windowEnd, sampleSeek, frameTime);
41324131
return {
41334132
samplesPerFrame: adaptiveSampleCount(probeDiffMagnitude(probeA.buffer, probeB.buffer)),
41344133
seekMs: probeA.seekMs + probeB.seekMs,
@@ -4160,6 +4159,13 @@ async function captureAccumulatedFrame(
41604159
const eventfulSeekStart = Date.now();
41614160
await seekPageTimeline(session.page, frameTime, undefined);
41624161
const totals = { seekMs: Date.now() - eventfulSeekStart, beforeCaptureMs: 0, screenshotMs: 0 };
4162+
if (session.onBeforeCapture) {
4163+
// The injector copies a video's own style only when its frame changes: inject at frameTime
4164+
// first so the held frame carries frameTime's style, not the first sample's.
4165+
const injectStart = Date.now();
4166+
await session.onBeforeCapture(session.page, frameTime, frameTime);
4167+
totals.beforeCaptureMs += Date.now() - injectStart;
4168+
}
41634169

41644170
const sampleSeek: HfSeekOptions = {
41654171
suppressEvents: true,
@@ -4175,6 +4181,7 @@ async function captureAccumulatedFrame(
41754181
plan,
41764182
fps,
41774183
sampleSeek,
4184+
frameTime,
41784185
);
41794186
samplesPerFrame = chosen.samplesPerFrame;
41804187
totals.seekMs += chosen.seekMs;
@@ -4184,7 +4191,13 @@ async function captureAccumulatedFrame(
41844191

41854192
const accumulator = new MotionBlurAccumulator(plan.blend);
41864193
for (const sampleTime of motionBlurSampleTimes(plan, absFrameIndex, fps, samplesPerFrame)) {
4187-
const sample = await captureFrameSurface(session, frameIndex, sampleTime, sampleSeek);
4194+
const sample = await captureFrameSurface(
4195+
session,
4196+
frameIndex,
4197+
sampleTime,
4198+
sampleSeek,
4199+
frameTime,
4200+
);
41884201
totals.seekMs += sample.seekMs;
41894202
totals.beforeCaptureMs += sample.beforeCaptureMs;
41904203
totals.screenshotMs += sample.screenshotMs;

‎packages/engine/src/services/videoFrameInjector.test.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -322,6 +322,60 @@ describe("createVideoFrameInjector cache hygiene against page-side skips", () =>
322322
});
323323
});
324324

325+
describe("createVideoFrameInjector holds video frames for motion blur (#5144)", () => {
326+
// Scene A's video plays until 1 s, scene B's from 1 s; frame index = 10 × time.
327+
const clips = [
328+
{ videoId: "a", start: 0, end: 1 },
329+
{ videoId: "b", start: 1, end: 2 },
330+
];
331+
const table = {
332+
frameDirs: () => [],
333+
getActiveFramePayloads: (time: number) =>
334+
new Map(
335+
clips
336+
.filter((clip) => time >= clip.start && time < clip.end)
337+
.map((clip) => {
338+
const frameIndex = Math.floor(time * 10);
339+
return [clip.videoId, { framePath: `/${clip.videoId}/${frameIndex}`, frameIndex }];
340+
}),
341+
),
342+
} as unknown as FrameLookupTable;
343+
const page = { evaluate: vi.fn(async () => undefined) } as unknown as Page;
344+
345+
beforeEach(() => {
346+
injectVideoFramesBatchMock.mockReset();
347+
injectVideoFramesBatchMock.mockImplementation(async (_page, updates) =>
348+
updates.map((u) => u.videoId),
349+
);
350+
syncVideoFrameVisibilityMock.mockReset();
351+
syncVideoFrameVisibilityMock.mockResolvedValue(undefined);
352+
});
353+
354+
it("shows the held frame of a video on screen at both times", async () => {
355+
const hook = createVideoFrameInjector(table, { frameSrcResolver: inlineResolver });
356+
357+
// The sample alone would pick frame 14; the held frame time picks 15.
358+
await hook!(page, 1.42, 1.5);
359+
360+
expect(syncVideoFrameVisibilityMock).toHaveBeenLastCalledWith(page, ["b"]);
361+
expect(injectVideoFramesBatchMock.mock.calls[0]?.[1]).toEqual([
362+
{ videoId: "b", dataUri: inlineResolver("/b/15") },
363+
]);
364+
});
365+
366+
it("follows the sample time across a cut, so neither scene's video drops out", async () => {
367+
const hook = createVideoFrameInjector(table, { frameSrcResolver: inlineResolver });
368+
369+
// Frame 1.0 is scene B's first frame; a sample just before it still shows scene A.
370+
await hook!(page, 0.98, 1.0);
371+
372+
expect(syncVideoFrameVisibilityMock).toHaveBeenLastCalledWith(page, ["a"]);
373+
expect(injectVideoFramesBatchMock.mock.calls[0]?.[1]).toEqual([
374+
{ videoId: "a", dataUri: inlineResolver("/a/9") },
375+
]);
376+
});
377+
});
378+
325379
describe("createVideoFrameInjector extraction-cache lease renewal", () => {
326380
// Regression: a render can hold a compiled-dir symlink into a shared
327381
// extraction-cache entry far longer than the entry's one-time cache-hit

‎packages/engine/src/services/videoFrameInjector.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,9 +204,14 @@ export function createVideoFrameInjector(
204204
}
205205

206206
// fallow-ignore-next-line complexity
207-
return async (page: Page, time: number) => {
207+
return async (page: Page, time: number, heldVideoTime?: number) => {
208208
for (const cacheDir of frameLookup.frameDirs()) renewCacheLease(cacheDir);
209209
const activePayloads = frameLookup.getActiveFramePayloads(time);
210+
if (heldVideoTime !== undefined) {
211+
for (const [videoId, payload] of frameLookup.getActiveFramePayloads(heldVideoTime)) {
212+
if (activePayloads.has(videoId)) activePayloads.set(videoId, payload);
213+
}
214+
}
210215

211216
const updates: Array<{ videoId: string; dataUri: string; frameIndex: number }> = [];
212217
const activeIds = new Set<string>();

‎packages/producer/scripts/test-classification.mjs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ const INTEGRATION_TEST_FILES = new Set([
3030
"src/services/htmlCompiler.mediaType.test.ts",
3131
"src/services/htmlCompiler.naturalDuration.test.ts",
3232
"src/services/render/hlsRender.integration.test.ts",
33+
"src/services/render/motionBlurVideo.integration.test.ts",
3334
"src/services/render/renderCancel.integration.test.ts",
3435
"src/services/render/stages/compileStage.mediaType.test.ts",
3536
"src/services/render/stages/gifEncodeArgs.test.ts",

0 commit comments

Comments
 (0)