Skip to content

Commit 6c3cc7c

Browse files
fix(producer): graded video renders split into chunks that finish inside the worker limit (#5883)
* fix(producer): graded video renders split into chunks that finish inside the worker limit The distributed planner priced every frame at a fixed cost per pixel, so a color-graded video, which costs about 10 s a frame on software GL, was planned into chunks that could never finish before the worker time limit. When the composition has color grading and the plan sizes chunks from the worker limit, the planner now captures the renderer's calibration frames in a session set up like a chunk worker's and prices frames at the measured p95 when it is higher than the estimate. Compositions without grading never pay for the measurement and keep their current plan. * docs(producer): cut the moved chunk frame comments to their invariants
1 parent c7ad19b commit 6c3cc7c

6 files changed

Lines changed: 537 additions & 288 deletions

File tree

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
// What a chunk capture session is built from, shared with the planner's frame-cost measurement.
2+
import { existsSync, readdirSync } from "node:fs";
3+
import { extname, join } from "node:path";
4+
import {
5+
type BeforeCaptureHook,
6+
BROWSER_GPU_NOT_SOFTWARE,
7+
compositionRequiresWebGpu,
8+
type CaptureOptions,
9+
createFrameLookupTable,
10+
createVideoFrameInjector,
11+
type EngineConfig,
12+
type ExtractedFrames,
13+
extractVideoFramesRange,
14+
type FrameRange,
15+
type FrameLookupTable,
16+
} from "@hyperframes/engine";
17+
import { resolveVideoCaptureBeyondViewport } from "../render/captureBeyondViewport.js";
18+
import { resolveCaptureJpegQuality } from "../render/captureImageFormat.js";
19+
import type { LockedRenderConfig } from "../render/stages/freezePlan.js";
20+
import { resolveRenderFpsConfig } from "../fileServer.js";
21+
import { type DistributedFormat, INVALID_VIDEO_METADATA, type PlanVideosJson } from "./shared.js";
22+
23+
/** Distinct codes so adapters route retries per cause: a missing artifact may heal, a hash mismatch never. */
24+
export const FFMPEG_VERSION_MISMATCH = "FFMPEG_VERSION_MISMATCH";
25+
export const PLAN_HASH_MISMATCH = "PLAN_HASH_MISMATCH";
26+
export const MISSING_PLAN_ARTIFACT = "MISSING_PLAN_ARTIFACT";
27+
export const CHUNK_INDEX_OUT_OF_RANGE = "CHUNK_INDEX_OUT_OF_RANGE";
28+
export const MISSING_RUNTIME_ENV_SNAPSHOT = "MISSING_RUNTIME_ENV_SNAPSHOT";
29+
export { INVALID_VIDEO_METADATA };
30+
31+
export type RenderChunkValidationCode =
32+
| typeof FFMPEG_VERSION_MISMATCH
33+
| typeof PLAN_HASH_MISMATCH
34+
| typeof MISSING_PLAN_ARTIFACT
35+
| typeof CHUNK_INDEX_OUT_OF_RANGE
36+
| typeof MISSING_RUNTIME_ENV_SNAPSHOT
37+
| typeof INVALID_VIDEO_METADATA
38+
| typeof BROWSER_GPU_NOT_SOFTWARE;
39+
40+
/** A malformed planDir or a worker that does not match the plan's fingerprint; adapters key retries off `code`. */
41+
export class RenderChunkValidationError extends Error {
42+
readonly code: RenderChunkValidationCode;
43+
constructor(code: RenderChunkValidationCode, message: string) {
44+
super(message);
45+
this.name = "RenderChunkValidationError";
46+
this.code = code;
47+
}
48+
}
49+
50+
/** One injector per browser session: a hook reused on a fresh page can skip its first injection. */
51+
export function createChunkVideoFrameInjectorFactory(
52+
frameLookup: FrameLookupTable | null,
53+
): () => BeforeCaptureHook | null {
54+
return () => createVideoFrameInjector(frameLookup);
55+
}
56+
57+
function frameNumberFromFileName(name: string): number | null {
58+
const match = /(\d+)(?=\.[^.]+$)/.exec(name);
59+
if (!match) return null;
60+
const frameNumber = Number(match[1]);
61+
return Number.isSafeInteger(frameNumber) ? frameNumber : null;
62+
}
63+
64+
/**
65+
* Rebuilds `ExtractedFrames[]` from `<planDir>/video-frames/<videoId>/`. `framePaths` keys are 0-based
66+
* frame indexes (what `getFrameAtTime` reads); any other key drops every video's first-paint frame.
67+
*/
68+
export function rebuildExtractedFramesFromPlanDir(
69+
planDir: string,
70+
videos: PlanVideosJson["extracted"],
71+
indexMode: "dense-v1" | "sparse-v2" = "dense-v1",
72+
): ExtractedFrames[] {
73+
const result: ExtractedFrames[] = [];
74+
for (const v of videos) {
75+
const outputDir = join(planDir, "video-frames", v.videoId);
76+
if (!existsSync(outputDir)) {
77+
throw new Error(
78+
`[renderChunk] planDir missing extracted video frames for ${JSON.stringify(v.videoId)}: ` +
79+
`${outputDir} not present. plan() should have written frames here; the planDir is malformed.`,
80+
);
81+
}
82+
// Numeric order: past the padding width, lexical order puts frame_100000 before frame_10001.
83+
const ext = (extname(v.framePattern) || ".jpg").toLowerCase();
84+
const frames = readdirSync(outputDir)
85+
.filter((name) => name.toLowerCase().endsWith(ext))
86+
.sort((left, right) => {
87+
const leftNumber = frameNumberFromFileName(left);
88+
const rightNumber = frameNumberFromFileName(right);
89+
if (leftNumber === null || rightNumber === null) return left.localeCompare(right);
90+
return leftNumber - rightNumber || left.localeCompare(right);
91+
});
92+
const framePaths = new Map<number, string>();
93+
for (let i = 0; i < frames.length; i++) {
94+
const frameName = frames[i];
95+
if (!frameName) continue;
96+
// v2 materialization is sparse, so only v2 reads the index from ffmpeg's 1-based name.
97+
const frameNumber = indexMode === "sparse-v2" ? frameNumberFromFileName(frameName) : null;
98+
const frameIndex = frameNumber === null ? i : frameNumber - 1;
99+
framePaths.set(frameIndex, join(outputDir, frameName));
100+
}
101+
result.push({
102+
videoId: v.videoId,
103+
srcPath: v.srcPath,
104+
outputDir,
105+
framePattern: v.framePattern,
106+
fps: v.fps,
107+
totalFrames: v.totalFrames,
108+
metadata: v.metadata,
109+
framePaths,
110+
// The planDir is shared read-only across workers; cleanup must not delete its frames.
111+
ownedByLookup: false,
112+
});
113+
}
114+
return result;
115+
}
116+
117+
function contiguousRuns(sortedIndexes: readonly number[]): FrameRange[] {
118+
const runs: FrameRange[] = [];
119+
for (const index of sortedIndexes) {
120+
const last = runs[runs.length - 1];
121+
if (last && last.firstFrame + last.frames === index) last.frames++;
122+
else runs.push({ firstFrame: index, frames: 1 });
123+
}
124+
return runs;
125+
}
126+
127+
/** Capture options a chunk session uses; the planner measures frame cost with the same ones. */
128+
export function buildChunkCaptureOptions(input: {
129+
dimensions: {
130+
width: number;
131+
height: number;
132+
fpsNum: number;
133+
fpsDen: number;
134+
format: DistributedFormat;
135+
};
136+
encoder: Pick<LockedRenderConfig, "quality" | "deviceScaleFactor" | "variables">;
137+
videoCount: number;
138+
html: string;
139+
}): CaptureOptions {
140+
const { dimensions, encoder } = input;
141+
const videoCaptureBeyondViewport = resolveVideoCaptureBeyondViewport(input.videoCount);
142+
return {
143+
width: dimensions.width,
144+
height: dimensions.height,
145+
fps: { num: dimensions.fpsNum, den: dimensions.fpsDen },
146+
format: dimensions.format === "mp4" ? "jpeg" : "png",
147+
quality: dimensions.format === "mp4" ? resolveCaptureJpegQuality(encoder.quality) : undefined,
148+
deviceScaleFactor: encoder.deviceScaleFactor,
149+
// The controller's snapshot, so the first capture sees the in-process renderer's `__hfVariables`.
150+
variables: encoder.variables,
151+
subTimelineWaitMemo: {},
152+
...(videoCaptureBeyondViewport !== undefined
153+
? { captureBeyondViewport: videoCaptureBeyondViewport }
154+
: {}),
155+
// A fixed warmup count keeps `beginFrameTimeTicks` host-independent.
156+
lockWarmupTicks: true,
157+
requiresWebGpu: compositionRequiresWebGpu(input.html),
158+
};
159+
}
160+
161+
/** Extracts the given frames of deferred videos under the names a full extraction gives them. */
162+
export async function extractDeferredVideoFramesForChunk(input: {
163+
planDir: string;
164+
planVideos: PlanVideosJson;
165+
frames: Iterable<number>;
166+
fps: { num: number; den: number };
167+
outputRoot: string;
168+
cfg: Pick<EngineConfig, "ffmpegProcessTimeout">;
169+
}): Promise<ExtractedFrames[]> {
170+
const deferred = input.planVideos.extracted.filter((video) => video.deferredRange);
171+
if (deferred.length === 0) return [];
172+
const table = createFrameLookupTable(
173+
input.planVideos.videos,
174+
deferred.map((video) => ({
175+
...video,
176+
outputDir: input.outputRoot,
177+
framePaths: new Map(Array.from({ length: video.totalFrames }, (_, i) => [i, String(i)])),
178+
})),
179+
resolveRenderFpsConfig(input.fps).value,
180+
);
181+
const shown = new Map<string, Set<number>>();
182+
for (const frame of input.frames) {
183+
const globalTime = (frame * input.fps.den) / input.fps.num;
184+
for (const [videoId, payload] of table.getActiveFramePayloads(globalTime)) {
185+
const indexes = shown.get(videoId) ?? new Set<number>();
186+
indexes.add(payload.frameIndex);
187+
shown.set(videoId, indexes);
188+
}
189+
}
190+
return Promise.all(
191+
deferred.map(async (video) => {
192+
const outputDir = join(input.outputRoot, video.videoId);
193+
const indexes = [...(shown.get(video.videoId) ?? [])].sort((a, b) => a - b);
194+
if (indexes.length === 0) {
195+
return { ...video, outputDir, framePaths: new Map<number, string>(), ownedByLookup: false };
196+
}
197+
const range = video.deferredRange!;
198+
const sourcePath = join(input.planDir, ...range.sourcePath.split("/"));
199+
if (!existsSync(sourcePath)) {
200+
throw new RenderChunkValidationError(
201+
MISSING_PLAN_ARTIFACT,
202+
`[renderChunk] planDir is missing the source for video ${JSON.stringify(video.videoId)}: ${range.sourcePath}`,
203+
);
204+
}
205+
const extracted = await extractVideoFramesRange(
206+
sourcePath,
207+
video.videoId,
208+
range.startTime,
209+
range.durationSeconds,
210+
{
211+
fps: input.fps,
212+
outputDir: input.outputRoot,
213+
format: range.format,
214+
frameRanges: contiguousRuns(indexes),
215+
},
216+
undefined,
217+
input.cfg,
218+
);
219+
const missing = indexes.find((index) => !extracted.framePaths.has(index));
220+
if (missing !== undefined) {
221+
throw new RenderChunkValidationError(
222+
INVALID_VIDEO_METADATA,
223+
`[renderChunk] video ${JSON.stringify(video.videoId)} has no frame ${missing}; ` +
224+
`the plan expected ${video.totalFrames} frames from ${range.sourcePath}.`,
225+
);
226+
}
227+
return { ...video, outputDir, framePaths: extracted.framePaths, ownedByLookup: false };
228+
}),
229+
);
230+
}

‎packages/producer/src/services/distributed/plan.test.ts‎

Lines changed: 74 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* inside the regression harness (`bun run --cwd packages/producer docker:test`).
1616
*/
1717

18-
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
18+
import { afterAll, beforeAll, describe, expect, it, mock } from "bun:test";
1919
import { spawnSync } from "node:child_process";
2020
import {
2121
chmodSync,
@@ -46,11 +46,13 @@ import {
4646
ChunkExceedsWorkerTimeoutError,
4747
DEFAULT_CHUNK_SIZE,
4848
DEFAULT_MAX_PARALLEL_CHUNKS,
49+
chunkSizingUsesFrameCost,
4950
estimateChunkSeconds,
5051
MIN_CHUNK_SIZE,
5152
plan,
5253
resolveDistributedEngineConfig,
5354
resolveChunkPlan,
55+
planNeedsFrameCostMeasurement,
5456
resolveMaxParallelChunks,
5557
} from "./plan.js";
5658
import { buildSyntheticRenderJob } from "./shared.js";
@@ -286,6 +288,45 @@ describe("resolveMaxParallelChunks", () => {
286288
expect(resolveMaxParallelChunks({ ...worker, targetChunkFrames: 300 }, ...args)).toBe(256);
287289
});
288290

291+
it("sizes chunks from a measured frame cost so expensive frames fit the limit", () => {
292+
// 45 s of 1080x1920 at 24 fps whose frames measured ~10.5 s each (a graded video on software GL).
293+
const frames = 45 * 24;
294+
const mp = (1080 * 1920) / 1e6;
295+
const worker = { workerTimeoutSeconds: LAMBDA_LIMIT, workerConcurrency: 8 };
296+
const { log } = recordingLog();
297+
const priced = resolveChunkPlan(
298+
frames,
299+
undefined,
300+
resolveMaxParallelChunks(worker, frames, mp),
301+
);
302+
expect(estimateChunkSeconds(priced.effectiveChunkSize, mp, 10.5)).toBeGreaterThan(LAMBDA_LIMIT);
303+
304+
const measured = resolveChunkPlan(
305+
frames,
306+
undefined,
307+
resolveMaxParallelChunks(worker, frames, mp, log, 10.5),
308+
);
309+
expect(estimateChunkSeconds(measured.effectiveChunkSize, mp, 10.5)).toBeLessThanOrEqual(
310+
LAMBDA_LIMIT * 0.75,
311+
);
312+
// A cheap measured frame keeps the per-pixel plan.
313+
expect(resolveMaxParallelChunks(worker, frames, mp, log, 0.07)).toBe(
314+
resolveMaxParallelChunks(worker, frames, mp, log),
315+
);
316+
});
317+
318+
it("measures frame cost only for graded media and only when sizing uses it", () => {
319+
expect(planNeedsFrameCostMeasurement('<video data-color-grading=\'{"preset":"x"}\'>')).toBe(
320+
true,
321+
);
322+
expect(planNeedsFrameCostMeasurement("<video src=a.mp4>")).toBe(false);
323+
expect(chunkSizingUsesFrameCost({ workerTimeoutSeconds: LAMBDA_LIMIT })).toBe(true);
324+
expect(chunkSizingUsesFrameCost({ workerTimeoutSeconds: LAMBDA_LIMIT, chunkSize: 48 })).toBe(
325+
false,
326+
);
327+
expect(chunkSizingUsesFrameCost({})).toBe(false);
328+
});
329+
289330
it("fails fast with a typed error when even 256 chunks cannot fit the limit", () => {
290331
const eightHours = 8 * 60 * 60 * 30;
291332
let caught: unknown;
@@ -525,6 +566,38 @@ describe("plan() defaults", () => {
525566
expect(sized.chunkCount).toBe(1);
526567
}, 30_000);
527568

569+
it("splits graded media into more chunks from its measured frame cost", async () => {
570+
const measuredFor: string[] = [];
571+
mock.module("./planFrameCost.js", () => ({
572+
measurePlanSecondsPerFrame: async (input: { html: string }) => {
573+
measuredFor.push(input.html);
574+
return 10;
575+
},
576+
}));
577+
const graded = join(runRoot, "project-graded");
578+
mkdirSync(graded, { recursive: true });
579+
writeFileSync(
580+
join(graded, "index.html"),
581+
FIXTURE_HTML.replace("<p>", `<p data-color-grading='{"exposure":0.2}'>`),
582+
"utf-8",
583+
);
584+
const config = {
585+
fps: 30,
586+
width: 320,
587+
height: 240,
588+
format: "mp4",
589+
workerTimeoutSeconds: 900,
590+
workerConcurrency: 8,
591+
} as const;
592+
const cheap = await plan(projectDir, config, join(runRoot, "plan-cheap"));
593+
expect(measuredFor).toHaveLength(0);
594+
const costly = await plan(graded, config, join(runRoot, "plan-graded"));
595+
expect(measuredFor).toHaveLength(1);
596+
expect(cheap.chunkCount).toBe(1);
597+
// 30 frames at 10 s each split as far as the 10-frame chunk floor allows.
598+
expect(costly.chunkCount).toBe(3);
599+
}, 30_000);
600+
528601
it("rejects at plan time when no chunk can finish inside the worker limit", async () => {
529602
await expect(
530603
plan(

0 commit comments

Comments
 (0)