Skip to content

Commit 16de58e

Browse files
vanceingallsclaude
andcommitted
test(producer): windowed frames runs drawElement through self-verify and a recycle
Adds a drawElement case to the windowed-frames integration test: drawElement capture, self-verify through the on-demand hook, and a browser recycle. It covers that hook only on a macOS or Windows host with a GPU. Linux always gates drawElement away (headless-shell launches for BeginFrame, which has no self-verify), and so does any host with no GPU; there the case asserts the named gate fallback (swiftshader, webgl_unavailable or beginframe_no_self_verify) and prints that drawElement did not run. HF_TEST_REQUIRE_DRAWELEMENT=1 turns that fallback into a failure, for the hosts that are meant to run it. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 9dbc955 commit 16de58e

1 file changed

Lines changed: 158 additions & 11 deletions

File tree

‎packages/producer/src/services/render/windowedFrames.integration.test.ts‎

Lines changed: 158 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,15 @@
11
// Windowed source-video frames (HF_VIDEO_FRAME_WINDOW), rendered for real: each case renders twice
22
// on the segmented route, once reading a segment's frames at a time and once extracting whole
33
// videos, and the two outputs must match frame for frame. Needs Chrome + ffmpeg: integration lane.
4-
// Known gap: this test pins `browserGpuMode: "software"`, so every render here takes the screenshot
5-
// path. Windowed drawElement output parity was measured by hand on real footage and has no automated
6-
// guard.
4+
// Every case but one pins `browserGpuMode: "software"`, which takes the screenshot path. The
5+
// drawElement case asks for `browserGpuMode: "hardware"` and `useDrawElement`, the only way to get
6+
// past the engine's software-rasterizer gate without HF_FORCE_DRAWELEMENT (which skips self-verify),
7+
// so it guards drawElement capture, self-verify through the on-demand hook, and a browser recycle.
8+
// The drawElement branch runs only on a macOS or Windows host with a GPU. Linux always gates it away,
9+
// with or without a GPU, because headless-shell launches for BeginFrame, which has no self-verify;
10+
// there, and on any host with no GPU, the case asserts the named fallback and says in its output that
11+
// drawElement did not run. HF_TEST_REQUIRE_DRAWELEMENT=1 turns that fallback into a failure, for the
12+
// hosts that are meant to run it.
713
//
814
// HF_TEST_RETRY_BROWSER_STALL=1 renders once more after Chrome dies, for an overloaded local host.
915
// Never set it in CI: a windowed render that needs the retry alone fails the case.
@@ -66,6 +72,19 @@ interface Case {
6672
calibrates?: boolean;
6773
/** Caps the worker count the orchestrator picks, to keep the Chromes it opens few. */
6874
maxWorkers?: number;
75+
/** Captures with drawElement and its self-verify, which software rasterization would route away from. */
76+
drawElement?: boolean;
77+
/** Recycles a worker's browser after this many segments. */
78+
recycleEverySegments?: number;
79+
/** Keeps the windowed render's work dir to read `segment-frames` once it closes, and polls it meanwhile. */
80+
inspectsFolder?: boolean;
81+
}
82+
83+
interface DrawElementFacts {
84+
mode: string | undefined;
85+
verifyArmed: number | undefined;
86+
selfVerifyFallback: boolean | undefined;
87+
gateReason: string | undefined;
6988
}
7089

7190
interface Run {
@@ -79,10 +98,15 @@ interface Run {
7998
peakBytes: number | undefined;
8099
extractedFrames: number | undefined;
81100
calibration: string;
101+
drawElement: DrawElementFacts;
102+
/** Cadence-driven browser restarts the segmented capture reported. */
103+
browserRecycles: number | undefined;
82104
/** Windowed renders only: what a poll of `segment-frames` saw while the render ran. */
83105
sampled: { peakBytes: number; largestFileBytes: number };
84106
/** `keepWorkDir` renders only: the entries left in `segment-frames` once the render closed. */
85107
leftInSegmentFrames: string[] | undefined;
108+
/** `keepWorkDir` renders only: the `on-demand_*` folders a poll saw while they held a frame. */
109+
onDemandWithFrames: number;
86110
}
87111

88112
let root: string;
@@ -193,15 +217,23 @@ function firstDifference(left: string[], right: string[]): number {
193217
const ENV_KEYS = [
194218
"HF_SEGMENTED_CAPTURE",
195219
"HF_SEGMENT_FRAMES",
220+
"HF_SEGMENT_BROWSER_RECYCLE",
196221
"HF_VIDEO_FRAME_WINDOW",
197222
"KEEP_TEMP",
198223
] as const;
199224

200225
/** Sets the render's env and returns the call that puts the caller's back. */
201-
function setRenderEnv(windowed: boolean, keepWorkDir: boolean | undefined): () => void {
226+
function setRenderEnv(
227+
windowed: boolean,
228+
keepWorkDir: boolean | undefined,
229+
recycleEverySegments: number | undefined,
230+
): () => void {
202231
const saved = Object.fromEntries(ENV_KEYS.map((key) => [key, process.env[key]]));
203232
process.env.HF_SEGMENTED_CAPTURE = "true";
204233
process.env.HF_SEGMENT_FRAMES = String(SEGMENT_FRAMES);
234+
if (recycleEverySegments !== undefined) {
235+
process.env.HF_SEGMENT_BROWSER_RECYCLE = String(recycleEverySegments);
236+
}
205237
if (windowed) process.env.HF_VIDEO_FRAME_WINDOW = "true";
206238
else delete process.env.HF_VIDEO_FRAME_WINDOW;
207239
if (keepWorkDir) process.env.KEEP_TEMP = "1";
@@ -240,6 +272,29 @@ function watchWorkDirs(output: string, jobId: string) {
240272
return { own };
241273
}
242274

275+
/**
276+
* Counts the `on-demand_*` folders seen holding a frame, by name. An extraction of one frame lasts
277+
* about as long as an ffmpeg spawn, so this polls much faster than {@link sampleSegmentFrames}.
278+
*/
279+
function watchOnDemandFolders(own: () => string[]) {
280+
const withFrames = new Set<string>();
281+
const poll = setInterval(() => {
282+
for (const workDir of own()) {
283+
const folder = join(workDir, "segment-frames");
284+
let names: string[];
285+
try {
286+
names = readdirSync(folder).filter((name) => name.startsWith("on-demand_"));
287+
} catch {
288+
continue;
289+
}
290+
for (const name of names) {
291+
if (directoryBytes(join(folder, name), () => {}) > 0) withFrames.add(name);
292+
}
293+
}
294+
}, 5);
295+
return { withFrames, stop: () => clearInterval(poll) };
296+
}
297+
243298
/** Polls `segment-frames` in the render's work dir for the most it holds at once. */
244299
function sampleSegmentFrames(own: () => string[]) {
245300
const sampled = { peakBytes: 0, largestFileBytes: 0 };
@@ -260,6 +315,7 @@ function describeRun(
260315
job: RenderJob,
261316
output: string,
262317
trace: Array<Record<string, unknown>>,
318+
seen: { browserRecycles: number | undefined },
263319
): Omit<Run, "sampled" | "leftInSegmentFrames"> {
264320
const perf = job.perfSummary;
265321
const capture = perf?.observability?.capture;
@@ -281,6 +337,13 @@ function describeRun(
281337
peakBytes: capture?.videoFrameWindowPeakBytes,
282338
extractedFrames: perf?.observability?.extraction?.totalFramesExtracted,
283339
calibration: String(calibration?.message ?? "not reached"),
340+
drawElement: {
341+
mode: perf?.drawElement?.mode,
342+
verifyArmed: perf?.drawElement?.verifyArmed,
343+
selfVerifyFallback: perf?.drawElement?.selfVerifyFallback,
344+
gateReason: perf?.drawElement?.gateReason,
345+
},
346+
browserRecycles: seen.browserRecycles,
284347
};
285348
}
286349

@@ -309,7 +372,11 @@ async function render(
309372
}
310373
}
311374

312-
function createJob(testCase: Case, trace: Array<Record<string, unknown>>): RenderJob {
375+
function createJob(
376+
testCase: Case,
377+
trace: Array<Record<string, unknown>>,
378+
seen: { browserRecycles: number | undefined },
379+
): RenderJob {
313380
return createRenderJob({
314381
fps: testCase.fps,
315382
quality: "draft",
@@ -320,13 +387,19 @@ function createJob(testCase: Case, trace: Array<Record<string, unknown>>): Rende
320387
logger: {
321388
info(message: string, meta?: Record<string, unknown>) {
322389
if (message === "[Render:trace]" && meta) trace.push(meta);
390+
if (message.startsWith("[Render] Segmented capture complete") && meta) {
391+
seen.browserRecycles = Number(meta.browserRecycles);
392+
}
323393
},
324394
warn() {},
325395
error() {},
326396
debug() {},
327397
},
328398
producerConfig: resolveConfig({
329-
browserGpuMode: "software",
399+
// "hardware" is what keeps drawElement from being routed to screenshot capture as software
400+
// rasterization; the engine still runs every gate and the self-verify on the render.
401+
browserGpuMode: testCase.drawElement ? "hardware" : "software",
402+
useDrawElement: testCase.drawElement ? true : undefined,
330403
concurrency: testCase.maxWorkers ?? "auto",
331404
}),
332405
});
@@ -345,24 +418,28 @@ async function renderOnce(testCase: Case, windowed: boolean, options: RenderOpti
345418
const label = `${testCase.name.replace(/\W+/g, "-")}-${windowed ? "windowed" : "full"}`;
346419
writeFileSync(join(root, "index.html"), composition(testCase));
347420
const trace: Array<Record<string, unknown>> = [];
348-
const job = createJob(testCase, trace);
349-
const restoreEnv = setRenderEnv(windowed, options.keepWorkDir);
421+
const seen = { browserRecycles: undefined as number | undefined };
422+
const job = createJob(testCase, trace, seen);
423+
const restoreEnv = setRenderEnv(windowed, options.keepWorkDir, testCase.recycleEverySegments);
350424
const output = join(root, `${label}.mp4`);
351425
const { own } = watchWorkDirs(output, job.id);
352426
const sampler = options.sampleFolder ? sampleSegmentFrames(own) : NO_SAMPLER;
427+
const onDemand = options.sampleFolder ? watchOnDemandFolders(own) : null;
353428
const kept: string[] = [];
354429
try {
355430
await executeRenderJob(job, root, output).catch((error: unknown) => {
356431
throw new Error(`${label} failed: ${String(error)}`, { cause: error });
357432
});
358433
if (options.keepWorkDir) kept.push(...own());
359434
return {
360-
...describeRun(job, output, trace),
435+
...describeRun(job, output, trace, seen),
361436
sampled: sampler.sampled,
362437
leftInSegmentFrames: leftInSegmentFrames(kept),
438+
onDemandWithFrames: onDemand?.withFrames.size ?? 0,
363439
};
364440
} finally {
365441
sampler.stop();
442+
onDemand?.stop();
366443
restoreEnv();
367444
for (const workDir of kept) rmSync(workDir, { recursive: true, force: true });
368445
}
@@ -376,9 +453,10 @@ function renderPair(testCase: Case): Promise<{ windowed: Run; full: Run }> {
376453
if (!pair) {
377454
pair = (async () => {
378455
const full = await render(testCase, false);
456+
const inspects = testCase.name === PEAK_CASE || testCase.inspectsFolder === true;
379457
const windowed = await render(testCase, true, {
380-
keepWorkDir: testCase.name === PEAK_CASE,
381-
sampleFolder: testCase.name === PEAK_CASE,
458+
keepWorkDir: inspects,
459+
sampleFolder: inspects,
382460
});
383461
if (windowed.retried && !full.retried) {
384462
throw new Error(`${testCase.name}: only the windowed render needed a retry`);
@@ -397,6 +475,9 @@ function renderPair(testCase: Case): Promise<{ windowed: Run; full: Run }> {
397475
}
398476

399477
const PEAK_CASE = "2 workers";
478+
const DRAW_ELEMENT_CASE = "drawElement capture with a browser recycle";
479+
/** The engine's `deGateReason`s for a host that cannot run drawElement: software or no WebGL, or a BeginFrame launch. */
480+
const HOST_GATES = new Set(["swiftshader", "webgl_unavailable", "beginframe_no_self_verify"]);
400481
// Four segments for two workers, so the window has something to hold back.
401482
const PEAK_SECONDS = 8;
402483
const CFR = "assets/cfr.mp4";
@@ -475,6 +556,16 @@ function cases(): Case[] {
475556
seconds: PEAK_SECONDS,
476557
clips: [{ id: "clip", src: "assets/long.mp4", start: 0, duration: PEAK_SECONDS }],
477558
},
559+
{
560+
...base,
561+
name: DRAW_ELEMENT_CASE,
562+
drawElement: true,
563+
recycleEverySegments: 1,
564+
inspectsFolder: true,
565+
// Two segments, so the one recycle falls between them.
566+
seconds: 4,
567+
clips: [{ id: "clip", src: CFR, start: 0, duration: 4 }],
568+
},
478569
{
479570
...base,
480571
name: "motion blur, which declines the window",
@@ -546,6 +637,62 @@ describe.skipIf(!HAS_FFMPEG)("windowed source-video frames — real render", ()
546637
}, 600_000);
547638
}
548639

640+
it("captures with drawElement where the host can run it, and names its gate where it cannot", async () => {
641+
const testCase = cases().find((candidate) => candidate.name === DRAW_ELEMENT_CASE)!;
642+
const { windowed, full } = await renderPair(testCase);
643+
644+
// `gateReason` is a `|`-joined set across the render's sessions.
645+
const gates = [windowed, full].map((run) => run.drawElement.gateReason ?? "");
646+
const gated = gates.every(
647+
(gate) => gate !== "" && gate.split("|").every((r) => HOST_GATES.has(r)),
648+
);
649+
if (gated) {
650+
if (process.env.HF_TEST_REQUIRE_DRAWELEMENT === "1") {
651+
throw new Error(
652+
`HF_TEST_REQUIRE_DRAWELEMENT=1 but drawElement was gated away (${gates.join(", ")}): ` +
653+
"this host cannot run it (no GPU, or a BeginFrame launch)",
654+
);
655+
}
656+
for (const run of [windowed, full]) {
657+
const { drawElement } = run;
658+
// Falling back is the contract here: the render completed, on the baseline capture, with no
659+
// self-verify armed and none to fail.
660+
expect(drawElement.mode).not.toContain("drawelement");
661+
expect(drawElement.verifyArmed ?? 0).toBe(0);
662+
expect(drawElement.selfVerifyFallback ?? false).toBe(false);
663+
}
664+
expect(windowed.videoFrameWindow).toBe("on");
665+
expect(windowed.leftInSegmentFrames).toEqual([]);
666+
console.warn(
667+
`[windowedFrames] ${DRAW_ELEMENT_CASE}: drawElement did NOT run on this host (gate ${gates.join(", ")}): ` +
668+
"no GPU, or a BeginFrame launch. Only the fallback was asserted; drawElement capture, self-verify " +
669+
"and the on-demand hook are unexercised here. Run on a macOS or Windows host with a GPU to cover them.",
670+
);
671+
return;
672+
}
673+
674+
for (const run of [windowed, full]) {
675+
const { drawElement } = run;
676+
// A fall back to screenshot capture, or past the capability or software gates, reads here.
677+
expect(`${drawElement.mode} gate=${drawElement.gateReason}`).toBe(
678+
"drawelement gate=undefined",
679+
);
680+
// Armed samples read off a video comp mean the ground truth had real frames, not black boxes.
681+
expect(drawElement.verifyArmed ?? 0).toBeGreaterThan(0);
682+
expect(drawElement.selfVerifyFallback).toBe(false);
683+
}
684+
expect(windowed.videoFrameWindow).toBe("on");
685+
expect(windowed.browserRecycles ?? 0).toBeGreaterThanOrEqual(1);
686+
// The ground truth went through the on-demand hook: a hook that injected nothing leaves no folder to see.
687+
expect(windowed.onDemandWithFrames).toBeGreaterThan(0);
688+
// The folder exists once the render closes and holds neither an `on-demand_*` leftover nor anything else.
689+
expect(windowed.leftInSegmentFrames).toEqual([]);
690+
console.log(
691+
`[windowedFrames] ${DRAW_ELEMENT_CASE}: ${JSON.stringify(windowed.drawElement)}; ` +
692+
`recycles=${windowed.browserRecycles}; onDemandWithFrames=${windowed.onDemandWithFrames}; left=${JSON.stringify(windowed.leftInSegmentFrames)}`,
693+
);
694+
}, 600_000);
695+
549696
it("never holds more than workers x segment frames of source frames", async () => {
550697
const testCase = cases().find((candidate) => candidate.name === PEAK_CASE)!;
551698
const { windowed } = await renderPair(testCase);

0 commit comments

Comments
 (0)