Skip to content

Commit 35a4fe1

Browse files
fix(producer): stop disk capture early when frames cannot fit on disk (#4164)
* fix(producer): stop disk capture early when frames cannot fit on disk * test(producer): a measured disk shortfall is never retried as a browser death
1 parent 7f452ad commit 35a4fe1

4 files changed

Lines changed: 325 additions & 39 deletions

File tree

‎packages/producer/src/services/render/stages/captureStage.test.ts‎

Lines changed: 205 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import { afterEach, describe, expect, it, vi } from "vitest";
2-
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
2+
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
33
import { tmpdir } from "node:os";
44
import { join } from "node:path";
5-
import { DEFAULT_CONFIG } from "@hyperframes/engine";
6-
import { createRenderJob } from "../../renderOrchestrator.js";
5+
import { DEFAULT_CONFIG, classifyCaptureFailure } from "@hyperframes/engine";
6+
import { createRenderJob, isTransientCaptureRetryEligible } from "../../renderOrchestrator.js";
77
import { formatCaptureFrameName } from "../../../utils/paths.js";
88

99
// Mock only the engine session primitives; `classifyCaptureFailure` stays
@@ -14,6 +14,7 @@ const captureFrame = vi.fn();
1414
const closeCaptureSession = vi.fn(async () => {});
1515
const verifyDiskDrawElementSamples = vi.fn(async () => {});
1616
const getCapturePerfSummary = vi.fn(() => ({}));
17+
const executeParallelCapture = vi.fn();
1718

1819
vi.mock("@hyperframes/engine", async (importOriginal) => {
1920
const actual = await importOriginal<typeof import("@hyperframes/engine")>();
@@ -25,11 +26,25 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
2526
closeCaptureSession: (...args: unknown[]) => closeCaptureSession(...args),
2627
verifyDiskDrawElementSamples: (...args: unknown[]) => verifyDiskDrawElementSamples(...args),
2728
getCapturePerfSummary: (...args: unknown[]) => getCapturePerfSummary(...args),
29+
executeParallelCapture: (...args: unknown[]) => executeParallelCapture(...args),
30+
mergeWorkerFrames: async () => {},
31+
};
32+
});
33+
34+
const freeBytes = vi.fn<(path: string) => { bavail: number; bsize: number }>();
35+
vi.mock("node:fs", async (importOriginal) => {
36+
const actual = await importOriginal<typeof import("node:fs")>();
37+
return {
38+
...actual,
39+
statfsSync: (path: string) => freeBytes(path) ?? actual.statfsSync(path),
2840
};
2941
});
3042

3143
import {
3244
assertDiskCaptureHeadroom,
45+
createDiskCaptureProjection,
46+
diskCaptureShortfallError,
47+
measureCaptureFrameBytes,
3348
estimateDiskCaptureBytes,
3449
inspectDiskCaptureHeadroom,
3550
runCaptureStage,
@@ -55,13 +70,19 @@ describe("disk capture capacity", () => {
5570
format: "jpeg" as const,
5671
};
5772

73+
const pngOptions = { ...captureOptions, format: "png" as const };
74+
5875
it("estimates output-resolution frame storage conservatively", () => {
59-
expect(estimateDiskCaptureBytes(10, captureOptions)).toBe(800_000);
76+
expect(estimateDiskCaptureBytes(10, pngOptions)).toBe(800_000);
77+
});
78+
79+
it("estimates jpeg frames well below the raw RGBA model", () => {
80+
expect(estimateDiskCaptureBytes(10, captureOptions)).toBe(100_000);
6081
});
6182

6283
it("fails before capture when estimated frames exceed available headroom", () => {
6384
expect(() =>
64-
assertDiskCaptureHeadroom("/render/captured-frames", 10, captureOptions, () => 800_000),
85+
assertDiskCaptureHeadroom("/render/captured-frames", 10, pngOptions, () => 800_000),
6586
).toThrow(/may need ~0\.8 MB.*0\.8 MB is free.*--low-memory-mode/s);
6687
});
6788

@@ -70,7 +91,7 @@ describe("disk capture capacity", () => {
7091
width: 1920,
7192
height: 1080,
7293
fps: { num: 30, den: 1 },
73-
format: "jpeg" as const,
94+
format: "png" as const,
7495
};
7596
let message = "";
7697
try {
@@ -92,28 +113,27 @@ describe("disk capture capacity", () => {
92113
});
93114

94115
it("exposes the same 90% headroom decision to fallback planning", () => {
95-
const estimatedBytes = estimateDiskCaptureBytes(10, captureOptions);
116+
const estimatedBytes = estimateDiskCaptureBytes(10, pngOptions);
96117

97118
expect(
98119
inspectDiskCaptureHeadroom(
99120
"/render/captured-frames",
100121
10,
101-
captureOptions,
122+
pngOptions,
102123
() => estimatedBytes / 0.9 - 1,
103124
),
104125
).toEqual({ available: false, estimatedBytes, freeBytes: estimatedBytes / 0.9 - 1 });
105126
expect(
106-
inspectDiskCaptureHeadroom("/render/captured-frames", 10, captureOptions, () => null)
107-
.available,
127+
inspectDiskCaptureHeadroom("/render/captured-frames", 10, pngOptions, () => null).available,
108128
).toBe(true);
109129
});
110130

111-
it("rejects the reported 5318-frame landscape disk route with 45 GiB free", () => {
131+
it("rejects the reported 5318-frame landscape png disk route with 45 GiB free", () => {
112132
const landscape = {
113133
width: 1920,
114134
height: 1080,
115135
fps: { num: 30, den: 1 },
116-
format: "jpeg" as const,
136+
format: "png" as const,
117137
};
118138
const headroom = inspectDiskCaptureHeadroom(
119139
"/render/captured-frames",
@@ -125,6 +145,179 @@ describe("disk capture capacity", () => {
125145
expect(headroom.estimatedBytes).toBe(5318 * 1920 * 1080 * 4);
126146
expect(headroom.available).toBe(false);
127147
});
148+
149+
it("lets the same 5318-frame landscape route through as jpeg", () => {
150+
const landscape = { ...captureOptions, width: 1920, height: 1080, deviceScaleFactor: 1 };
151+
expect(
152+
inspectDiskCaptureHeadroom("/render/captured-frames", 5318, landscape, () => 45 * 1024 ** 3)
153+
.available,
154+
).toBe(true);
155+
});
156+
});
157+
158+
describe("createDiskCaptureProjection", () => {
159+
it("throws a measured shortfall once ten frames show the remainder cannot fit", () => {
160+
const measure = vi.fn(() => 10e6);
161+
const check = createDiskCaptureProjection({
162+
framesDir: "/frames",
163+
totalFrames: 1010,
164+
measureBytes: measure,
165+
freeDiskBytes: () => 500e6,
166+
});
167+
check(9);
168+
expect(measure).not.toHaveBeenCalled();
169+
expect(() => check(10)).toThrow(
170+
/~1000\.0 MB.*measured from the first 10 frames.*500\.0 MB is free at \/frames/s,
171+
);
172+
expect(() => check(11)).not.toThrow();
173+
});
174+
175+
it("does not spend its one check on an empty measurement", () => {
176+
let bytes = 0;
177+
const check = createDiskCaptureProjection({
178+
framesDir: "/frames",
179+
totalFrames: 1010,
180+
measureBytes: () => bytes,
181+
freeDiskBytes: () => 500e6,
182+
});
183+
expect(() => check(10)).not.toThrow();
184+
bytes = 10e6;
185+
expect(() => check(11)).toThrow(/measured from the first/);
186+
});
187+
188+
it("measures frames in retry-batch worker dirs as well as capture-attempt dirs", () => {
189+
const workDir = mkdtempSync(join(tmpdir(), "hf-measure-"));
190+
try {
191+
const framesDir = join(workDir, "captured-frames");
192+
for (const dir of [
193+
framesDir,
194+
join(workDir, "capture-attempt-0", "worker-0"),
195+
join(workDir, "retry-1-batch-0-worker-0"),
196+
]) {
197+
mkdirSync(dir, { recursive: true });
198+
writeFileSync(join(dir, "frame_000001.jpg"), Buffer.alloc(1000));
199+
}
200+
expect(measureCaptureFrameBytes(workDir, framesDir)).toBe(3000);
201+
} finally {
202+
rmSync(workDir, { recursive: true, force: true });
203+
}
204+
});
205+
206+
it("stays quiet when the projected remainder fits", () => {
207+
const check = createDiskCaptureProjection({
208+
framesDir: "/frames",
209+
totalFrames: 1010,
210+
measureBytes: () => 10e6,
211+
freeDiskBytes: () => 5000e6,
212+
});
213+
expect(() => check(10)).not.toThrow();
214+
});
215+
});
216+
217+
describe("runCaptureStage measured disk projection (both branches)", () => {
218+
const roots: string[] = [];
219+
const oneMb = Buffer.alloc(1_000_000);
220+
const makeInput = (workerCount: number, workDir: string, framesDir: string) => ({
221+
fileServer: { url: "http://localhost:0" } as never,
222+
workDir,
223+
framesDir,
224+
job: createRenderJob({ fps: { num: 30, den: 1 }, quality: "draft" }),
225+
totalFrames: 20,
226+
cfg: DEFAULT_CONFIG,
227+
plan: { workerCount, forceScreenshot: false },
228+
log: { error: vi.fn(), warn: vi.fn(), info: vi.fn(), debug: vi.fn() } as never,
229+
probeSession: null,
230+
captureAttempts: [],
231+
dedupPerfs: [],
232+
buildCaptureOptions: () => ({
233+
width: 64,
234+
height: 64,
235+
fps: { num: 30, den: 1 },
236+
format: "jpeg" as const,
237+
}),
238+
createRenderVideoFrameInjector: () => null,
239+
abortSignal: undefined,
240+
assertNotAborted: () => {},
241+
});
242+
const makeDirs = () => {
243+
const workDir = mkdtempSync(join(tmpdir(), "hf-disk-projection-"));
244+
const framesDir = join(workDir, "captured-frames");
245+
mkdirSync(framesDir);
246+
roots.push(workDir);
247+
return { workDir, framesDir };
248+
};
249+
250+
afterEach(() => {
251+
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
252+
createCaptureSession.mockReset();
253+
captureFrame.mockReset();
254+
executeParallelCapture.mockReset();
255+
freeBytes.mockReset();
256+
closeCaptureSession.mockReset().mockResolvedValue(undefined);
257+
});
258+
259+
it("sequential: stops after ten frames when the measured remainder cannot fit", async () => {
260+
const { workDir, framesDir } = makeDirs();
261+
// 8 MB free: passes the static jpeg estimate, fails the 10 MB measured remainder.
262+
freeBytes.mockReturnValue({ bavail: 8_000_000, bsize: 1 });
263+
createCaptureSession.mockResolvedValue({
264+
isInitialized: false,
265+
browserConsoleBuffer: [],
266+
options: { captureBeyondViewport: false, format: "jpeg" },
267+
workerEncodeEnabled: false,
268+
outputDir: framesDir,
269+
});
270+
captureFrame.mockImplementation(async (_s: unknown, index: number) => {
271+
writeFileSync(join(framesDir, formatCaptureFrameName(index, "jpg")), oneMb);
272+
});
273+
274+
await expect(runCaptureStage(makeInput(1, workDir, framesDir))).rejects.toThrow(
275+
/measured from the first 10 frames/,
276+
);
277+
expect(captureFrame).toHaveBeenCalledTimes(10);
278+
expect(createCaptureSession).toHaveBeenCalledTimes(1);
279+
});
280+
281+
it("a measured shortfall is never classified as a retryable browser death", () => {
282+
const shortfall = diskCaptureShortfallError(10e9, 1e9, "/frames", "measured");
283+
const kind = classifyCaptureFailure(shortfall);
284+
expect(kind).not.toBe("transient_browser");
285+
expect(isTransientCaptureRetryEligible(kind, [{ startFrame: 10, endFrame: 20 }], 0)).toBe(
286+
false,
287+
);
288+
});
289+
290+
it("parallel: the progress callback raises the same shortfall from worker dirs", async () => {
291+
const { workDir, framesDir } = makeDirs();
292+
freeBytes.mockReturnValue({ bavail: 8_000_000, bsize: 1 });
293+
executeParallelCapture.mockImplementation(
294+
async (
295+
_url: string,
296+
attemptWorkDir: string,
297+
_tasks: unknown,
298+
_opts: unknown,
299+
_hook: unknown,
300+
_signal: unknown,
301+
onProgress: (p: {
302+
capturedFrames: number;
303+
totalFrames: number;
304+
activeWorkers: number;
305+
}) => void,
306+
) => {
307+
const workerDir = join(attemptWorkDir, "worker-0");
308+
mkdirSync(workerDir, { recursive: true });
309+
for (let i = 0; i < 10; i++) {
310+
writeFileSync(join(workerDir, formatCaptureFrameName(i, "jpg")), oneMb);
311+
}
312+
onProgress({ capturedFrames: 10, totalFrames: 20, activeWorkers: 2 });
313+
return [];
314+
},
315+
);
316+
317+
await expect(runCaptureStage(makeInput(2, workDir, framesDir))).rejects.toThrow(
318+
/measured from the first 10 frames/,
319+
);
320+
});
128321
});
129322

130323
describe("runCaptureStage — sequential path transient Target-closed single retry (integration)", () => {

0 commit comments

Comments
 (0)