Skip to content

Commit d403d17

Browse files
committed
fix(producer): every retry batch gets its own first-frame window; the stall test waits for the render to settle
A retry that runs as several batches opened one stall window for the whole attempt, so a later batch whose browsers take longer than the no-progress window to land a frame was killed where main would have finished. The window now opens per batch. The stall tests also waited a fixed number of turns for real file IO after the last frame, which failed about 3 runs in 10 on a loaded machine; they now wait for the render to settle.
1 parent 2f20159 commit d403d17

2 files changed

Lines changed: 38 additions & 9 deletions

File tree

‎packages/producer/src/services/renderOrchestrator.stall.test.ts‎

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import { createRenderJob, executeRenderJob, type ProgressCallback } from "./rend
88

99
// Entry: executeRenderJob with the real disk capture stage and retry ladder. Each parallel attempt is
1010
// scripted: "stall" lands three frames then hangs until its signal aborts; a pace lands every frame.
11-
type AttemptScript = "stall" | { firstFrameAfterMs: number; frameEveryMs: number };
11+
type AttemptScript = "stall" | "spread" | { firstFrameAfterMs: number; frameEveryMs: number };
1212
const capture = vi.hoisted(() => ({
1313
scripts: [] as AttemptScript[],
1414
calls: 0,
@@ -49,10 +49,16 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
4949
index: task.startFrame + i,
5050
})),
5151
);
52-
const landed = frames.slice(0, script === "stall" ? 3 : frames.length);
52+
const stalls = script === "stall" || script === "spread";
53+
// "spread" lands the first frame of each worker's range, leaving four missing ranges.
54+
const landed =
55+
script === "spread"
56+
? frames.filter((frame) => frame.index % 10 === 0)
57+
: frames.slice(0, script === "stall" ? 3 : frames.length);
5358
let captured = 0;
5459
for (const frame of landed) {
55-
await wait(captured === 0 && script !== "stall" ? script.firstFrameAfterMs : 1_000);
60+
await wait(captured === 0 && !stalls ? script.firstFrameAfterMs : 1_000);
61+
if (signal?.aborted) throw new Error("worker cancelled");
5662
mkdirSync(frame.dir, { recursive: true });
5763
writeFileSync(
5864
join(
@@ -63,9 +69,9 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
6369
);
6470
captured += 1;
6571
onProgress({ capturedFrames: captured, totalFrames: frames.length, activeWorkers: 1 });
66-
if (script !== "stall") await wait(script.frameEveryMs - 1_000);
72+
if (!stalls) await wait(script.frameEveryMs - 1_000);
6773
}
68-
if (script === "stall") {
74+
if (stalls) {
6975
await new Promise((_, reject) =>
7076
signal?.addEventListener("abort", () => reject(new Error("worker cancelled")), {
7177
once: true,
@@ -215,6 +221,14 @@ async function ioTurns(): Promise<void> {
215221
for (let turn = 0; turn < 50; turn++) await new Promise((resolve) => setImmediate(resolve));
216222
}
217223

224+
/** Real file IO after the last frame takes a varying number of turns: wait for the render to settle. */
225+
async function settled(outcome: () => unknown): Promise<unknown> {
226+
for (let turn = 0; turn < 20_000 && outcome() === PENDING; turn++) {
227+
await new Promise((resolve) => setImmediate(resolve));
228+
}
229+
return outcome();
230+
}
231+
218232
/** Advances fake time a second at a time, letting real file IO run between steps. */
219233
async function runFor(ms: number, outcome: () => unknown): Promise<void> {
220234
for (let elapsed = 0; elapsed < ms && outcome() === PENDING; elapsed += 1_000) {
@@ -229,7 +243,21 @@ it("retries a stalled attempt on fewer workers, with a fresh window, and the ren
229243
const outcome = startRender();
230244
await runFor(400_000, outcome);
231245
expect(capture.retriedWithWorkers).toEqual([2]);
232-
expect(outcome()).toBe("completed");
246+
expect(await settled(outcome)).toBe("completed");
247+
});
248+
249+
it("gives each retry batch its own first-frame window", async () => {
250+
// Four missing ranges on two workers: two batches; the second pool lands its first frame after 100 s.
251+
capture.scripts = [
252+
"spread",
253+
{ firstFrameAfterMs: 1_000, frameEveryMs: 1_000 },
254+
{ firstFrameAfterMs: 100_000, frameEveryMs: 1_000 },
255+
];
256+
const outcome = startRender();
257+
await runFor(600_000, outcome);
258+
expect(capture.retriedWithWorkers).toEqual([2]);
259+
expect(capture.calls).toBe(3);
260+
expect(await settled(outcome)).toBe("completed");
233261
});
234262

235263
it("fails typed and non-retryable once every attempt down the ladder stalls", async () => {
@@ -249,7 +277,7 @@ it("never kills a capture that keeps landing frames", async () => {
249277
const outcome = startRender();
250278
await runFor(250_000 + 40 * 50_000 + 10_000, outcome);
251279
expect(capture.calls).toBe(1);
252-
expect(outcome()).toBe("completed");
280+
expect(await settled(outcome)).toBe("completed");
253281
});
254282

255283
it("fails a sequential disk capture at the first frame after its window ran out", async () => {
@@ -273,5 +301,5 @@ it("completes even when the progress sink never drains", async () => {
273301
const sinkNeverDrains: ProgressCallback = () => new Promise<void>(() => {});
274302
const outcome = startRender(sinkNeverDrains);
275303
await runFor(120_000, outcome);
276-
expect(outcome()).toBe("completed");
304+
expect(await settled(outcome)).toBe("completed");
277305
});

‎packages/producer/src/services/renderOrchestrator.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1300,9 +1300,10 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
13001300
// twice, inflating reused/predicted past totalFrames. The common no-retry path keeps
13011301
// exactly one attempt's perf; a retry reports only the final attempt's set.
13021302
options.dedupPerfs.length = 0;
1303-
const attemptSignal = options.openAttempt?.(options.abortSignal) ?? options.abortSignal;
1303+
let attemptSignal = options.abortSignal;
13041304
try {
13051305
for (const tasks of batches) {
1306+
attemptSignal = options.openAttempt?.(options.abortSignal) ?? options.abortSignal;
13061307
const capturedBeforeBatch = countCapturedFrames(
13071308
options.totalFrames,
13081309
options.framesDir,

0 commit comments

Comments
 (0)