Skip to content

Commit 9e0d05d

Browse files
fix(engine): auto worker count stays within the heap budget (#5149)
* fix(engine): automatic worker count never exceeds what the V8 heap can feed A 24 GB Mac exporting a 43 s portrait project with 22 videos picked 5 capture workers against a 4192 MB heap that feeds about 4, and the render died at 62% with JavaScript heap out of memory. The heap budget was only advisory; it now caps the automatic count (bound 'heap'). An explicit --workers N is untouched. * test(engine): heap-cap test restores its mocks even when an assertion fails Review findings on #5149: the os and v8 mocks were restored only if every assertion passed, and two comments still called the heap cap advisory. * docs(engine): heap budget comments say it caps auto sizing
1 parent 9fbec22 commit 9e0d05d

5 files changed

Lines changed: 57 additions & 20 deletions

File tree

‎packages/cli/src/telemetry/events.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -342,8 +342,8 @@ export function trackRenderComplete(
342342
catalogUsage?: CatalogUsage;
343343
workers?: number;
344344
// Worker auto-sizing provenance (RenderPerfSummary.workerSizing). Answers
345-
// "why N workers?" fleet-wide, and validates the advisory per-worker heap
346-
// budget before it's enforced (field OOM: 6 auto workers on a 24GB/4GB-heap
345+
// "why N workers?" fleet-wide, and reports the per-worker heap budget
346+
// that caps auto sizing (field OOM: 6 auto workers on a 24GB/4GB-heap
347347
// machine — see computeWorkerSizing in @hyperframes/engine).
348348
workersBoundBy?: string;
349349
workersCpuBased?: number;

‎packages/engine/src/services/parallelCoordinator.test.ts‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
import { describe, it, expect, vi } from "vitest";
1+
import { cpus, totalmem } from "os";
2+
import { getHeapStatistics } from "v8";
3+
import { afterEach, describe, it, expect, vi } from "vitest";
24
import {
35
calculateOptimalWorkers,
46
computeWorkerSizing,
@@ -128,7 +130,43 @@ describe("calculateOptimalWorkers", () => {
128130
});
129131
});
130132

133+
vi.mock("os", async (importOriginal) => {
134+
const os = await importOriginal<typeof import("os")>();
135+
return { ...os, default: os, cpus: vi.fn(os.cpus), totalmem: vi.fn(os.totalmem) };
136+
});
137+
vi.mock("v8", async (importOriginal) => {
138+
const v8 = await importOriginal<typeof import("v8")>();
139+
return { ...v8, default: v8, getHeapStatistics: vi.fn(v8.getHeapStatistics) };
140+
});
141+
131142
describe("computeWorkerSizing", () => {
143+
afterEach(() => {
144+
vi.mocked(cpus).mockRestore();
145+
vi.mocked(totalmem).mockRestore();
146+
vi.mocked(getHeapStatistics).mockRestore();
147+
});
148+
149+
// The field case: a 24GB 14-core Mac, Node's default ~4GB heap, a long 1080x1920 project.
150+
it("never auto-picks more workers than the V8 heap can feed", () => {
151+
vi.mocked(cpus).mockReturnValue(
152+
Array.from({ length: 14 }, () => ({}) as ReturnType<typeof cpus>[number]),
153+
);
154+
vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3);
155+
vi.mocked(getHeapStatistics).mockReturnValue({
156+
heap_size_limit: 4192 * 1024 ** 2,
157+
} as ReturnType<typeof getHeapStatistics>);
158+
const sizing = computeWorkerSizing(1300, undefined, { concurrency: "auto" });
159+
expect(sizing.heapBasedWorkers).toBe(4);
160+
expect(sizing.workers).toBe(4);
161+
expect(sizing.boundBy).toBe("heap");
162+
expect(sizing.exceedsHeapAdvisory).toBe(false);
163+
expect(computeWorkerSizing(1300, 6, { concurrency: "auto" }).workers).toBe(6);
164+
vi.mocked(getHeapStatistics).mockReturnValue({
165+
heap_size_limit: 8192 * 1024 ** 2,
166+
} as ReturnType<typeof getHeapStatistics>);
167+
expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).workers).toBe(5);
168+
});
169+
132170
it("matches calculateOptimalWorkers and reports every constraint", () => {
133171
const config = { concurrency: "auto" as const };
134172
const sizing = computeWorkerSizing(900, undefined, config);

‎packages/engine/src/services/parallelCoordinator.ts‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -141,10 +141,8 @@ const MEMORY_PER_WORKER_MB = 1536;
141141
const HEAP_RESERVED_MB = 1024;
142142
// Parent-process V8 heap consumed per worker (protocol buffers + in-flight
143143
// frame buffers). Derived from the field OOM: 6 workers exhausted a ~4GB
144-
// default heap ⇒ >~500MB/worker + base. ponytail: advisory-only until the
145-
// workers_heap_* telemetry added alongside this constant validates the figure
146-
// — enforcing a guessed budget could silently cut worker counts fleet-wide.
147-
// TODO(PRINFRA-341): decide enforcement after ~2 weeks of fleet soak.
144+
// default heap ⇒ >~500MB/worker + base. Caps auto sizing; an explicit
145+
// `--workers N` is still the operator's call.
148146
const HEAP_PER_WORKER_MB = 640;
149147
const MIN_WORKERS = 1;
150148
const MAX_WORKER_DIAGNOSTIC_LINES = 8;
@@ -275,7 +273,8 @@ export type WorkerSizingBound =
275273
| "frames"
276274
| "max_workers"
277275
| "min_parallel_floor"
278-
| "contention";
276+
| "contention"
277+
| "heap";
279278

280279
/**
281280
* Full provenance of a worker-sizing decision. Threaded into render
@@ -290,17 +289,16 @@ export interface WorkerSizing {
290289
frameBasedWorkers: number;
291290
effectiveMaxWorkers: number;
292291
/**
293-
* ADVISORY, not enforced (see HEAP_PER_WORKER_MB): how many workers the
294-
* parent process's V8 heap could feed. Compare against `workers` in
295-
* telemetry to validate the budget before enforcement.
292+
* How many workers the parent process's V8 heap can feed (see
293+
* HEAP_PER_WORKER_MB); auto sizing never exceeds it.
296294
*/
297295
heapBasedWorkers: number;
298296
/** V8 `heap_size_limit` for the parent process, MB. */
299297
heapLimitMb: number;
300298
totalMemoryMb: number;
301299
cpuCount: number;
302300
captureCostMultiplier: number;
303-
/** true when the chosen count exceeds the advisory heap budget. */
301+
/** true when the chosen count exceeds the heap budget (only an explicit request can). */
304302
exceedsHeapAdvisory: boolean;
305303
}
306304

@@ -405,6 +403,11 @@ export function computeWorkerSizing(
405403
}
406404
}
407405

406+
if (finalWorkers > heapBasedWorkers) {
407+
finalWorkers = heapBasedWorkers;
408+
boundBy = "heap";
409+
}
410+
408411
return finish(finalWorkers, boundBy, effectiveMaxWorkers);
409412
}
410413

‎packages/producer/src/services/render/captureCost.ts‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -116,11 +116,8 @@ function combineCaptureCostEstimates(
116116
* - Auto-sized renders only (`requestedWorkers === undefined`) — the field
117117
* failure was auto sizing, and an explicit `--workers N` is the operator's
118118
* own call.
119-
* - Not enforced as a cap yet — the per-worker budget constant is derived
120-
* from one field report; the `workers_heap_*` telemetry emitted with the
121-
* sizing decides whether to enforce (see the TODO on HEAP_PER_WORKER_MB in
122-
* @hyperframes/engine's parallelCoordinator). The message gives the
123-
* operator the actionable knobs today.
119+
* - Auto sizing is capped at the heap budget (computeWorkerSizing), so the
120+
* warning cannot fire today; it stays for a budget that stops being a cap.
124121
*
125122
* Pure so the message shape + firing condition are unit-testable with a
126123
* synthetic `WorkerSizing` (the real one depends on the host's heap).

‎packages/producer/src/services/renderOrchestrator.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -455,9 +455,8 @@ export interface RenderPerfSummary {
455455
/**
456456
* Provenance of the auto worker-sizing decision (undefined when the
457457
* htmlInCanvas / low-memory pins short-circuited sizing). `boundBy` names
458-
* the binding constraint; the heap fields are the advisory budget being
459-
* validated by fleet telemetry before enforcement — see
460-
* `computeWorkerSizing` in @hyperframes/engine.
458+
* the binding constraint; the heap fields are the budget that caps auto
459+
* sizing — see `computeWorkerSizing` in @hyperframes/engine.
461460
*/
462461
workerSizing?: WorkerSizing;
463462
chunkedEncode: boolean;

0 commit comments

Comments
 (0)