Skip to content

Commit cd003e2

Browse files
perf(engine): size render workers from measured heap and the container's CPU quota (#6641)
* perf(cli): give render the heap its auto workers need in small containers Auto worker sizing caps capture workers at (V8 heap - 1024 MB) / 640 MB. Node sizes its default heap from visible memory, so in a 10 GB container the heap is 2,096 MB and every render captures with one worker, whatever the core count. When no --workers is given, no --docker, no heap was set by hand, and the heap is the only bound holding workers back, render relaunches itself once with --max-old-space-size set to what CPU and memory allow (capped at half of memory). The engine exposes heapMbForAutoWorkers() for this. * fix(cli): count a heap-relaunched render once, keep detached renders alive The relaunched child skips the CLI's command telemetry, update and skills notices, and app-history notice; the parent already records and prints them. A detached render no longer gets SIGHUP forwarded, which its default handler would treat as a kill. A heap set as --max_old_space_size counts as set by hand. * fix(engine): keep the desktop app's per-worker heap figure at 640 MB Inside Electron the heap is a hard 4,192 MB cage, and that is where the field OOM behind the old figure happened (#5149: 5 auto workers on a heavy export). The new 320 MB figure is measured only in a system Node, so the desktop app keeps 640 MB (4 workers there) until a heavy desktop export is measured. Containers and plain Node keep 320. * fix(engine): know the packaged desktop app by its client tag, which runs the CLI on plain Node
1 parent 49e0312 commit cd003e2

4 files changed

Lines changed: 128 additions & 11 deletions

File tree

‎packages/engine/src/services/parallelCoordinator.test.ts‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -153,11 +153,11 @@ describe("computeWorkerSizing", () => {
153153
);
154154
vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3);
155155
vi.mocked(getHeapStatistics).mockReturnValue({
156-
heap_size_limit: 4192 * 1024 ** 2,
156+
heap_size_limit: 2096 * 1024 ** 2,
157157
} as ReturnType<typeof getHeapStatistics>);
158158
const sizing = computeWorkerSizing(1300, undefined, { concurrency: "auto" });
159-
expect(sizing.heapBasedWorkers).toBe(4);
160-
expect(sizing.workers).toBe(4);
159+
expect(sizing.heapBasedWorkers).toBe(3);
160+
expect(sizing.workers).toBe(3);
161161
expect(sizing.boundBy).toBe("heap");
162162
expect(sizing.exceedsHeapAdvisory).toBe(false);
163163
expect(computeWorkerSizing(1300, 6, { concurrency: "auto" }).workers).toBe(6);
@@ -167,6 +167,35 @@ describe("computeWorkerSizing", () => {
167167
expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).workers).toBe(5);
168168
});
169169

170+
it("keeps the desktop app's 4,192 MB heap at the old per-worker figure (#5149)", () => {
171+
vi.mocked(cpus).mockReturnValue(
172+
Array.from({ length: 14 }, () => ({}) as ReturnType<typeof cpus>[number]),
173+
);
174+
vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3);
175+
vi.mocked(getHeapStatistics).mockReturnValue({
176+
heap_size_limit: 4192 * 1024 ** 2,
177+
} as ReturnType<typeof getHeapStatistics>);
178+
expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).heapBasedWorkers).toBe(9);
179+
Object.defineProperty(process.versions, "electron", { value: "44.2.0", configurable: true });
180+
try {
181+
const sizing = computeWorkerSizing(1300, undefined, { concurrency: "auto" });
182+
expect(sizing.heapBasedWorkers).toBe(4);
183+
expect(sizing.workers).toBe(4);
184+
expect(sizing.boundBy).toBe("heap");
185+
} finally {
186+
delete (process.versions as Record<string, string | undefined>).electron;
187+
}
188+
// A packaged app runs the CLI on its own plain Node, tagged as the desktop client.
189+
vi.stubEnv("HYPERFRAMES_CLIENT", "desktop/1.4.0/stable");
190+
try {
191+
expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).heapBasedWorkers).toBe(
192+
4,
193+
);
194+
} finally {
195+
vi.unstubAllEnvs();
196+
}
197+
});
198+
170199
it("sizes Apple-GPU disk capture by its own heap and core budget", () => {
171200
vi.mocked(cpus).mockReturnValue(
172201
Array.from({ length: 14 }, () => ({}) as ReturnType<typeof cpus>[number]),

‎packages/engine/src/services/parallelCoordinator.ts‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
3131
import { assertSwiftShader } from "../utils/assertSwiftShader.js";
3232
import { readWebGlVendorInfoFromCanvas } from "../utils/readWebGlVendorInfoFromCanvas.js";
3333
import { resolveHeadlessShellPath } from "./browserManager.js";
34-
import { getSystemTotalMb } from "./systemMemory.js";
34+
import { getCgroupCpuLimit, getSystemTotalMb } from "./systemMemory.js";
3535
import {
3636
CaptureFailure,
3737
classifyCaptureFailure,
@@ -148,11 +148,14 @@ const MEMORY_PER_WORKER_MB = 1536;
148148
// Parent-process V8 heap the coordinator itself needs regardless of worker
149149
// count (compile artifacts, puppeteer sessions, encoder state).
150150
const HEAP_RESERVED_MB = 1024;
151-
// Parent-process V8 heap consumed per worker (protocol buffers + in-flight
152-
// frame buffers). Derived from the field OOM: 6 workers exhausted a ~4GB
153-
// default heap ⇒ >~500MB/worker + base. Caps auto sizing; an explicit
154-
// `--workers N` is still the operator's call.
155-
const HEAP_PER_WORKER_MB = 640;
151+
// Parent heap per worker, by --trace-gc in a 10 GB container (10-28): 21 MB on a short film, 228 MB on a 1080×1920
152+
// video-heavy one (427 → 1,110 MB from 1 to 4 workers). 320 keeps 1.4× over that; `--workers N` still overrides.
153+
const HEAP_PER_WORKER_MB = 320;
154+
// The desktop app keeps the old figure: its 4,192 MB heap is where 5 workers ran out (#5149). A packaged app runs the
155+
// CLI on its own plain Node, so it is known by the client tag it sets on every CLI call, or by Electron itself.
156+
const DESKTOP_HEAP_PER_WORKER_MB = 640;
157+
const inDesktopApp = () =>
158+
Boolean(process.versions.electron) || /^desktop\//.test(process.env.HYPERFRAMES_CLIENT ?? "");
156159
// Apple-GPU disk capture measured ~15-25 MB parent heap per worker; 14 cores kept gaining to 8.
157160
const APPLE_DISK_HEAP_PER_WORKER_MB = 128;
158161
const APPLE_DISK_CORES_PER_WORKER = 1.75;
@@ -332,10 +335,19 @@ export function computeWorkerSizing(
332335
1,
333336
Math.floor(
334337
(heapLimitMb - HEAP_RESERVED_MB) /
335-
(appleDisk ? APPLE_DISK_HEAP_PER_WORKER_MB : HEAP_PER_WORKER_MB),
338+
(appleDisk
339+
? APPLE_DISK_HEAP_PER_WORKER_MB
340+
: inDesktopApp()
341+
? DESKTOP_HEAP_PER_WORKER_MB
342+
: HEAP_PER_WORKER_MB),
336343
),
337344
);
338-
const cpuBasedWorkers = Math.max(1, cpuCount - 2);
345+
// A CPU quota (docker --cpus) bounds workers at its whole CPUs: at 3.5, 3 beat 2 and 4 (johnny, 10-28).
346+
const cpuQuota = getCgroupCpuLimit();
347+
const cpuBasedWorkers = Math.max(
348+
1,
349+
cpuQuota === null ? cpuCount - 2 : Math.min(cpuCount - 2, Math.floor(cpuQuota)),
350+
);
339351
const memoryBasedWorkers = Math.max(1, Math.floor((totalMemoryMb * 0.5) / MEMORY_PER_WORKER_MB));
340352
const frameBasedWorkers = Math.floor(totalFrames / MIN_FRAMES_PER_WORKER);
341353
const captureCostMultiplier = Math.max(1, config?.captureCostMultiplier ?? 1);

‎packages/engine/src/services/systemMemory.test.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";
33
import {
44
_resetCgroupLimitCacheForTests,
55
isLowMemorySystem,
6+
parseCgroupCpus,
67
LOW_MEMORY_TOTAL_MB_THRESHOLD,
78
parseCgroupLimitMb,
89
} from "./systemMemory.js";
@@ -155,6 +156,40 @@ describe("parseCgroupLimitMb", () => {
155156
});
156157
});
157158

159+
describe("parseCgroupCpus", () => {
160+
it("reads a v2 cpu.max quota as CPUs", () => {
161+
expect(parseCgroupCpus("350000 100000\n", null, null)).toBe(3.5);
162+
expect(parseCgroupCpus("max 100000", null, null)).toBeNull();
163+
});
164+
165+
it("reads a v1 quota over its period, and -1 as no quota", () => {
166+
expect(parseCgroupCpus(null, "400000\n", "100000\n")).toBe(4);
167+
expect(parseCgroupCpus(null, "-1", "100000")).toBeNull();
168+
});
169+
170+
it("ignores absent and malformed quotas", () => {
171+
expect(parseCgroupCpus(null, null, null)).toBeNull();
172+
for (const content of ["", "garbage", "0 100000", "100000 0"])
173+
expect(parseCgroupCpus(content, null, null)).toBeNull();
174+
});
175+
});
176+
177+
describe("getCgroupCpuLimit", () => {
178+
it("reads the quota from cpu.max, and none from a host without one", async () => {
179+
await withSystemMemoryMocks(
180+
{ files: { "/sys/fs/cgroup/cpu.max": "350000 100000" } },
181+
({ getCgroupCpuLimit }) => expect(getCgroupCpuLimit()).toBe(3.5),
182+
);
183+
await withSystemMemoryMocks(
184+
{ files: { "/sys/fs/cgroup/cpu.max": "max 100000" } },
185+
({ getCgroupCpuLimit }) => expect(getCgroupCpuLimit()).toBeNull(),
186+
);
187+
await withSystemMemoryMocks({ platform: "darwin" }, ({ getCgroupCpuLimit }) =>
188+
expect(getCgroupCpuLimit()).toBeNull(),
189+
);
190+
});
191+
});
192+
158193
describe("getCgroupMemoryLimitMb", () => {
159194
it("returns only an actual cgroup limit and never host RAM", async () => {
160195
await withSystemMemoryMocks(

‎packages/engine/src/services/systemMemory.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,12 @@ const CGROUP_V1_MEMORY_LIMIT_PATH = "/sys/fs/cgroup/memory/memory.limit_in_bytes
2626
// Kernel no-limit sentinel is page-rounded 2^63-1 (~9223372036854771712); >= 2^60 is implausible as a real limit.
2727
const CGROUP_V1_NO_LIMIT_CUTOFF_BYTES = 2n ** 60n;
2828

29+
const CGROUP_V2_CPU_MAX_PATH = "/sys/fs/cgroup/cpu.max";
30+
const CGROUP_V1_CPU_QUOTA_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us";
31+
const CGROUP_V1_CPU_PERIOD_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_period_us";
32+
2933
let _cachedCgroupLimitMb: number | null | undefined;
34+
let _cachedCgroupCpus: number | null | undefined;
3035
let _warnedCgroupReadFailure = false;
3136

3237
/** Parse cgroup v2/v1 memory limits from sysfs file contents into MiB. */
@@ -75,12 +80,48 @@ function parsePositiveByteLimitMb(content: string, noLimitCutoffBytes?: bigint):
7580
return Number(bytes / BYTES_PER_MIB_BIGINT);
7681
}
7782

83+
/** The CPUs a cgroup quota allows (v2 `cpu.max`, else v1 quota over period), or null when it sets none. */
84+
export function parseCgroupCpus(
85+
v2Content: string | null,
86+
v1Quota: string | null,
87+
v1Period: string | null,
88+
): number | null {
89+
if (v2Content !== null) {
90+
const [quota, period] = v2Content.trim().split(/\s+/);
91+
return quotaCpus(quota, period);
92+
}
93+
return quotaCpus(v1Quota?.trim(), v1Period?.trim());
94+
}
95+
96+
function quotaCpus(quota: string | undefined, period: string | undefined): number | null {
97+
if (!quota || !period || !/^\d+$/.test(quota) || !/^\d+$/.test(period)) return null;
98+
const [q, p] = [Number(quota), Number(period)];
99+
return q > 0 && p > 0 ? q / p : null;
100+
}
101+
78102
/** Test-only: reset the cached cgroup memory probe. */
79103
export function _resetCgroupLimitCacheForTests(): void {
80104
_cachedCgroupLimitMb = undefined;
105+
_cachedCgroupCpus = undefined;
81106
_warnedCgroupReadFailure = false;
82107
}
83108

109+
/** The CPUs this process's cgroup may use (docker `--cpus`), or null off Linux or without a quota. */
110+
export function getCgroupCpuLimit(): number | null {
111+
if (_cachedCgroupCpus !== undefined) return _cachedCgroupCpus;
112+
if (process.platform !== "linux") return (_cachedCgroupCpus = null);
113+
const v2 = readCgroupFile(CGROUP_V2_CPU_MAX_PATH);
114+
_cachedCgroupCpus =
115+
v2 !== null
116+
? parseCgroupCpus(v2, null, null)
117+
: parseCgroupCpus(
118+
null,
119+
readCgroupFile(CGROUP_V1_CPU_QUOTA_PATH),
120+
readCgroupFile(CGROUP_V1_CPU_PERIOD_PATH),
121+
);
122+
return _cachedCgroupCpus;
123+
}
124+
84125
/**
85126
* Actual Linux cgroup memory ceiling in MiB, or null when the process is not
86127
* cgroup-limited. Unlike getSystemTotalMb this never falls back to host RAM.

0 commit comments

Comments
 (0)