Skip to content

Commit d699cbf

Browse files
committed
fix(producer): attribute and stop oversized plans early
1 parent 5bfda4e commit d699cbf

3 files changed

Lines changed: 470 additions & 50 deletions

File tree

‎packages/producer/src/services/distributed/plan.ts‎

Lines changed: 101 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,7 @@
2424
* never have to handle them.
2525
*/
2626

27-
import {
28-
cpSync,
29-
existsSync,
30-
mkdirSync,
31-
readdirSync,
32-
renameSync,
33-
rmSync,
34-
statSync,
35-
writeFileSync,
36-
} from "node:fs";
27+
import { cpSync, existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
3728
import { join, relative, sep } from "node:path";
3829
import { type CanvasResolution, fpsToNumber } from "@hyperframes/core";
3930
import {
@@ -82,6 +73,11 @@ import {
8273
readProducerVersion,
8374
} from "./shared.js";
8475
import { CURRENT_PLAN_PROTOCOL, type PlanProtocolV1Descriptor } from "./planProtocol.js";
76+
import {
77+
measurePlanSizeBreakdown,
78+
type PlanSizeBreakdown,
79+
type PlanSizeRootKind,
80+
} from "./planSize.js";
8581

8682
/**
8783
* Caller-supplied configuration for a distributed render. `fps`, `width`,
@@ -360,18 +356,37 @@ export class PlanTooLargeError extends Error {
360356
readonly code: typeof PLAN_TOO_LARGE = PLAN_TOO_LARGE;
361357
readonly sizeBytes: number;
362358
readonly limitBytes: number;
363-
constructor(sizeBytes: number, limitBytes: number) {
359+
readonly breakdown?: PlanSizeBreakdown;
360+
readonly observedAt?: string;
361+
constructor(
362+
sizeBytes: number,
363+
limitBytes: number,
364+
breakdown?: PlanSizeBreakdown,
365+
observedAt?: string,
366+
) {
367+
const breakdownSuffix = breakdown
368+
? ` Breakdown: video-frames=${formatBytes(breakdown.videoFramesBytes)}, ` +
369+
`compiled=${formatBytes(breakdown.compiledBytes)} ` +
370+
`(source-media=${formatBytes(breakdown.sourceMediaBytes)}), ` +
371+
`audio=${formatBytes(breakdown.audioBytes)}, metadata=${formatBytes(breakdown.metadataBytes)}, ` +
372+
`other=${formatBytes(breakdown.otherBytes)}, files=${breakdown.fileCount}.`
373+
: "";
374+
const observationSuffix = observedAt ? ` Observed at ${observedAt}.` : "";
364375
super(
365376
`[plan] planDir size ${formatBytes(sizeBytes)} exceeds the configured ceiling ` +
366377
`${formatBytes(limitBytes)} (PLAN_TOO_LARGE). The default 2 GB cap fits inside AWS ` +
367378
`Lambda's 10 GB /tmp budget alongside the chunk worker's frame buffer and ffmpeg's ` +
368379
`working set. To unblock: use the content-addressed \`planV2()\` transport, shorten ` +
369380
`the composition, lower the framerate, or use the in-process renderer ` +
370-
`(\`executeRenderJob\`) — it has no planDir size cap.`,
381+
`(\`executeRenderJob\`) — it has no planDir size cap.` +
382+
observationSuffix +
383+
breakdownSuffix,
371384
);
372385
this.name = "PlanTooLargeError";
373386
this.sizeBytes = sizeBytes;
374387
this.limitBytes = limitBytes;
388+
this.breakdown = breakdown;
389+
this.observedAt = observedAt;
375390
}
376391
}
377392

@@ -454,30 +469,33 @@ export function rejectUnsupportedDistributedFormat(
454469
* walker outside the planDir.
455470
*/
456471
export function measurePlanDirBytes(planDir: string): number {
457-
let total = 0;
458-
function walk(dir: string): void {
459-
let entries;
460-
try {
461-
entries = readdirSync(dir, { withFileTypes: true });
462-
} catch {
463-
return;
464-
}
465-
for (const entry of entries) {
466-
const full = join(dir, entry.name);
467-
if (entry.isDirectory()) {
468-
walk(full);
469-
} else if (entry.isFile()) {
470-
try {
471-
total += statSync(full).size;
472-
} catch {
473-
// Ignore — a file disappearing during the walk shouldn't crash
474-
// the measurement.
475-
}
476-
}
477-
}
478-
}
479-
walk(planDir);
480-
return total;
472+
return measurePlanSizeBreakdown(planDir).totalBytes;
473+
}
474+
475+
function assertPlanSizeWithinLimit(input: {
476+
rootDir: string;
477+
rootKind: PlanSizeRootKind;
478+
limitBytes: number;
479+
observedAt: string;
480+
log: ProducerLogger;
481+
}): PlanSizeBreakdown {
482+
const breakdown = measurePlanSizeBreakdown(input.rootDir, input.rootKind);
483+
if (breakdown.totalBytes <= input.limitBytes) return breakdown;
484+
input.log.warn("[plan] size budget exceeded", {
485+
observedAt: input.observedAt,
486+
sizeBytes: breakdown.totalBytes,
487+
limitBytes: input.limitBytes,
488+
fileCount: breakdown.fileCount,
489+
compiledBytes: breakdown.compiledBytes,
490+
sourceMediaBytes: breakdown.sourceMediaBytes,
491+
videoFramesBytes: breakdown.videoFramesBytes,
492+
videoFrameFileCount: breakdown.videoFrameFileCount,
493+
audioBytes: breakdown.audioBytes,
494+
metadataBytes: breakdown.metadataBytes,
495+
otherBytes: breakdown.otherBytes,
496+
topComponents: breakdown.topComponents,
497+
});
498+
throw new PlanTooLargeError(breakdown.totalBytes, input.limitBytes, breakdown, input.observedAt);
481499
}
482500

483501
/**
@@ -566,6 +584,19 @@ function assertPositiveInteger(name: string, value: number): void {
566584
}
567585
}
568586

587+
const FREEZE_OWNED_PLAN_FILES = [
588+
"plan.json",
589+
join("meta", "composition.json"),
590+
join("meta", "encoder.json"),
591+
join("meta", "chunks.json"),
592+
] as const;
593+
594+
function removeFreezeOwnedPlanFiles(planDir: string): void {
595+
for (const relativePath of FREEZE_OWNED_PLAN_FILES) {
596+
rmSync(join(planDir, relativePath), { force: true });
597+
}
598+
}
599+
569600
/**
570601
* Slice `totalFrames` into `chunkCount` consecutive ranges. Each chunk gets
571602
* `effectiveChunkSize` frames except the last, which absorbs the remainder
@@ -770,6 +801,7 @@ export async function plan(
770801
if (!existsSync(planDir)) mkdirSync(planDir, { recursive: true });
771802

772803
const log = config.logger ?? defaultLogger;
804+
const sizeLimitBytes = config.planDirSizeLimitBytes ?? PLAN_DIR_SIZE_LIMIT_BYTES;
773805
const abortSignal = config.abortSignal;
774806
const assertNotAborted = (): void => {
775807
if (abortSignal?.aborted) {
@@ -935,6 +967,17 @@ export async function plan(
935967
}
936968
}
937969

970+
// The compiled tree is now stable. If it already exceeds the final plan
971+
// budget, extraction/audio/freeze can only add retained bytes, so fail before
972+
// generating a multi-GiB frame tree.
973+
assertPlanSizeWithinLimit({
974+
rootDir: compiledDir,
975+
rootKind: "compiled",
976+
limitBytes: sizeLimitBytes,
977+
observedAt: "pre-extract",
978+
log,
979+
});
980+
938981
// ── Extract videos ──
939982
// `materializeSymlinks: true` recursively copies frames so the planDir is
940983
// self-contained (symlinks don't survive S3/GCS round-trips).
@@ -1078,6 +1121,21 @@ export async function plan(
10781121
});
10791122
}
10801123

1124+
// A caller may reuse an existing planDir. freezePlan overwrites these four
1125+
// files, so remove stale versions before the preliminary measurement; the
1126+
// exact post-freeze check below still includes the newly written metadata.
1127+
removeFreezeOwnedPlanFiles(planDir);
1128+
1129+
// All retained heavy artifacts have been promoted and transient work files
1130+
// removed. Reject before freezePlan performs its full content-hash read.
1131+
assertPlanSizeWithinLimit({
1132+
rootDir: planDir,
1133+
rootKind: "plan",
1134+
limitBytes: sizeLimitBytes,
1135+
observedAt: "pre-freeze",
1136+
log,
1137+
});
1138+
10811139
const freezeResult = await freezePlan({
10821140
planDir,
10831141
composition: compositionJson,
@@ -1096,11 +1154,13 @@ export async function plan(
10961154
// alongside the chunk worker's frame buffer + ffmpeg working set. The
10971155
// check runs AFTER cleanup so the workDir tree doesn't double-count.
10981156
// Non-retryable: the same planDir would trip the cap on every retry.
1099-
const sizeLimitBytes = config.planDirSizeLimitBytes ?? PLAN_DIR_SIZE_LIMIT_BYTES;
1100-
const planDirBytes = measurePlanDirBytes(planDir);
1101-
if (planDirBytes > sizeLimitBytes) {
1102-
throw new PlanTooLargeError(planDirBytes, sizeLimitBytes);
1103-
}
1157+
assertPlanSizeWithinLimit({
1158+
rootDir: planDir,
1159+
rootKind: "plan",
1160+
limitBytes: sizeLimitBytes,
1161+
observedAt: "post-freeze",
1162+
log,
1163+
});
11041164

11051165
return {
11061166
planDir,

0 commit comments

Comments
 (0)