Skip to content

Commit f715280

Browse files
fix(render): honour storage-aware verification fallback for every routing kind
The disk-headroom flag was computed by the orchestrator for both worker_inversion and parallel_router routings but only honoured by replanAfterFailure under worker_inversion, so the default-on parallel_router path still reverted to an sdr_disk plan that the 90% gate would reject and assertDiskCaptureHeadroom aborted the render. Give the predicate a single owner: drawElementVerificationFailure decides when headroom can change the fallback (non-default routing, sdr_disk preferred fallback, off-disk memory-exhaustion fallback) and consults the shared inspector only then. replanAfterFailure now honours the resulting flag for any routing kind, and the orchestrator merges the two revert log branches keyed on that flag instead of on plan kind, which also stops OOM retries from being logged as headroom diversions. inspectDiskCaptureHeadroom returns a discriminated union so the assert no longer needs a redundant null guard to narrow freeBytes. Tests: parallel_router headroom diversion and restoration, the failure constructor across default / streaming-only / disk-only routings, and an exact-once statfs count. Co-Authored-By: Miguel Ángel <[email protected]>
1 parent c167565 commit f715280

4 files changed

Lines changed: 178 additions & 48 deletions

File tree

‎packages/producer/src/services/render/capturePlan.test.ts‎

Lines changed: 100 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
import { describe, expect, it } from "vitest";
2-
import { createCapturePlan, replanAfterFailure, type CaptureRouting } from "./capturePlan.js";
2+
import {
3+
createCapturePlan,
4+
drawElementVerificationFailure,
5+
replanAfterFailure,
6+
type CaptureRouting,
7+
} from "./capturePlan.js";
38

49
function streaming(routing?: CaptureRouting) {
510
return createCapturePlan({
@@ -122,6 +127,100 @@ describe("CapturePlan", () => {
122127
expect(Object.isFrozen(next.routing)).toBe(true);
123128
});
124129

130+
it("keeps parallel-router verification recovery streaming when the disk fallback lacks headroom", () => {
131+
const initial = streaming({
132+
kind: "parallel_router",
133+
state: "active",
134+
fallback: { kind: "sdr_disk", workerCount: 5, forceParallelStream: false },
135+
memoryExhaustionFallback: {
136+
kind: "sdr_streaming",
137+
workerCount: 1,
138+
forceParallelStream: false,
139+
},
140+
});
141+
142+
const next = replanAfterFailure(initial, {
143+
kind: "draw_element_verification",
144+
diskFallbackAvailable: false,
145+
});
146+
147+
expect(next).toMatchObject({
148+
kind: "sdr_streaming",
149+
workerCount: 1,
150+
forceScreenshot: true,
151+
routing: { kind: "parallel_router", state: "reverted" },
152+
});
153+
expect(
154+
replanAfterFailure(initial, {
155+
kind: "draw_element_verification",
156+
diskFallbackAvailable: true,
157+
}),
158+
).toMatchObject({ kind: "sdr_disk", workerCount: 5 });
159+
});
160+
161+
describe("drawElementVerificationFailure", () => {
162+
const diskFallback = { kind: "sdr_disk", workerCount: 5, forceParallelStream: false } as const;
163+
const streamFallback = {
164+
kind: "sdr_streaming",
165+
workerCount: 1,
166+
forceParallelStream: false,
167+
} as const;
168+
169+
it("consults disk headroom for every routing whose disk fallback has an off-disk escape", () => {
170+
for (const kind of ["worker_inversion", "parallel_router"] as const) {
171+
const plan = streaming({
172+
kind,
173+
state: "active",
174+
fallback: diskFallback,
175+
memoryExhaustionFallback: streamFallback,
176+
});
177+
for (const available of [false, true]) {
178+
let inspections = 0;
179+
const failure = drawElementVerificationFailure(plan, () => {
180+
inspections += 1;
181+
return available;
182+
});
183+
expect(failure).toEqual({
184+
kind: "draw_element_verification",
185+
diskFallbackAvailable: available,
186+
});
187+
expect(inspections).toBe(1);
188+
}
189+
}
190+
});
191+
192+
it("skips the disk inspection when headroom cannot change the fallback", () => {
193+
const inspect = () => {
194+
throw new Error("disk must not be inspected");
195+
};
196+
const unchanged = { kind: "draw_element_verification", diskFallbackAvailable: undefined };
197+
198+
expect(drawElementVerificationFailure(streaming(), inspect)).toEqual(unchanged);
199+
expect(
200+
drawElementVerificationFailure(
201+
streaming({
202+
kind: "parallel_router",
203+
state: "active",
204+
fallback: streamFallback,
205+
memoryExhaustionFallback: streamFallback,
206+
}),
207+
inspect,
208+
),
209+
).toEqual(unchanged);
210+
expect(
211+
drawElementVerificationFailure(
212+
streaming({
213+
kind: "worker_inversion",
214+
state: "active",
215+
fallback: diskFallback,
216+
memoryExhaustionFallback: { ...diskFallback, workerCount: 1 },
217+
}),
218+
inspect,
219+
),
220+
).toEqual(unchanged);
221+
});
222+
});
223+
125224
it("retries an inversion OOM in single-worker screenshot streaming mode", () => {
126225
const initial = streaming({
127226
kind: "worker_inversion",

‎packages/producer/src/services/render/capturePlan.ts‎

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,40 @@ export interface CreateCapturePlanInput {
6565

6666
export type CapturePlanFailure =
6767
| Readonly<{ kind: "streaming_unavailable" }>
68-
| Readonly<{ kind: "draw_element_verification"; diskFallbackAvailable?: boolean }>
68+
| Readonly<{
69+
kind: "draw_element_verification";
70+
/**
71+
* Set by `drawElementVerificationFailure` only when the routing's
72+
* preferred fallback writes frames to disk and its precomputed
73+
* low-resource target does not. `false` means that disk route lacks
74+
* headroom, so `replanAfterFailure` takes the off-disk target instead.
75+
*/
76+
diskFallbackAvailable?: boolean;
77+
}>
6978
| Readonly<{ kind: "capture_failure"; memoryExhaustion: boolean }>;
7079

80+
/**
81+
* Build the verification failure for `plan`, consulting disk headroom only
82+
* when the answer can change the fallback: a non-default routing (worker
83+
* inversion or parallel router) whose preferred fallback is `sdr_disk` while
84+
* its memory-exhaustion fallback is off-disk. Every other plan leaves the flag
85+
* `undefined` without calling `hasDiskFallbackHeadroom`.
86+
*/
87+
export function drawElementVerificationFailure(
88+
plan: CapturePlan,
89+
hasDiskFallbackHeadroom: () => boolean,
90+
): CapturePlanFailure {
91+
const { routing } = plan;
92+
const needsHeadroom =
93+
routing.kind !== "default" &&
94+
routing.fallback.kind === "sdr_disk" &&
95+
routing.memoryExhaustionFallback.kind === "sdr_streaming";
96+
return {
97+
kind: "draw_element_verification",
98+
diskFallbackAvailable: needsHeadroom ? hasDiskFallbackHeadroom() : undefined,
99+
};
100+
}
101+
71102
function assertWorkerCount(workerCount: number): void {
72103
if (!Number.isInteger(workerCount) || workerCount < 1) {
73104
throw new Error(`CapturePlan workerCount must be a positive integer; got ${workerCount}`);
@@ -151,13 +182,11 @@ export function replanAfterFailure(plan: CapturePlan, failure: CapturePlanFailur
151182

152183
const isMemoryExhaustion = failure.kind === "capture_failure" && failure.memoryExhaustion;
153184
const diskFallbackUnavailable =
154-
failure.kind === "draw_element_verification" &&
155-
failure.diskFallbackAvailable === false &&
156-
plan.routing.kind === "worker_inversion" &&
157-
plan.routing.fallback.kind === "sdr_disk";
185+
failure.kind === "draw_element_verification" && failure.diskFallbackAvailable === false;
158186
// memoryExhaustionFallback is the routing decision's precomputed
159187
// low-resource target. It is also the viable choice when disk, rather than
160-
// RAM, makes the preferred fallback impossible.
188+
// RAM, makes the preferred fallback impossible — for any routing kind
189+
// (see drawElementVerificationFailure for when that flag is populated).
161190
const fallback =
162191
plan.routing.kind === "default"
163192
? {

‎packages/producer/src/services/render/stages/captureStage.ts‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,14 @@ export function estimateDiskCaptureBytes(
147147
return Math.ceil(totalFrames) * outputWidth * outputHeight * 4;
148148
}
149149

150+
/**
151+
* Unknown free space (`freeBytes: null`) counts as available: the gate can
152+
* only reject when it has a measurement.
153+
*/
154+
export type DiskCaptureHeadroom =
155+
| { available: true; estimatedBytes: number; freeBytes: number | null }
156+
| { available: false; estimatedBytes: number; freeBytes: number };
157+
150158
/** Shared 90% disk gate used by both fallback planning and disk execution. */
151159
export function inspectDiskCaptureHeadroom(
152160
framesDir: string,
@@ -160,14 +168,13 @@ export function inspectDiskCaptureHeadroom(
160168
return null;
161169
}
162170
},
163-
): { available: boolean; estimatedBytes: number; freeBytes: number | null } {
171+
): DiskCaptureHeadroom {
164172
const freeBytes = freeDiskBytes(framesDir);
165173
const estimatedBytes = estimateDiskCaptureBytes(totalFrames, captureOptions);
166-
return {
167-
available: freeBytes === null || estimatedBytes <= freeBytes * 0.9,
168-
estimatedBytes,
169-
freeBytes,
170-
};
174+
if (freeBytes === null || estimatedBytes <= freeBytes * 0.9) {
175+
return { available: true, estimatedBytes, freeBytes };
176+
}
177+
return { available: false, estimatedBytes, freeBytes };
171178
}
172179

173180
export function assertDiskCaptureHeadroom(
@@ -182,7 +189,7 @@ export function assertDiskCaptureHeadroom(
182189
captureOptions,
183190
freeDiskBytes,
184191
);
185-
if (headroom.available || headroom.freeBytes === null) return;
192+
if (headroom.available) return;
186193
throw new Error(
187194
`Disk capture may need ~${(headroom.estimatedBytes / 1e6).toFixed(1)} MB of temporary frame storage, ` +
188195
`but only ${(headroom.freeBytes / 1e6).toFixed(1)} MB is free at ${framesDir}. ` +

‎packages/producer/src/services/renderOrchestrator.ts‎

Lines changed: 29 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,10 @@ import { RenderExecutionContext } from "./render/renderExecutionContext.js";
117117
import { ArtifactTransaction } from "./render/artifactTransaction.js";
118118
import {
119119
createCapturePlan,
120+
drawElementVerificationFailure,
120121
replanAfterFailure,
121122
type CapturePlan,
123+
type CapturePlanFailure,
122124
type SdrDiskCapturePlan,
123125
type CaptureRouting,
124126
} from "./render/capturePlan.js";
@@ -3636,21 +3638,15 @@ async function executeRenderPipeline(input: {
36363638
: "capture failed on pinned worker count; retrying with forceScreenshot",
36373639
);
36383640
const failedRouting = capturePlan.routing.kind;
3639-
const verificationDiskFallbackAvailable =
3640-
isVerifyError &&
3641-
capturePlan.routing.kind !== "default" &&
3642-
capturePlan.routing.fallback.kind === "sdr_disk"
3643-
? inspectDiskCaptureHeadroom(framesDir, totalFrames, buildCaptureOptions()).available
3644-
: undefined;
3645-
capturePlan = replanAfterFailure(
3646-
capturePlan,
3647-
isVerifyError
3648-
? {
3649-
kind: "draw_element_verification",
3650-
diskFallbackAvailable: verificationDiskFallbackAvailable,
3651-
}
3652-
: { kind: "capture_failure", memoryExhaustion: isMemoryExhaustion },
3653-
);
3641+
const failure: CapturePlanFailure = isVerifyError
3642+
? drawElementVerificationFailure(
3643+
capturePlan,
3644+
() =>
3645+
inspectDiskCaptureHeadroom(framesDir, totalFrames, buildCaptureOptions())
3646+
.available,
3647+
)
3648+
: { kind: "capture_failure", memoryExhaustion: isMemoryExhaustion };
3649+
capturePlan = replanAfterFailure(capturePlan, failure);
36543650
syncCapturePlan();
36553651
updateCaptureObservability({
36563652
forceScreenshot: capturePlan.forceScreenshot,
@@ -3674,28 +3670,27 @@ async function executeRenderPipeline(input: {
36743670
probeSession = null;
36753671
await closeOrphanedProbeForRetry(orphaned, closeCaptureSession, log, "streaming");
36763672
}
3677-
if (failedRouting === "worker_inversion") {
3678-
// The inversion bet on drawElement and lost. Prefer its
3679-
// pre-inversion parallel screenshot route, but retain the already
3680-
// supported one-worker screenshot stream when that disk route is
3681-
// not feasible.
3682-
// "reverted" (not cleared) so telemetry keeps the lost-inversion
3683-
// cohort distinguishable from renders that never inverted.
3673+
if (failedRouting !== "default") {
3674+
// The inversion's / router's bet on the pinned path lost. Prefer
3675+
// the pre-routing parallel screenshot route; a verification
3676+
// failure alone also retains the already supported low-worker
3677+
// screenshot stream when that disk route lacks storage headroom
3678+
// (OOM takes the same target for RAM; other capture failures
3679+
// revert unconditionally). Routing stays "reverted" (not
3680+
// cleared) so telemetry keeps the lost-bet cohort distinguishable
3681+
// from renders that never routed.
3682+
const route =
3683+
failedRouting === "worker_inversion" ? "worker inversion" : "parallel router";
3684+
const diskFallbackLacksHeadroom =
3685+
failure.kind === "draw_element_verification" &&
3686+
failure.diskFallbackAvailable === false;
36843687
log.info(
3685-
capturePlan.kind === "sdr_streaming"
3686-
? "[Render] Worker-inversion disk fallback lacks headroom; retrying with one-worker screenshot streaming."
3687-
: `[Render] Reverting worker inversion for the retry: ${capturePlan.workerCount} workers, ` +
3688+
diskFallbackLacksHeadroom
3689+
? `[Render] ${route} disk fallback lacks headroom; retrying with ` +
3690+
`${capturePlan.workerCount}-worker screenshot streaming.`
3691+
: `[Render] Reverting ${route} for the retry: ${capturePlan.workerCount} workers, ` +
36883692
`plan=${capturePlan.kind}.`,
36893693
);
3690-
} else if (failedRouting === "parallel_router") {
3691-
// The router's bet on verified parallel streaming lost — re-render
3692-
// on the ordinary (non-DE) parallel path at the pre-router worker
3693-
// count, same "reverted, not cleared" telemetry contract as the
3694-
// inversion above.
3695-
log.info(
3696-
`[Render] Reverting parallel router for the retry: ${capturePlan.workerCount} workers, ` +
3697-
`plan=${capturePlan.kind}.`,
3698-
);
36993694
}
37003695
if (capturePlan.kind === "sdr_streaming") {
37013696
streamingRes = await invokeStreaming();

0 commit comments

Comments
 (0)