Skip to content

Commit 5f2066e

Browse files
feat(producer): fail a render that cannot finish within its maxRenderMs budget
1 parent 9b80c2d commit 5f2066e

8 files changed

Lines changed: 472 additions & 4 deletions

File tree

‎packages/producer/src/renderRequest.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ export interface RenderRequestOptions {
4545
gifLoop?: number;
4646
hlsSegmentSeconds?: number;
4747
workers?: number;
48+
maxRenderMs?: number;
4849
useGpu?: boolean;
4950
debug?: boolean;
5051
/** Segmented capture: reuse segments recorded in a matching manifest. */
@@ -172,6 +173,7 @@ function assertRequestOptionScalars(options: Record<string, unknown>): void {
172173
assertOptionalInteger(options, "gifLoop");
173174
assertOptionalInteger(options, "hlsSegmentSeconds", 1);
174175
assertOptionalInteger(options, "workers", 1);
176+
assertOptionalInteger(options, "maxRenderMs");
175177
assertOptionalInteger(options, "crf");
176178
for (const field of ["useGpu", "debug", "outputResolutionAspectAgnostic"] as const) {
177179
assertOptionalBoolean(options, field);
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import { Hono } from "hono";
2+
import { describe, expect, it, vi } from "vitest";
3+
4+
const createdConfigs = vi.hoisted(() => [] as Record<string, unknown>[]);
5+
vi.mock("./services/renderOrchestrator.js", async () => {
6+
const { RenderTooSlowError } = await import("./services/render/renderEtaGuard.js");
7+
class RenderCancelledError extends Error {}
8+
return {
9+
RenderCancelledError,
10+
createRenderJob: (config: Record<string, unknown>) => {
11+
createdConfigs.push(config);
12+
return {
13+
config,
14+
progress: 0,
15+
currentStage: "capture",
16+
framesRendered: 0,
17+
totalFrames: 0,
18+
warnings: [],
19+
};
20+
},
21+
executeRenderJob: async () => {
22+
throw new RenderTooSlowError("capture", {
23+
maxRenderMs: 120_000,
24+
elapsedMs: 70_000,
25+
framesCompleted: 127,
26+
totalFrames: 21_540,
27+
msPerFrame: 472,
28+
etaMs: 10_000_000,
29+
});
30+
},
31+
};
32+
});
33+
34+
import { createRenderHandlers } from "./server.js";
35+
36+
function post(path: string, extra: Record<string, unknown> = {}): Promise<Response> {
37+
const app = new Hono();
38+
const handlers = createRenderHandlers({
39+
getRequestId: () => "too-slow-test",
40+
maxConcurrentRenders: 1,
41+
});
42+
app.post("/v1/render", handlers.render);
43+
app.post("/v1/render-stream", handlers.renderStream);
44+
return app.request(path, {
45+
method: "POST",
46+
headers: { "content-type": "application/json" },
47+
body: JSON.stringify({ html: "<html><body></body></html>", maxRenderMs: 120_000, ...extra }),
48+
});
49+
}
50+
51+
describe("server render too slow", () => {
52+
it("forwards maxRenderMs into the render config", async () => {
53+
await post("/v1/render");
54+
expect(createdConfigs.at(-1)?.maxRenderMs).toBe(120_000);
55+
});
56+
57+
it("emits RENDER_TOO_SLOW as non-retryable in blocking JSON", async () => {
58+
const response = await post("/v1/render");
59+
const body = await response.json();
60+
expect(response.status).toBe(500);
61+
expect(body).toMatchObject({
62+
success: false,
63+
errorCode: "RENDER_TOO_SLOW",
64+
retryable: false,
65+
errorOwner: "user",
66+
errorMetadata: {
67+
stage: "capture",
68+
framesCompleted: 127,
69+
totalFrames: 21_540,
70+
msPerFrame: 472,
71+
maxRenderMs: 120_000,
72+
},
73+
});
74+
expect(body.error).toMatch(/^render too slow:/);
75+
});
76+
77+
it("emits the same event over SSE", async () => {
78+
const body = await (await post("/v1/render-stream")).text();
79+
expect(body).toContain('"errorCode":"RENDER_TOO_SLOW"');
80+
expect(body).toContain('"retryable":false');
81+
expect(body).toContain('"error":"render too slow:');
82+
});
83+
84+
it.each([-1, 1.5, "90000", null])("rejects maxRenderMs=%p with a 400", async (value) => {
85+
const response = await post("/v1/render", { maxRenderMs: value });
86+
expect(response.status).toBe(400);
87+
expect((await response.json()).error).toContain("maxRenderMs");
88+
});
89+
});

‎packages/producer/src/server.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ interface RenderInput {
8888
pageMotionBlur?: RenderConfig["pageMotionBlur"];
8989
outputDynamicRange?: "auto" | "hdr" | "sdr";
9090
workers?: number;
91+
maxRenderMs?: number;
9192
crf?: number;
9293
videoBitrate?: string;
9394
useGpu: boolean;
@@ -130,6 +131,7 @@ const SAFE_RENDER_ERROR_CODES = new Set<string>([
130131
"VIDEO_SOURCE_UNRENDERABLE",
131132
"VIDEO_EXTRACTION_FAILED",
132133
"ENCODER_INTERRUPTED",
134+
"RENDER_TOO_SLOW",
133135
]);
134136

135137
/**
@@ -224,6 +226,7 @@ export function parseRenderOptions(body: Record<string, unknown>): Omit<RenderIn
224226
const fps = parseServerFps(body.fps);
225227
const quality = parseServerQuality(body.quality);
226228
const workers = typeof body.workers === "number" ? body.workers : undefined;
229+
const maxRenderMs = typeof body.maxRenderMs === "number" ? body.maxRenderMs : undefined;
227230
const useGpu = body.gpu === true;
228231
const debug = body.debug === true;
229232
// Preserve the pre-structured-warning HTTP contract for callers that do
@@ -248,6 +251,7 @@ export function parseRenderOptions(body: Record<string, unknown>): Omit<RenderIn
248251
fps,
249252
quality,
250253
workers,
254+
maxRenderMs,
251255
crf,
252256
videoBitrate,
253257
useGpu,
@@ -309,6 +313,7 @@ function buildRenderJobConfig(input: RenderInput, outputPath: string, log: Produ
309313
quality: input.quality,
310314
format: input.format ?? "mp4",
311315
workers: input.workers,
316+
maxRenderMs: input.maxRenderMs,
312317
crf: input.crf,
313318
videoBitrate: input.videoBitrate,
314319
useGpu: input.useGpu,
@@ -352,6 +357,14 @@ function resolvePreparedRenderOutput(
352357
function validateRenderOverrides(body: Record<string, unknown>): string | undefined {
353358
const encoder = parseEncoderOptions(body, parseServerFormat(body.format) ?? "mp4");
354359
if (!encoder.ok) return encoder.error;
360+
if (
361+
body.maxRenderMs !== undefined &&
362+
(typeof body.maxRenderMs !== "number" ||
363+
!Number.isInteger(body.maxRenderMs) ||
364+
body.maxRenderMs < 0)
365+
) {
366+
return "maxRenderMs must be a non-negative integer number of milliseconds";
367+
}
355368
if (body.variables !== undefined && !isPlainObject(body.variables)) {
356369
return 'variables must be a JSON object keyed by variable id (e.g. {"title":"Hello"})';
357370
}
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
2+
import { RenderTooSlowError, checkRenderEta, startRenderEtaGuard } from "./renderEtaGuard.js";
3+
4+
const RENDER_TOO_SLOW = "RENDER_TOO_SLOW";
5+
6+
describe("checkRenderEta", () => {
7+
const base = { maxRenderMs: 100_000, elapsedMs: 0, framesCompleted: 0, totalFrames: 100 };
8+
9+
it("passes at the 1.2x margin and fails just over it", () => {
10+
expect(checkRenderEta("capture", { ...base, msPerFrame: 1_200 })).toBeUndefined();
11+
const error = checkRenderEta("capture", { ...base, msPerFrame: 1_201 });
12+
expect(error).toBeInstanceOf(RenderTooSlowError);
13+
expect(error?.message).toMatch(/^render too slow:/);
14+
expect(error?.publicMetadata).toMatchObject({ stage: "capture", etaMs: 120_100 });
15+
// The producer server reads these off the error to build the wire envelope.
16+
expect(error?.code).toBe(RENDER_TOO_SLOW);
17+
expect(error?.owner).toBe("user");
18+
expect(error?.retryable).toBe(false);
19+
});
20+
21+
it("counts time already spent", () => {
22+
expect(
23+
checkRenderEta("capture", { ...base, elapsedMs: 30_000, msPerFrame: 1_000 }),
24+
).toBeInstanceOf(RenderTooSlowError);
25+
});
26+
27+
it("never fires without a budget or without an estimate", () => {
28+
expect(
29+
checkRenderEta("capture", { ...base, maxRenderMs: undefined, msPerFrame: 1e9 }),
30+
).toBeUndefined();
31+
expect(checkRenderEta("capture", { ...base, maxRenderMs: 0, msPerFrame: 1e9 })).toBeUndefined();
32+
expect(checkRenderEta("capture", { ...base, msPerFrame: 0 })).toBeUndefined();
33+
});
34+
35+
it("matches the two prod renders on the 90 minute budget", () => {
36+
const budget = 90 * 60_000;
37+
// 21540 frames, 4 workers, p95 784 ms: the calibration estimate (70 min) is under budget.
38+
expect(
39+
checkRenderEta("capture_calibration", {
40+
maxRenderMs: budget,
41+
elapsedMs: 0,
42+
framesCompleted: 0,
43+
totalFrames: 21_540,
44+
msPerFrame: 784 / 4,
45+
}),
46+
).toBeUndefined();
47+
// 570 frames, 1 worker, p95 15.8 s: 150 min, caught before capture.
48+
expect(
49+
checkRenderEta("capture_calibration", {
50+
maxRenderMs: budget,
51+
elapsedMs: 600_000,
52+
framesCompleted: 0,
53+
totalFrames: 570,
54+
msPerFrame: 15_800 / 1,
55+
}),
56+
).toBeInstanceOf(RenderTooSlowError);
57+
});
58+
});
59+
60+
describe("startRenderEtaGuard", () => {
61+
beforeEach(() => vi.useFakeTimers());
62+
afterEach(() => vi.useRealTimers());
63+
64+
/** Advances 5 s per step, adding `framesPerStep` frames, until tripped or `steps` run out. */
65+
function run(opts: {
66+
maxRenderMs?: number;
67+
framesPerStep: number;
68+
steps: number;
69+
total: number;
70+
}) {
71+
const job = { framesRendered: 0 };
72+
let tripped: RenderTooSlowError | undefined;
73+
const stop = startRenderEtaGuard({
74+
maxRenderMs: opts.maxRenderMs,
75+
job,
76+
totalFrames: opts.total,
77+
pipelineStartMs: Date.now(),
78+
onTooSlow: (error) => (tripped = error),
79+
});
80+
let at = 0;
81+
for (let i = 0; i < opts.steps && !tripped; i++) {
82+
job.framesRendered += opts.framesPerStep;
83+
vi.advanceTimersByTime(5_000);
84+
at += 5_000;
85+
}
86+
stop();
87+
return { tripped, at, job };
88+
}
89+
90+
it("fails a crawling render within about two minutes of capture", () => {
91+
const { tripped, at } = run({
92+
maxRenderMs: 120_000,
93+
framesPerStep: 1,
94+
steps: 100,
95+
total: 1_000,
96+
});
97+
expect(tripped?.message).toMatch(/^render too slow:/);
98+
expect(at).toBeLessThanOrEqual(120_000);
99+
expect(tripped?.publicMetadata).toMatchObject({
100+
stage: "capture",
101+
totalFrames: 1_000,
102+
maxRenderMs: 120_000,
103+
});
104+
});
105+
106+
it("leaves a render that will finish inside the budget alone", () => {
107+
expect(
108+
run({ maxRenderMs: 10_000_000, framesPerStep: 100, steps: 100, total: 100_000 }).tripped,
109+
).toBeUndefined();
110+
});
111+
112+
it("is never tripped by the encode and assemble time after capture completes", () => {
113+
const job = { framesRendered: 0 };
114+
const onTooSlow = vi.fn();
115+
const stop = startRenderEtaGuard({
116+
maxRenderMs: 120_000,
117+
job,
118+
totalFrames: 100,
119+
pipelineStartMs: Date.now(),
120+
onTooSlow,
121+
});
122+
// Capture finishes inside the budget...
123+
for (let i = 0; i < 10; i++) {
124+
job.framesRendered += 10;
125+
vi.advanceTimersByTime(5_000);
126+
}
127+
// ...then a mux that runs for an hour with the counter parked at the total.
128+
vi.advanceTimersByTime(60 * 60_000);
129+
stop();
130+
expect(onTooSlow).not.toHaveBeenCalled();
131+
expect(vi.getTimerCount()).toBe(0);
132+
});
133+
134+
it("is a no-op without a budget", () => {
135+
expect(
136+
run({ maxRenderMs: undefined, framesPerStep: 1, steps: 200, total: 1_000 }).tripped,
137+
).toBeUndefined();
138+
expect(
139+
run({ maxRenderMs: 0, framesPerStep: 1, steps: 200, total: 1_000 }).tripped,
140+
).toBeUndefined();
141+
});
142+
143+
it("does not decide on zero frames (the stall watchdog owns that)", () => {
144+
expect(
145+
run({ maxRenderMs: 1_000, framesPerStep: 0, steps: 200, total: 1_000 }).tripped,
146+
).toBeUndefined();
147+
});
148+
});

0 commit comments

Comments
 (0)