Skip to content

Commit 88db830

Browse files
fix(producer): refuse a too-slow render only when its typical frame says it cannot finish (heygen-com#7479)
* fix(producer): refuse a too-slow render only when its typical frame says it cannot finish * fix(producer): keep the calibration estimate fields self-describing
1 parent 1a758fd commit 88db830

5 files changed

Lines changed: 110 additions & 13 deletions

File tree

‎packages/producer/src/services/render/captureCost.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ export interface CaptureCostEstimate {
3535
multiplier: number;
3636
reasons: string[];
3737
p95Ms?: number;
38+
medianMs?: number;
3839
}
3940

4041
export interface CaptureCalibrationSample {
@@ -100,12 +101,14 @@ function combineCaptureCostEstimates(
100101
multiplier: staticCost.multiplier,
101102
reasons: [...staticCost.reasons, ...measuredCost.reasons],
102103
p95Ms: measuredCost.p95Ms,
104+
medianMs: measuredCost.medianMs,
103105
};
104106
}
105107
return {
106108
multiplier: measuredCost.multiplier,
107109
reasons: [...measuredCost.reasons, ...staticCost.reasons],
108110
p95Ms: measuredCost.p95Ms,
111+
medianMs: measuredCost.medianMs,
109112
};
110113
}
111114

@@ -228,6 +231,12 @@ export function estimateMeasuredCaptureCostMultiplier(
228231
return { multiplier: 1, reasons: [] };
229232
}
230233
const p95Ms = Math.round(p95Sample.captureTimeMs);
234+
const mid = sorted.length / 2;
235+
const medianMs = Math.round(
236+
sorted.length % 2 === 1
237+
? sorted[Math.floor(mid)]!.captureTimeMs
238+
: (sorted[mid - 1]!.captureTimeMs + sorted[mid]!.captureTimeMs) / 2,
239+
);
231240
const multiplier = Math.min(
232241
MAX_MEASURED_CAPTURE_COST_MULTIPLIER,
233242
Math.max(1, Math.round((p95Ms / CAPTURE_CALIBRATION_TARGET_MS) * 100) / 100),
@@ -237,6 +246,7 @@ export function estimateMeasuredCaptureCostMultiplier(
237246
multiplier,
238247
reasons: multiplier > 1 ? [`calibration-p95=${p95Ms}ms`] : [],
239248
p95Ms,
249+
medianMs,
240250
};
241251
}
242252

‎packages/producer/src/services/render/renderEtaGuard.test.ts‎

Lines changed: 64 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,22 @@
11
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
2-
import { RenderTooSlowError, checkRenderEta, startRenderEtaGuard } from "./renderEtaGuard.js";
2+
import {
3+
RenderTooSlowError,
4+
checkCalibratedRenderEta,
5+
checkRenderEta,
6+
startRenderEtaGuard,
7+
} from "./renderEtaGuard.js";
38

49
const RENDER_TOO_SLOW = "RENDER_TOO_SLOW";
510

611
describe("checkRenderEta", () => {
712
const base = { maxRenderMs: 100_000, elapsedMs: 0, framesCompleted: 0, totalFrames: 100 };
813

9-
it("passes at the 1.2x margin and fails just over it", () => {
10-
expect(checkRenderEta("capture", { ...base, msPerFrame: 1_200 })).toBeUndefined();
11-
const error = checkRenderEta("capture", { ...base, msPerFrame: 1_201 });
14+
it("passes at the 2x margin and fails just over it", () => {
15+
expect(checkRenderEta("capture", { ...base, msPerFrame: 2_000 })).toBeUndefined();
16+
const error = checkRenderEta("capture", { ...base, msPerFrame: 2_001 });
1217
expect(error).toBeInstanceOf(RenderTooSlowError);
1318
expect(error?.message).toMatch(/^render too slow:/);
14-
expect(error?.publicMetadata).toMatchObject({ stage: "capture", etaMs: 120_100 });
19+
expect(error?.publicMetadata).toMatchObject({ stage: "capture", etaMs: 200_100 });
1520
// The producer server reads these off the error to build the wire envelope.
1621
expect(error?.code).toBe(RENDER_TOO_SLOW);
1722
expect(error?.owner).toBe("user");
@@ -20,7 +25,7 @@ describe("checkRenderEta", () => {
2025

2126
it("counts time already spent", () => {
2227
expect(
23-
checkRenderEta("capture", { ...base, elapsedMs: 30_000, msPerFrame: 1_000 }),
28+
checkRenderEta("capture", { ...base, elapsedMs: 110_000, msPerFrame: 1_000 }),
2429
).toBeInstanceOf(RenderTooSlowError);
2530
});
2631

@@ -44,7 +49,7 @@ describe("checkRenderEta", () => {
4449
msPerFrame: 784 / 4,
4550
}),
4651
).toBeUndefined();
47-
// 570 frames, 1 worker, p95 15.8 s: 150 min, caught before capture.
52+
// 570 frames, 1 worker, 15.8 s a frame: 160 min, so it runs to the attempt limit as before.
4853
expect(
4954
checkRenderEta("capture_calibration", {
5055
maxRenderMs: budget,
@@ -53,8 +58,60 @@ describe("checkRenderEta", () => {
5358
totalFrames: 570,
5459
msPerFrame: 15_800 / 1,
5560
}),
61+
).toBeUndefined();
62+
// Twice as many frames at that cost: over 5 h, caught before capture.
63+
expect(
64+
checkRenderEta("capture_calibration", {
65+
maxRenderMs: budget,
66+
elapsedMs: 600_000,
67+
framesCompleted: 0,
68+
totalFrames: 1_140,
69+
msPerFrame: 15_800 / 1,
70+
}),
71+
).toBeInstanceOf(RenderTooSlowError);
72+
});
73+
});
74+
75+
describe("checkCalibratedRenderEta", () => {
76+
const budget = { maxRenderMs: 5_100_000, elapsedMs: 15_000 };
77+
78+
it("lets a render through whose slowest calibration frame alone would refuse it", () => {
79+
// One slow calibration frame among fast ones: the slowest predicts about 3 h, the typical one under 1 h.
80+
const input = { maxRenderMs: 5_100_000, elapsedMs: 33_000, totalFrames: 1_800, workerCount: 1 };
81+
expect(checkCalibratedRenderEta({ ...input, estimate: { medianMs: 1_421 } })).toBeUndefined();
82+
expect(
83+
checkRenderEta("capture_calibration", { ...input, framesCompleted: 0, msPerFrame: 5_772 }),
5684
).toBeInstanceOf(RenderTooSlowError);
5785
});
86+
87+
it("lets through a long many-worker render whose typical frame fits the budget", () => {
88+
const input = {
89+
maxRenderMs: 5_100_000,
90+
elapsedMs: 438_000,
91+
totalFrames: 61_092,
92+
workerCount: 6,
93+
};
94+
expect(checkCalibratedRenderEta({ ...input, estimate: { medianMs: 613 } })).toBeUndefined();
95+
});
96+
97+
it("still refuses a render that would take hours on its typical frame", () => {
98+
const tooSlow = checkCalibratedRenderEta({
99+
...budget,
100+
totalFrames: 7_890,
101+
workerCount: 1,
102+
estimate: { medianMs: 1_905 },
103+
});
104+
expect(tooSlow?.publicMetadata).toMatchObject({
105+
stage: "capture_calibration",
106+
totalFrames: 7_890,
107+
});
108+
});
109+
110+
it("skips the check without a calibration estimate", () => {
111+
expect(
112+
checkCalibratedRenderEta({ ...budget, totalFrames: 2_145, workerCount: 1 }),
113+
).toBeUndefined();
114+
});
58115
});
59116

60117
describe("startRenderEtaGuard", () => {

‎packages/producer/src/services/render/renderEtaGuard.ts‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
const RENDER_TOO_SLOW = "RENDER_TOO_SLOW" as const;
22

3-
/** Fail only when the ETA clears the budget by this factor, so borderline renders survive. */
4-
const ETA_MARGIN = 1.2;
3+
/** Fail only at twice the budget: a film's per-frame cost varies, so early ETAs overshoot. */
4+
const ETA_MARGIN = 2;
55
/** The measured rate needs this much capture time and frames; the first frame's warm-up is excluded. */
66
const MIN_SAMPLE_MS = 60_000;
77
const MIN_SAMPLE_FRAMES = 3;
@@ -50,6 +50,23 @@ export function checkRenderEta(
5050
return new RenderTooSlowError(stage, { ...input, maxRenderMs, etaMs });
5151
}
5252

53+
/** The pre-capture check. The typical calibration frame predicts the render; the slowest one over-refuses. */
54+
export function checkCalibratedRenderEta(input: {
55+
maxRenderMs?: number;
56+
elapsedMs: number;
57+
totalFrames: number;
58+
estimate?: { medianMs?: number };
59+
workerCount: number;
60+
}): RenderTooSlowError | undefined {
61+
return checkRenderEta("capture_calibration", {
62+
maxRenderMs: input.maxRenderMs,
63+
elapsedMs: input.elapsedMs,
64+
framesCompleted: 0,
65+
totalFrames: input.totalFrames,
66+
msPerFrame: (input.estimate?.medianMs ?? 0) / input.workerCount,
67+
});
68+
}
69+
5370
interface RenderEtaGuardOptions {
5471
maxRenderMs?: number;
5572
job: { framesRendered?: number };

‎packages/producer/src/services/renderOrchestrator.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1681,6 +1681,18 @@ describe("estimateMeasuredCaptureCostMultiplier", () => {
16811681
expect(estimate.reasons).toEqual(["calibration-p95=2400ms"]);
16821682
});
16831683

1684+
it("records the typical calibration frame next to the slowest", () => {
1685+
const estimate = estimateMeasuredCaptureCostMultiplier(
1686+
[92, 1243, 1183, 983, 717].map((captureTimeMs, i) => ({
1687+
frameIndex: i * 100,
1688+
captureTimeMs,
1689+
})),
1690+
);
1691+
1692+
expect(estimate.p95Ms).toBe(1243);
1693+
expect(estimate.medianMs).toBe(983);
1694+
});
1695+
16841696
it("keeps fast calibration samples at baseline cost", () => {
16851697
const estimate = estimateMeasuredCaptureCostMultiplier([
16861698
{ frameIndex: 0, captureTimeMs: 120 },

‎packages/producer/src/services/renderOrchestrator.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ import {
170170
import { runCaptureSegmentedStage } from "./render/stages/captureSegmentedStage.js";
171171
import {
172172
RenderTooSlowError,
173-
checkRenderEta,
173+
checkCalibratedRenderEta,
174174
startRenderEtaGuard,
175175
} from "./render/renderEtaGuard.js";
176176
import { checkPageNeverReady } from "./render/pageNeverReady.js";
@@ -4257,6 +4257,7 @@ async function executeRenderPipeline(input: {
42574257
forceScreenshot: captureForceScreenshot,
42584258
multiplier: outcome.calibration?.estimate.multiplier ?? null,
42594259
p95Ms: outcome.calibration?.estimate.p95Ms ?? null,
4260+
medianMs: outcome.calibration?.estimate.medianMs ?? null,
42604261
});
42614262
} else {
42624263
observability.checkpoint("capture_calibration", "skipped", {
@@ -4393,12 +4394,12 @@ async function executeRenderPipeline(input: {
43934394

43944395
// No calibration estimate (explicit workers, short or forced-screenshot renders) skips the
43954396
// pre-capture check; the measured-rate guard below still covers capture.
4396-
const calibratedTooSlow = checkRenderEta("capture_calibration", {
4397+
const calibratedTooSlow = checkCalibratedRenderEta({
43974398
maxRenderMs: job.config.maxRenderMs,
43984399
elapsedMs: Date.now() - pipelineStart,
4399-
framesCompleted: 0,
44004400
totalFrames,
4401-
msPerFrame: (captureCalibration?.estimate.p95Ms ?? 0) / workerCount,
4401+
estimate: captureCalibration?.estimate,
4402+
workerCount,
44024403
});
44034404
if (calibratedTooSlow) throw calibratedTooSlow;
44044405
execution.defer(

0 commit comments

Comments
 (0)