Skip to content

Commit 5cea10e

Browse files
fix(producer): fail a render once when its page never became ready (#7154)
* fix(producer): fail a render once when its page never became ready * refactor(producer): keep the page-never-ready error private to its module * test(producer): drop a mock field the page-never-ready test never set
1 parent 11f4326 commit 5cea10e

6 files changed

Lines changed: 244 additions & 2 deletions

File tree

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import { expect, it, vi } from "vitest";
2+
import { resolveConfig } from "@hyperframes/engine";
3+
import { runCaptureCalibration } from "./captureCost.js";
4+
5+
const { state } = vi.hoisted(() => ({ state: { message: "" } }));
6+
vi.mock("@hyperframes/engine", async (importOriginal) => ({
7+
...(await importOriginal<typeof import("@hyperframes/engine")>()),
8+
createCaptureSession: async () => {
9+
throw new Error(state.message);
10+
},
11+
closeCaptureSession: async () => {},
12+
}));
13+
14+
function calibrate() {
15+
return runCaptureCalibration({
16+
cfg: resolveConfig({ forceScreenshot: true }),
17+
fileServer: { url: "http://127.0.0.1:1" } as never,
18+
workDir: "/tmp/unused",
19+
log: { info() {}, warn() {}, error() {}, debug() {} },
20+
job: { config: { fps: 30 } } as never,
21+
totalFrames: 10,
22+
forceScreenshot: true,
23+
probeSession: null,
24+
buildCaptureOptions: () => ({}) as never,
25+
createRenderVideoFrameInjector: () => null,
26+
assertNotAborted: () => {},
27+
});
28+
}
29+
30+
it("reports an evaluate timeout during calibration", async () => {
31+
state.message = "Runtime.evaluate timed out. Increase the 'protocolTimeout' setting";
32+
expect((await calibrate()).calibrationEvaluateTimedOut).toBe(true);
33+
});
34+
35+
it("does not report other calibration failures as an evaluate timeout", async () => {
36+
state.message = "Navigation timeout of 60000 ms exceeded";
37+
expect((await calibrate()).calibrationEvaluateTimedOut).toBe(false);
38+
});

‎packages/producer/src/services/render/captureCost.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,8 @@ export interface CaptureCalibrationOutcome {
343343
probeSession: CaptureSession | null;
344344
/** Buffer of whichever session was active last; the sequencer uses it for the error-path tail. */
345345
lastBrowserConsole: string[];
346+
/** The final calibration failure was a `Runtime.evaluate` timeout: the page never answered. */
347+
calibrationEvaluateTimedOut: boolean;
346348
}
347349

348350
/**
@@ -381,6 +383,7 @@ export async function runCaptureCalibration(input: {
381383
let probeSession = input.probeSession;
382384
let forceScreenshot = input.forceScreenshot;
383385
let lastBrowserConsole: string[] = [];
386+
let calibrationEvaluateTimedOut = false;
384387

385388
const fps = fpsToNumber(job.config.fps);
386389
// Holds whichever calibration session is currently open. The closure
@@ -440,6 +443,7 @@ export async function runCaptureCalibration(input: {
440443
!forceScreenshot && shouldFallbackToScreenshotAfterCalibrationError(error);
441444
if (!shouldFallback) {
442445
calibration = createFailedCaptureCalibrationEstimate("calibration-failed");
446+
calibrationEvaluateTimedOut = isEvaluateTimeout(error);
443447
log.warn("[Render] Auto-worker calibration failed; using conservative worker budget.", {
444448
protocolTimeout: calibrationCfg.protocolTimeout,
445449
error: error instanceof Error ? error.message : String(error),
@@ -481,6 +485,7 @@ export async function runCaptureCalibration(input: {
481485
);
482486
} catch (fallbackError) {
483487
calibration = createFailedCaptureCalibrationEstimate("calibration-screenshot-failed");
488+
calibrationEvaluateTimedOut = isEvaluateTimeout(fallbackError);
484489
log.warn(
485490
"[Render] Screenshot auto-worker calibration failed after BeginFrame fallback; using conservative worker budget.",
486491
{
@@ -497,7 +502,17 @@ export async function runCaptureCalibration(input: {
497502
}
498503
}
499504

500-
return { calibration, forceScreenshot, probeSession, lastBrowserConsole };
505+
return {
506+
calibration,
507+
forceScreenshot,
508+
probeSession,
509+
lastBrowserConsole,
510+
calibrationEvaluateTimedOut,
511+
};
512+
}
513+
514+
export function isEvaluateTimeout(error: unknown): boolean {
515+
return /Runtime\.evaluate timed out/i.test(normalizeErrorMessage(error));
501516
}
502517

503518
/**
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import { isEvaluateTimeout } from "./captureCost.js";
2+
3+
const PAGE_NEVER_READY = "PAGE_NEVER_READY" as const;
4+
5+
class PageNeverReadyError extends Error {
6+
readonly code = PAGE_NEVER_READY;
7+
// The producer server reads these off the error for the wire envelope.
8+
readonly owner = "user" as const;
9+
readonly retryable = false as const;
10+
11+
constructor(cause: unknown) {
12+
super(
13+
"The composition page never became ready: it did not answer browser commands during " +
14+
"calibration or capture, and no frame was captured. Its script may never return.",
15+
{ cause },
16+
);
17+
this.name = "PageNeverReadyError";
18+
}
19+
}
20+
21+
/** Calibration and capture both timed out on `Runtime.evaluate` with no frame captured. */
22+
export function checkPageNeverReady(input: {
23+
calibrationEvaluateTimedOut: boolean;
24+
framesRendered: number;
25+
error: unknown;
26+
}): Error | undefined {
27+
const { calibrationEvaluateTimedOut, framesRendered, error } = input;
28+
if (!calibrationEvaluateTimedOut || framesRendered > 0 || !isEvaluateTimeout(error)) {
29+
return undefined;
30+
}
31+
return new PageNeverReadyError(error);
32+
}
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
import { afterEach, beforeEach, expect, it, vi } from "vitest";
2+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
3+
import { tmpdir } from "node:os";
4+
import { join } from "node:path";
5+
import { resolveConfig } from "@hyperframes/engine";
6+
import { CaptureStageError } from "./render/captureStageError.js";
7+
import { extractSafeRenderErrorMetadata } from "../server.js";
8+
import { createRenderJob, executeRenderJob } from "./renderOrchestrator.js";
9+
10+
// Entry: executeRenderJob on the streaming route; calibration and the streaming stage are scripted.
11+
const { state } = vi.hoisted(() => ({
12+
state: { calibrationEvaluateTimedOut: false, framesBeforeFailure: 0 },
13+
}));
14+
15+
vi.mock("@hyperframes/engine", async (importOriginal) => ({
16+
...(await importOriginal<typeof import("@hyperframes/engine")>()),
17+
assertConfiguredFfmpegBinariesExist: () => {},
18+
resolveBrowserGpuMode: async () => "software",
19+
resolveHeadlessShellPath: () => "/opt/chrome-headless-shell",
20+
createCaptureSession: async () => ({ browserConsoleBuffer: [], options: {} }),
21+
initializeSession: async () => {},
22+
closeCaptureSession: async () => {},
23+
}));
24+
vi.mock("./fileServer.js", async (importOriginal) => ({
25+
...(await importOriginal<typeof import("./fileServer.js")>()),
26+
createFileServer: async () => ({
27+
url: "http://127.0.0.1:1",
28+
port: 1,
29+
close: () => {},
30+
addPreHeadScript: () => {},
31+
}),
32+
}));
33+
vi.mock("./render/stages/compileStage.js", () => ({
34+
runCompileStage: async () => ({
35+
compiled: {
36+
html: "<div></div>",
37+
renderModeHints: { reasons: [] },
38+
hasShaderTransitions: false,
39+
unresolvedCompositions: [],
40+
},
41+
composition: { width: 320, height: 180, duration: 4, videos: [], images: [], audios: [] },
42+
deviceScaleFactor: 1,
43+
outputWidth: 320,
44+
outputHeight: 180,
45+
compileOnlyMs: 0,
46+
forceScreenshot: false,
47+
}),
48+
}));
49+
vi.mock("./render/stages/probeStage.js", () => ({
50+
runProbeStage: async ({ compiled }: { compiled: object }) => ({
51+
compiled,
52+
fileServer: null,
53+
probeSession: null,
54+
lastBrowserConsole: [],
55+
duration: 4,
56+
totalFrames: 120,
57+
browserProbeMs: 0,
58+
}),
59+
}));
60+
vi.mock("./render/stages/extractVideosStage.js", async (importOriginal) => ({
61+
...(await importOriginal<typeof import("./render/stages/extractVideosStage.js")>()),
62+
runExtractVideosStage: async () => ({
63+
extractionResult: null,
64+
frameLookup: null,
65+
videoReadinessSkipIds: new Set<string>(),
66+
videoMetadataHints: new Map(),
67+
nativeHdrVideoIds: new Set<string>(),
68+
videoTransfers: new Map(),
69+
nativeHdrImageIds: new Set<string>(),
70+
imageTransfers: new Map(),
71+
hdrImageSrcPaths: new Map(),
72+
imageColorSpaces: [],
73+
videoExtractMs: 0,
74+
failureToEnforce: undefined,
75+
}),
76+
}));
77+
vi.mock("./render/stages/audioStage.js", () => ({
78+
runAudioStage: async () => ({ audioOutputPath: undefined, hasAudio: false, audioProcessMs: 0 }),
79+
}));
80+
vi.mock("./render/captureCost.js", async (importOriginal) => ({
81+
...(await importOriginal<typeof import("./render/captureCost.js")>()),
82+
resolveRenderWorkerCount: () => 1,
83+
runCaptureCalibration: async (input: { forceScreenshot: boolean; probeSession: unknown }) => ({
84+
calibration: undefined,
85+
forceScreenshot: input.forceScreenshot,
86+
probeSession: input.probeSession,
87+
lastBrowserConsole: [],
88+
calibrationEvaluateTimedOut: state.calibrationEvaluateTimedOut,
89+
}),
90+
}));
91+
vi.mock("./render/stages/captureStreamingStage.js", async (importOriginal) => ({
92+
...(await importOriginal<typeof import("./render/stages/captureStreamingStage.js")>()),
93+
runCaptureStreamingStage: async (input: { job: { framesRendered?: number } }) => {
94+
input.job.framesRendered = state.framesBeforeFailure;
95+
throw new CaptureStageError({ cause: new Error(EVALUATE_TIMEOUT), browserConsole: [] });
96+
},
97+
}));
98+
99+
const EVALUATE_TIMEOUT = "Runtime.evaluate timed out. Increase the 'protocolTimeout' setting";
100+
101+
const dirs: string[] = [];
102+
beforeEach(() => {
103+
state.calibrationEvaluateTimedOut = false;
104+
state.framesBeforeFailure = 0;
105+
vi.stubEnv("HF_SEGMENTED_CAPTURE", "false");
106+
});
107+
afterEach(() => {
108+
vi.unstubAllEnvs();
109+
for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
110+
});
111+
112+
function render(): Promise<void> {
113+
const root = mkdtempSync(join(tmpdir(), "hf-page-never-ready-"));
114+
dirs.push(root);
115+
writeFileSync(join(root, "index.html"), "<div></div>");
116+
const job = createRenderJob({
117+
fps: 30,
118+
quality: "standard",
119+
hdrMode: "force-sdr",
120+
producerConfig: resolveConfig({ forceScreenshot: true, browserGpuMode: "software" }),
121+
logger: { info() {}, warn() {}, error() {}, debug() {} },
122+
});
123+
return executeRenderJob(job, root, join(root, "output.mp4"));
124+
}
125+
126+
it("fails once as non-retryable when calibration and capture both timed out with no frame", async () => {
127+
state.calibrationEvaluateTimedOut = true;
128+
const error = await render().then(
129+
() => undefined,
130+
(caught: unknown) => caught,
131+
);
132+
expect(error).toMatchObject({ code: "PAGE_NEVER_READY", owner: "user", retryable: false });
133+
expect(extractSafeRenderErrorMetadata(error)).toMatchObject({
134+
errorCode: "PAGE_NEVER_READY",
135+
retryable: false,
136+
});
137+
expect((error as Error).message).toContain("never became ready");
138+
});
139+
140+
it("keeps the original error when calibration did not time out", async () => {
141+
await expect(render()).rejects.toThrow(EVALUATE_TIMEOUT);
142+
});
143+
144+
it("keeps the original error once a frame was captured", async () => {
145+
state.calibrationEvaluateTimedOut = true;
146+
state.framesBeforeFailure = 3;
147+
await expect(render()).rejects.toThrow(EVALUATE_TIMEOUT);
148+
});

‎packages/producer/src/services/renderOrchestrator.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,7 @@ import {
172172
checkRenderEta,
173173
startRenderEtaGuard,
174174
} from "./render/renderEtaGuard.js";
175+
import { checkPageNeverReady } from "./render/pageNeverReady.js";
175176
import { resolveSegmentFrames } from "./render/segmentPlan.js";
176177
import { resolveSegmentBrowserRecycle } from "./render/segmentRecycle.js";
177178
import {
@@ -3088,6 +3089,7 @@ async function executeRenderPipeline(input: {
30883089
const onProgress = execution.onProgress;
30893090
const tooSlow = new AbortController();
30903091
let tooSlowError: RenderTooSlowError | undefined;
3092+
let calibrationEvaluateTimedOut = false;
30913093
const executionSignal = job.config.maxRenderMs
30923094
? AbortSignal.any([tooSlow.signal, ...(execution.signal ? [execution.signal] : [])])
30933095
: execution.signal;
@@ -4169,6 +4171,7 @@ async function executeRenderPipeline(input: {
41694171
{ heartbeatMessage: "browser calibrating (frames not started)" },
41704172
);
41714173
captureCalibration = outcome.calibration;
4174+
calibrationEvaluateTimedOut = outcome.calibrationEvaluateTimedOut;
41724175
captureForceScreenshot = outcome.forceScreenshot;
41734176
updateCaptureObservability({ forceScreenshot: captureForceScreenshot });
41744177
probeSession = outcome.probeSession;
@@ -5677,7 +5680,12 @@ async function executeRenderPipeline(input: {
56775680
await eventPublisher.flush();
56785681
} catch (caught) {
56795682
await pendingAudio;
5680-
const error = surfaceScriptFailure(tooSlowError ?? caught, job);
5683+
const pageNeverReady = checkPageNeverReady({
5684+
calibrationEvaluateTimedOut,
5685+
framesRendered: job.framesRendered ?? 0,
5686+
error: caught,
5687+
});
5688+
const error = surfaceScriptFailure(tooSlowError ?? pageNeverReady ?? caught, job);
56815689
if (!tooSlowError && (error instanceof RenderCancelledError || executionSignal?.aborted)) {
56825690
job.error = error instanceof Error ? error.message : "render_cancelled";
56835691
updateJobStatus(job, "cancelled", "Render cancelled", job.progress, onProgress);

‎packages/producer/src/services/safeRenderErrorCode.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ const SAFE_RENDER_ERROR_CODES = new Set<string>([
99
"ENCODER_INTERRUPTED",
1010
"RENDER_TOO_SLOW",
1111
"RENDER_STAGE_STALLED",
12+
"PAGE_NEVER_READY",
1213
]);
1314

1415
/**

0 commit comments

Comments
 (0)