Skip to content

Commit f7ef18d

Browse files
fix(producer): renders leave no captured frames behind when interrupted (#4694)
Ctrl+C during a render left its whole work dir of captured frames (and a staging dir) in the renders folder. Puppeteer's default SIGINT handler kills Chrome and exits the process before the render's own cancel path can remove them. The engine keeps that default unless the host owns Ctrl+C; the CLI's render cancellation scope claims it while a render runs, so the render aborts and cleans up after itself. A render killed outright (SIGKILL, crash) cannot clean up, so each work dir and staging dir now records its owner process (pid, hostname, boot, and on Linux the pid namespace), and the next render into the same folder removes the ones whose owner has exited. Cancelling a render while its browsers start no longer leaves anything running that kept a batch render alive after Ctrl+C: the BeginFrame warm-up loop, the probe browser and file server, or a capture browser whose launch finished after the cancel (now closed before the worker settles, since Puppeteer's exit hook skips every other live browser).
1 parent e0b940a commit f7ef18d

19 files changed

Lines changed: 940 additions & 467 deletions

‎packages/cli/src/utils/renderCancellation.test.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
import { afterEach, describe, expect, it, vi } from "vitest";
33
import { createRenderCancellationScope, stoppedByCancelSignal } from "./renderCancellation.js";
44

5+
const { setHostHandlesSigint } = vi.hoisted(() => ({ setHostHandlesSigint: vi.fn() }));
6+
vi.mock("@hyperframes/engine", () => ({ setHostHandlesSigint }));
7+
58
afterEach(() => {
69
vi.useRealTimers();
710
});
@@ -29,6 +32,17 @@ describe("createRenderCancellationScope ancestor monitoring", () => {
2932
cancellation.dispose();
3033
});
3134

35+
it("owns Ctrl+C for the engine's browsers while the scope is open", () => {
36+
const cancellation = createRenderCancellationScope({
37+
signalTarget: { on: vi.fn(), off: vi.fn() },
38+
pid: 100,
39+
parentPid: () => 1,
40+
});
41+
expect(setHostHandlesSigint).toHaveBeenLastCalledWith(true);
42+
cancellation.dispose();
43+
expect(setHostHandlesSigint).toHaveBeenLastCalledWith(false);
44+
});
45+
3246
it("keeps termination handlers installed until cancellation cleanup finishes", () => {
3347
const handlers = new Map<string, () => void>();
3448
const signalTarget = {

‎packages/cli/src/utils/renderCancellation.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { setHostHandlesSigint } from "@hyperframes/engine";
12
import {
23
processAncestorSnapshot,
34
processIdentity,
@@ -115,6 +116,7 @@ export function createRenderCancellationScope(
115116

116117
const detached = options.detached ?? process.env.HYPERFRAMES_RENDER_DETACHED === "1";
117118
const signalHandlers = installSignalHandlers(detached, signalTarget, abort);
119+
setHostHandlesSigint(true);
118120
const lookupIdentity = options.identity ?? processIdentity;
119121
const isAlive = options.isAlive ?? processIsAlive;
120122
const platform = options.platform ?? process.platform;
@@ -155,6 +157,7 @@ export function createRenderCancellationScope(
155157
disposed = true;
156158
if (watchdogTimer !== undefined) clearInterval(watchdogTimer);
157159
for (const [signal, handler] of signalHandlers) signalTarget.off(signal, handler);
160+
setHostHandlesSigint(false);
158161
},
159162
};
160163
}

‎packages/engine/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ export {
7777
export { chromeMajorCeiling } from "./services/chromeHostCeiling.js";
7878
export {
7979
acquireBrowser,
80+
setHostHandlesSigint,
8081
releaseBrowser,
8182
drainBrowserPool,
8283
closeBrowserPool,

‎packages/engine/src/services/browserManager.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,15 @@ export type {
3131

3232
let _puppeteer: PuppeteerNode | undefined;
3333

34+
let hostHandlesSigint = false;
35+
36+
/** Set while the host cancels renders on Ctrl+C; Puppeteer's own handler would exit before any cleanup ran. */
37+
export function setHostHandlesSigint(owned: boolean): void {
38+
hostHandlesSigint = owned;
39+
}
40+
41+
export const sigintLaunchOptions = () => ({ handleSIGINT: !hostHandlesSigint });
42+
3443
interface WebGlProbeInfo {
3544
hasWebGL: boolean;
3645
vendor: string;
@@ -74,6 +83,7 @@ async function probeHardwareWebGlInfo(
7483
let probeBrowser: Browser | undefined;
7584
try {
7685
probeBrowser = await ppt.launch({
86+
...sigintLaunchOptions(),
7787
headless: true,
7888
args: options.args,
7989
defaultViewport: { width: 64, height: 64 },
@@ -699,6 +709,7 @@ async function launchBrowser(
699709
let browser: Browser | undefined;
700710
try {
701711
browser = await ppt.launch({
712+
...sigintLaunchOptions(),
702713
headless: true,
703714
args: [...fingerprint.args],
704715
defaultViewport: null,
@@ -730,6 +741,7 @@ async function launchBrowser(
730741
);
731742
captureMode = "screenshot";
732743
browser = await ppt.launch({
744+
...sigintLaunchOptions(),
733745
headless: true,
734746
args: stripBeginFrameFlags([...fingerprint.args]),
735747
defaultViewport: null,

‎packages/engine/src/services/frameCapture-warmupTicks.test.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
LOCKED_WARMUP_TICKS,
1515
deriveBeginFrameTimelineTicks,
1616
deriveBeginFrameTimeTicks,
17+
cancelWarmupOnError,
1718
driveWarmupTicks,
1819
prepareBeginFrameTimeline,
1920
warmupFrameTimeTicks,
@@ -54,6 +55,45 @@ async function runWithSimulatedPageLoad(
5455
return state;
5556
}
5657

58+
describe("driveWarmupTicks — cancelled", () => {
59+
it.each([false, true])(
60+
"stops once cancelled even while every tick fails (locked: %s)",
61+
async (locked) => {
62+
const state = makeState();
63+
let calls = 0;
64+
await driveWarmupTicks(
65+
{
66+
intervalMs: 33,
67+
lockWarmupTicks: locked,
68+
// Yields to the event loop so a loop that never stops fails on the test timeout instead of hanging.
69+
sleep: () => new Promise((settle) => setImmediate(settle)),
70+
tick: async () => {
71+
if (++calls === 3) state.cancelled = true;
72+
throw new Error("Target closed");
73+
},
74+
},
75+
state,
76+
);
77+
expect(calls).toBe(3);
78+
},
79+
);
80+
});
81+
82+
describe("cancelWarmupOnError", () => {
83+
it("stops the warm-up loop when setup fails and leaves it alone when setup succeeds", async () => {
84+
const failed = makeState();
85+
const setupError = new Error("render_cancelled");
86+
await expect(cancelWarmupOnError(failed, () => Promise.reject(setupError))).rejects.toBe(
87+
setupError,
88+
);
89+
expect(failed.cancelled).toBe(true);
90+
91+
const succeeded = makeState();
92+
await expect(cancelWarmupOnError(succeeded, async () => "ready")).resolves.toBe("ready");
93+
expect(succeeded.cancelled).toBeUndefined();
94+
});
95+
});
96+
5797
describe("driveWarmupTicks — unlocked", () => {
5898
it("stops when state.running flips false", async () => {
5999
const state = await runWithSimulatedPageLoad(false, 10);

‎packages/engine/src/services/frameCapture.ts‎

Lines changed: 96 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -679,6 +679,8 @@ export const LOCKED_WARMUP_TICKS = 60;
679679
export interface WarmupTickState {
680680
running: boolean;
681681
ticks: number;
682+
/** Set when session init fails: stops the loop in either mode, since a closed page fails every tick. */
683+
cancelled?: boolean;
682684
}
683685

684686
export interface WarmupTickOptions {
@@ -794,12 +796,25 @@ export function prepareBeginFrameTimeline(
794796
};
795797
}
796798

799+
/** Runs BeginFrame session setup; a failure stops the warm-up loop, which would otherwise tick forever. */
800+
export async function cancelWarmupOnError<T>(
801+
state: WarmupTickState,
802+
setup: () => Promise<T>,
803+
): Promise<T> {
804+
try {
805+
return await setup();
806+
} catch (error) {
807+
state.cancelled = true;
808+
throw error;
809+
}
810+
}
811+
797812
export async function driveWarmupTicks(
798813
options: WarmupTickOptions,
799814
state: WarmupTickState,
800815
): Promise<void> {
801816
const sleep = options.sleep ?? realSleep;
802-
while (true) {
817+
while (!state.cancelled) {
803818
if (options.lockWarmupTicks) {
804819
// Locked mode exits on the iteration count, ignoring `state.running` —
805820
// the caller flips `running=false` after page-readiness but we keep
@@ -2462,95 +2477,93 @@ export async function initializeSession(session: CaptureSession): Promise<void>
24622477
warmupLoopPromise.catch(() => {});
24632478
logInitPhase("warmup loop started");
24642479

2465-
await gotoEntryPage();
2466-
logInitPhase("page.goto complete");
2480+
await cancelWarmupOnError(warmupState, async () => {
2481+
await gotoEntryPage();
2482+
logInitPhase("page.goto complete");
24672483

2468-
// Flush the GSAP proxy queue synchronously. In BeginFrame mode the rAF-based
2469-
// batch drain runs on the warmup loop's 33ms ticks — for tween-heavy
2470-
// compositions this is the dominant init cost. Flushing synchronously
2471-
// eliminates the wait entirely.
2472-
await page.evaluate(`window.__hfFlushSync?.()`);
2473-
logInitPhase("GSAP proxy flush complete");
2484+
// Flush the GSAP proxy queue synchronously. In BeginFrame mode the rAF-based
2485+
// batch drain runs on the warmup loop's 33ms ticks — for tween-heavy
2486+
// compositions this is the dominant init cost. Flushing synchronously
2487+
// eliminates the wait entirely.
2488+
await page.evaluate(`window.__hfFlushSync?.()`);
2489+
logInitPhase("GSAP proxy flush complete");
24742490

2475-
// Poll for window.__hf readiness using manual evaluate loop (waitForFunction
2476-
// uses rAF polling internally, which won't fire in beginFrame mode).
2477-
const pageReadyTimeout = session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout;
2478-
try {
2491+
// Poll for window.__hf readiness using manual evaluate loop (waitForFunction
2492+
// uses rAF polling internally, which won't fire in beginFrame mode).
2493+
const pageReadyTimeout =
2494+
session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout;
24792495
await pollHfReady(page, pageReadyTimeout);
24802496
logInitPhase("pollHfReady complete");
2481-
} catch (err) {
2482-
warmupState.running = false;
2483-
throw err;
2484-
}
24852497

2486-
session.subTimelineWaitOutcome = await pollSubCompositionTimelines(
2487-
page,
2488-
pageReadyTimeout,
2489-
undefined,
2490-
() => session.scriptLoadFailures,
2491-
undefined,
2492-
(ids) => {
2493-
session.pendingTimelineIds = [...ids];
2494-
},
2495-
);
2496-
logInitPhase(`pollSubCompositionTimelines complete (${session.subTimelineWaitOutcome})`);
2497-
recordSubTimelineWarning(session, pageReadyTimeout);
2498-
2499-
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
2500-
logInitPhase("applyVideoMetadataHints complete");
2501-
2502-
// Run independent readiness checks in parallel — videos, images, fonts,
2503-
// and Tailwind don't depend on each other's completion.
2504-
const bfSkipVideoIds = session.options.skipReadinessVideoIds ?? [];
2505-
const [bfVideosReady] = await Promise.all([
2506-
pollVideosReady(page, bfSkipVideoIds, pageReadyTimeout),
2507-
pollImagesReady(page, pageReadyTimeout).then(async (ready) => {
2508-
if (!ready) {
2509-
const failedImages = await page.evaluate(() => {
2510-
return Array.from(document.querySelectorAll("img"))
2511-
.filter((img) => {
2512-
const ie = img as HTMLImageElement;
2513-
const src = ie.getAttribute("src") || "";
2514-
if (!src || src.startsWith("data:")) return false;
2515-
return !(ie.complete && ie.naturalWidth > 0);
2516-
})
2517-
.map((img) => (img as HTMLImageElement).src || img.getAttribute("src") || "(no src)")
2518-
.join(", ");
2519-
});
2520-
console.warn(
2521-
`[FrameCapture] Some image elements did not load within ${pageReadyTimeout}ms: ${failedImages}. ` +
2522-
`Continuing render — affected images may appear blank/missing in early frames.`,
2523-
);
2524-
}
2525-
await decodeAllImages(page);
2526-
return ready;
2527-
}),
2528-
page.evaluate(`document.fonts?.ready`),
2529-
waitForOptionalTailwindReady(page, pageReadyTimeout),
2530-
]);
2531-
logInitPhase("media + fonts + tailwind ready");
2532-
2533-
if (!bfVideosReady) {
2534-
const failedVideos = await page.evaluate((skipIdList: readonly string[]) => {
2535-
const skip = new Set(skipIdList);
2536-
return Array.from(document.querySelectorAll("video"))
2537-
.filter((v) => !skip.has(v.id))
2538-
.filter((v) => (v as HTMLVideoElement).readyState < 2 && !(v as HTMLVideoElement).error)
2539-
.map((v) => (v as HTMLVideoElement).src || v.getAttribute("src") || "(no src)")
2540-
.join(", ");
2541-
}, bfSkipVideoIds);
2542-
console.warn(
2543-
`[FrameCapture] Some video elements did not decode within ${pageReadyTimeout}ms: ${failedVideos}. ` +
2544-
`Continuing render — affected videos will appear as blank/black frames.`,
2498+
session.subTimelineWaitOutcome = await pollSubCompositionTimelines(
2499+
page,
2500+
pageReadyTimeout,
2501+
undefined,
2502+
() => session.scriptLoadFailures,
2503+
undefined,
2504+
(ids) => {
2505+
session.pendingTimelineIds = [...ids];
2506+
},
25452507
);
2546-
}
2547-
recordCaptureWarnings(
2548-
session,
2549-
await collectMediaReadinessWarnings(page, bfSkipVideoIds, pageReadyTimeout),
2550-
);
2551-
await recordLiveMapWarning(session, page);
2508+
logInitPhase(`pollSubCompositionTimelines complete (${session.subTimelineWaitOutcome})`);
2509+
recordSubTimelineWarning(session, pageReadyTimeout);
25522510

2553-
await recordSessionInitTelemetry(session, initStart);
2511+
await applyVideoMetadataHints(page, session.options.videoMetadataHints);
2512+
logInitPhase("applyVideoMetadataHints complete");
2513+
2514+
// Run independent readiness checks in parallel — videos, images, fonts,
2515+
// and Tailwind don't depend on each other's completion.
2516+
const bfSkipVideoIds = session.options.skipReadinessVideoIds ?? [];
2517+
const [bfVideosReady] = await Promise.all([
2518+
pollVideosReady(page, bfSkipVideoIds, pageReadyTimeout),
2519+
pollImagesReady(page, pageReadyTimeout).then(async (ready) => {
2520+
if (!ready) {
2521+
const failedImages = await page.evaluate(() => {
2522+
return Array.from(document.querySelectorAll("img"))
2523+
.filter((img) => {
2524+
const ie = img as HTMLImageElement;
2525+
const src = ie.getAttribute("src") || "";
2526+
if (!src || src.startsWith("data:")) return false;
2527+
return !(ie.complete && ie.naturalWidth > 0);
2528+
})
2529+
.map((img) => (img as HTMLImageElement).src || img.getAttribute("src") || "(no src)")
2530+
.join(", ");
2531+
});
2532+
console.warn(
2533+
`[FrameCapture] Some image elements did not load within ${pageReadyTimeout}ms: ${failedImages}. ` +
2534+
`Continuing render — affected images may appear blank/missing in early frames.`,
2535+
);
2536+
}
2537+
await decodeAllImages(page);
2538+
return ready;
2539+
}),
2540+
page.evaluate(`document.fonts?.ready`),
2541+
waitForOptionalTailwindReady(page, pageReadyTimeout),
2542+
]);
2543+
logInitPhase("media + fonts + tailwind ready");
2544+
2545+
if (!bfVideosReady) {
2546+
const failedVideos = await page.evaluate((skipIdList: readonly string[]) => {
2547+
const skip = new Set(skipIdList);
2548+
return Array.from(document.querySelectorAll("video"))
2549+
.filter((v) => !skip.has(v.id))
2550+
.filter((v) => (v as HTMLVideoElement).readyState < 2 && !(v as HTMLVideoElement).error)
2551+
.map((v) => (v as HTMLVideoElement).src || v.getAttribute("src") || "(no src)")
2552+
.join(", ");
2553+
}, bfSkipVideoIds);
2554+
console.warn(
2555+
`[FrameCapture] Some video elements did not decode within ${pageReadyTimeout}ms: ${failedVideos}. ` +
2556+
`Continuing render — affected videos will appear as blank/black frames.`,
2557+
);
2558+
}
2559+
recordCaptureWarnings(
2560+
session,
2561+
await collectMediaReadinessWarnings(page, bfSkipVideoIds, pageReadyTimeout),
2562+
);
2563+
await recordLiveMapWarning(session, page);
2564+
2565+
await recordSessionInitTelemetry(session, initStart);
2566+
});
25542567

25552568
// Stop warmup, then drain the loop in BOTH modes before any further
25562569
// BeginFrame on this session (drawElement init, the render-frame commit tick

‎packages/engine/src/services/hdrCapture.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import type { Page, Browser, PuppeteerNode } from "puppeteer-core";
2323
import { existsSync, readdirSync } from "fs";
2424
import { join } from "path";
2525
import { homedir } from "os";
26+
import { sigintLaunchOptions } from "./browserManager.js";
2627

2728
// ── PQ (SMPTE 2084) OETF ─────────────────────────────────────────────────────
2829

@@ -282,6 +283,7 @@ export async function launchHdrBrowser(
282283
}
283284

284285
const browser = await ppt.launch({
286+
...sigintLaunchOptions(),
285287
headless: false,
286288
executablePath: chromePath,
287289
args: buildHdrChromeArgs(width, height),

0 commit comments

Comments
 (0)