Skip to content

Commit 6fab219

Browse files
fix(producer): segmented capture gives up on a session stuck opening or initializing (heygen-com#7329)
* fix(producer): segmented capture gives up on a session stuck opening or initializing * fix(producer): sequential disk capture bounds the opt-in drawElement setup the same way
1 parent 18fc75d commit 6fab219

5 files changed

Lines changed: 193 additions & 19 deletions

File tree

‎packages/producer/src/services/render/stages/captureSegmentedStage.test.ts‎

Lines changed: 82 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
// fallow-ignore-file code-duplication
2-
import { afterAll, describe, expect, it, mock } from "bun:test";
2+
import { afterAll, describe, expect, it, mock, vi } from "bun:test";
33
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
44
import { tmpdir } from "node:os";
55
import { join } from "node:path";
@@ -26,6 +26,9 @@ let sessionWorkerEncodeEnabled = false;
2626
let captureSessionMode: "drawelement" | "screenshot" = "drawelement";
2727
let failPrepareCaptureSessionForReuse = false;
2828
let initializeSessionErrorMessage = "initialize failed";
29+
let hangDeferredDrawElementInit = false;
30+
/** While set, opening a session waits for this promise instead of returning a fresh one. */
31+
let pendingOpen: Promise<unknown> | null = null;
2932
const browserConsoleBuffer = ["[FrameCapture:ERROR] page.goto failed"];
3033
const closeCaptureSession = mock(async () => {});
3134
const createdSessionConfigs: Array<{ enableBrowserPool?: boolean }> = [];
@@ -62,10 +65,12 @@ mock.module("@hyperframes/engine", () => ({
6265
return { buffer: Buffer.from("frame"), captureTimeMs: 1 };
6366
},
6467
closeCaptureSession,
65-
completeDeferredDrawElementInit: async () => {},
68+
completeDeferredDrawElementInit: () =>
69+
hangDeferredDrawElementInit ? new Promise<void>(() => {}) : Promise.resolve(),
6670
createCaptureSession: async (...args: unknown[]) => {
6771
createdSessionConfigs.push(args[4] as { enableBrowserPool?: boolean });
6872
createdSessionHooks.push(args[3]);
73+
if (pendingOpen) return pendingOpen;
6974
return {
7075
isInitialized: false,
7176
browserConsoleBuffer,
@@ -451,6 +456,81 @@ describe("runCaptureSegmentedStage", () => {
451456
expect(result).toEqual({ success: false });
452457
});
453458

459+
/** Runs the stage on a fake clock, a second at a time, until it settles or 30 minutes pass. */
460+
const settleOnFakeClock = async (run: Promise<unknown>) => {
461+
const state = { done: false, error: undefined as unknown, elapsedMs: 0 };
462+
run.then(
463+
() => (state.done = true),
464+
(error: unknown) => Object.assign(state, { done: true, error }),
465+
);
466+
const flush = async () => {
467+
for (let i = 0; i < 200; i++) await Promise.resolve();
468+
};
469+
await flush();
470+
while (!state.done && state.elapsedMs < 30 * 60_000) {
471+
vi.advanceTimersByTime(1_000);
472+
state.elapsedMs += 1_000;
473+
await flush();
474+
}
475+
return state;
476+
};
477+
478+
it("gives up on a deferred drawElement init that never finishes, within the session-init budget", async () => {
479+
hangDeferredDrawElementInit = true;
480+
closeCaptureSession.mockClear();
481+
const prev = process.env.HF_DE_STALL_MS;
482+
delete process.env.HF_DE_STALL_MS;
483+
vi.useFakeTimers();
484+
try {
485+
const input = fakeStageInput({ totalFrames: 3 });
486+
const state = await settleOnFakeClock(
487+
runCaptureSegmentedStage({ ...input, segmentFrames: 3 }),
488+
);
489+
expect(state.done).toBe(true);
490+
expect(state.elapsedMs).toBeLessThanOrEqual(305_000);
491+
expect((state.error as Error).message).toContain(
492+
"no frame progress for 300000ms (stuck at frame 0/3)",
493+
);
494+
expect(closeCaptureSession.mock.calls.map((c) => c[0])).toContain(input.probeSession);
495+
} finally {
496+
vi.useRealTimers();
497+
hangDeferredDrawElementInit = false;
498+
if (prev !== undefined) process.env.HF_DE_STALL_MS = prev;
499+
}
500+
});
501+
502+
it("gives up on a session that never finishes opening, and closes it when it opens late", async () => {
503+
let openLate: (session: unknown) => void = () => {};
504+
pendingOpen = new Promise((resolve) => (openLate = resolve));
505+
closeCaptureSession.mockClear();
506+
const prev = process.env.HF_DE_STALL_MS;
507+
delete process.env.HF_DE_STALL_MS;
508+
vi.useFakeTimers();
509+
try {
510+
const state = await settleOnFakeClock(
511+
runCaptureSegmentedStage({
512+
...fakeStageInput({ totalFrames: 3 }),
513+
probeSession: undefined,
514+
segmentFrames: 3,
515+
}),
516+
);
517+
expect(state.done).toBe(true);
518+
expect(state.elapsedMs).toBeLessThanOrEqual(305_000);
519+
expect((state.error as Error).message).toContain(
520+
"no frame progress for 300000ms (stuck at frame 0/3 while opening the capture session)",
521+
);
522+
523+
const late = { browserConsoleBuffer: [] };
524+
openLate(late);
525+
for (let i = 0; i < 20; i++) await Promise.resolve();
526+
expect(closeCaptureSession).toHaveBeenCalledWith(late);
527+
} finally {
528+
vi.useRealTimers();
529+
pendingOpen = null;
530+
if (prev !== undefined) process.env.HF_DE_STALL_MS = prev;
531+
}
532+
});
533+
454534
it("returns success:false when the first encoder cannot spawn, and has closed the probe", async () => {
455535
const spawnEncoder = mock(async () => {
456536
throw new Error("ffmpeg missing");

‎packages/producer/src/services/render/stages/captureSegmentedStage.ts‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ import type { SegmentVideoFrames, SegmentWorkerFrames } from "../segmentVideoFra
4747
import { createSegmentQueue } from "../segmentQueue.js";
4848
import { isTargetLossError } from "../segmentRecycle.js";
4949
import {
50+
OPENING_CAPTURE_SESSION,
51+
createSessionInitBudget,
5052
raceAgainstStall,
5153
resolveCaptureStallTimeoutMs,
5254
verifyDrawElementSample,
@@ -337,14 +339,28 @@ function defaultSessionFactory(input: CaptureSegmentedStageInput): SessionFactor
337339
await closeCaptureSession(probeSession).catch(() => {});
338340
probeSession = null;
339341
}
340-
const session = probeSession ?? (await openSegmentedSession(input, injector));
342+
const withinInitBudget = createSessionInitBudget({
343+
frameIndex: input.job.framesRendered ?? 0,
344+
totalFrames: input.totalFrames,
345+
abortSignal: input.abortSignal,
346+
});
347+
const openSession = async (): Promise<CaptureSession> => {
348+
const opening = openSegmentedSession(input, injector);
349+
try {
350+
return await withinInitBudget(opening, OPENING_CAPTURE_SESSION);
351+
} catch (error) {
352+
void opening.then(closeCaptureSession, () => {});
353+
throw error;
354+
}
355+
};
356+
const session = probeSession ?? (await openSession());
341357
if (probeSession) {
342358
openSegmentedSessionReuse(input, probeSession, injector);
343359
probeSession = null;
344360
}
345361
try {
346-
if (!session.isInitialized) await initializeSession(session);
347-
await completeDeferredDrawElementInit(session);
362+
if (!session.isInitialized) await withinInitBudget(initializeSession(session), session);
363+
await withinInitBudget(completeDeferredDrawElementInit(session), session);
348364
} catch (error) {
349365
await closeCaptureSession(session).catch(() => {});
350366
throw error;

‎packages/producer/src/services/render/stages/captureStage.test.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ const initializeSession = vi.fn(async () => {});
1515
const captureFrame = vi.fn();
1616
const closeCaptureSession = vi.fn(async () => {});
1717
const verifyDiskDrawElementSamples = vi.fn(async () => {});
18+
const completeDeferredDrawElementInit = vi.fn(async () => {});
1819
const getCapturePerfSummary = vi.fn(() => ({}));
1920
const executeParallelCapture = vi.fn();
2021

@@ -27,6 +28,8 @@ vi.mock("@hyperframes/engine", async (importOriginal) => {
2728
captureFrame: (...args: unknown[]) => captureFrame(...args),
2829
closeCaptureSession: (...args: unknown[]) => closeCaptureSession(...args),
2930
verifyDiskDrawElementSamples: (...args: unknown[]) => verifyDiskDrawElementSamples(...args),
31+
completeDeferredDrawElementInit: (...args: unknown[]) =>
32+
completeDeferredDrawElementInit(...args),
3033
getCapturePerfSummary: (...args: unknown[]) => getCapturePerfSummary(...args),
3134
executeParallelCapture: (...args: unknown[]) => executeParallelCapture(...args),
3235
mergeWorkerFrames: async () => {},
@@ -265,6 +268,52 @@ describe("runCaptureStage measured disk projection (both branches)", () => {
265268
closeCaptureSession.mockReset().mockResolvedValue(undefined);
266269
});
267270

271+
it("sequential: gives up on a deferred drawElement init that never finishes, within the session-init budget", async () => {
272+
const { workDir, framesDir } = makeDirs();
273+
freeBytes.mockReturnValue({ bavail: 10_000_000_000, bsize: 1 });
274+
completeDeferredDrawElementInit.mockImplementation(() => new Promise<void>(() => {}));
275+
const probeSession = {
276+
isInitialized: true,
277+
browserConsoleBuffer: [],
278+
options: { captureBeyondViewport: false, format: "jpeg" },
279+
workerEncodeEnabled: false,
280+
outputDir: framesDir,
281+
};
282+
const prevOptIn = process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE;
283+
process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE = "true";
284+
vi.useFakeTimers();
285+
try {
286+
const state = { done: false, error: undefined as unknown, elapsedMs: 0 };
287+
runCaptureStage({
288+
...makeInput(1, workDir, framesDir),
289+
probeSession: probeSession as never,
290+
}).then(
291+
() => (state.done = true),
292+
(error: unknown) => Object.assign(state, { done: true, error }),
293+
);
294+
const flush = async () => {
295+
for (let i = 0; i < 200; i++) await Promise.resolve();
296+
};
297+
await flush();
298+
while (!state.done && state.elapsedMs < 30 * 60_000) {
299+
vi.advanceTimersByTime(1_000);
300+
state.elapsedMs += 1_000;
301+
await flush();
302+
}
303+
expect(state.done).toBe(true);
304+
expect(state.elapsedMs).toBeLessThanOrEqual(305_000);
305+
expect((state.error as Error).message).toContain(
306+
"no frame progress for 300000ms (stuck at frame 0/20)",
307+
);
308+
expect(closeCaptureSession).toHaveBeenCalledWith(probeSession);
309+
} finally {
310+
vi.useRealTimers();
311+
completeDeferredDrawElementInit.mockReset().mockResolvedValue(undefined);
312+
if (prevOptIn === undefined) delete process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE;
313+
else process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE = prevOptIn;
314+
}
315+
});
316+
268317
it("sequential: stops after ten frames when the measured remainder cannot fit", async () => {
269318
const { workDir, framesDir } = makeDirs();
270319
// 8 MB free: passes the static jpeg estimate, fails the 10 MB measured remainder.

‎packages/producer/src/services/render/stages/captureStage.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ import { reportFrameProgress, reportWorkerStartup } from "../shared.js";
7171
import { withWorkerPageErrors } from "../perfSummary.js";
7272
import type { SdrDiskCapturePlan } from "../capturePlan.js";
7373
import type { OpenStallAttempt } from "../observability.js";
74+
import { createSessionInitBudget } from "./captureStreamingStage.js";
7475

7576
export interface CaptureStageInput {
7677
fileServer: FileServerHandle;
@@ -501,7 +502,11 @@ async function runSequentialDiskCapture(
501502
prepareCaptureSessionForReuse(session, framesDir, videoInjector);
502503
probeSession = null;
503504
}
504-
await ensureSessionReady(session);
505+
await ensureSessionReady(session, {
506+
frameIndex: resumeFrom,
507+
totalFrames,
508+
abortSignal: input.abortSignal,
509+
});
505510
assertAttemptActive();
506511
lastBrowserConsole = session.browserConsoleBuffer;
507512

@@ -576,15 +581,18 @@ function planTransientResume(
576581
: null;
577582
}
578583

579-
async function ensureSessionReady(session: CaptureSession): Promise<void> {
584+
async function ensureSessionReady(
585+
session: CaptureSession,
586+
budget: Parameters<typeof createSessionInitBudget>[0],
587+
): Promise<void> {
580588
if (!session.isInitialized) {
581589
await initializeSession(session);
582590
return;
583591
}
584592
// Deferred drawElement init (probe-initialized video comps). The disk path has no
585593
// drain-time self-verification, so only the explicit fast-capture opt-in completes it.
586594
if (process.env.PRODUCER_EXPERIMENTAL_FAST_CAPTURE === "true") {
587-
await completeDeferredDrawElementInit(session);
595+
await createSessionInitBudget(budget)(completeDeferredDrawElementInit(session), session);
588596
}
589597
}
590598

‎packages/producer/src/services/render/stages/captureStreamingStage.ts‎

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -189,9 +189,9 @@ function captureModeLabel(mode: CaptureSession["captureMode"]): string {
189189
return "screenshot";
190190
}
191191

192-
type StalledSession = { captureMode: CaptureSession["captureMode"]; captureStep?: string };
192+
export type StalledSession = { captureMode: CaptureSession["captureMode"]; captureStep?: string };
193193

194-
const OPENING_CAPTURE_SESSION: StalledSession = {
194+
export const OPENING_CAPTURE_SESSION: StalledSession = {
195195
captureMode: "screenshot",
196196
captureStep: "opening the capture session",
197197
};
@@ -271,6 +271,31 @@ export function raceAgainstStall<T>(
271271
});
272272
}
273273

274+
/**
275+
* One budget for everything before a session's first frame: opening it, initializing it, and the
276+
* deferred drawElement init. Each call races against what is left of the same clock.
277+
*/
278+
export function createSessionInitBudget(input: {
279+
frameIndex: number;
280+
totalFrames: number;
281+
abortSignal: AbortSignal | undefined;
282+
}): <T>(work: Promise<T>, stalled: StalledSession) => Promise<T> {
283+
const initStallMs = Math.max(SESSION_INIT_STALL_MS, resolveCaptureStallTimeoutMs());
284+
const setup = createStallClock();
285+
return (work, stalled) =>
286+
raceAgainstStall(
287+
work,
288+
initStallMs - setup.idle(0),
289+
{
290+
session: stalled,
291+
frameIndex: input.frameIndex,
292+
totalFrames: input.totalFrames,
293+
stallTimeoutMs: initStallMs,
294+
},
295+
input.abortSignal,
296+
);
297+
}
298+
274299
/**
275300
* Pre-built ffmpeg streaming-encoder options, exactly matching the
276301
* second argument to `spawnStreamingEncoder`. The sequencer constructs
@@ -1037,15 +1062,11 @@ export async function runCaptureStreamingStage(
10371062
let resumes = 0;
10381063
for (;;) {
10391064
const videoInjector = createRenderVideoFrameInjector();
1040-
const initStallMs = Math.max(SESSION_INIT_STALL_MS, resolveCaptureStallTimeoutMs());
1041-
const setup = createStallClock();
1042-
const withinInitBudget = <T>(work: Promise<T>, stalled: StalledSession) =>
1043-
raceAgainstStall(
1044-
work,
1045-
initStallMs - setup.idle(0),
1046-
{ session: stalled, frameIndex: resume.from, totalFrames, stallTimeoutMs: initStallMs },
1047-
abortSignal,
1048-
);
1065+
const withinInitBudget = createSessionInitBudget({
1066+
frameIndex: resume.from,
1067+
totalFrames,
1068+
abortSignal,
1069+
});
10491070
const openSession = async (): Promise<CaptureSession> => {
10501071
const opening = createCaptureSession(
10511072
fileServer.url,

0 commit comments

Comments
 (0)