Skip to content

Commit 73f3986

Browse files
fix(producer): retry a dead browser tab once on the single-worker capture path (#4162)
* fix(producer): retry a dead browser tab once on the single-worker capture path * fix(producer): keep the caller-owned dedup perf list untouched on sequential retry * test(producer): extend the streaming-stage orchestrator mock with the retry exports
1 parent 0915aff commit 73f3986

4 files changed

Lines changed: 375 additions & 106 deletions

File tree

‎packages/producer/src/services/render/stages/captureStage.test.ts‎

Lines changed: 168 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,38 @@
1-
import { describe, expect, it } from "vitest";
1+
import { afterEach, describe, expect, it, vi } from "vitest";
2+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
3+
import { tmpdir } from "node:os";
4+
import { join } from "node:path";
5+
import { DEFAULT_CONFIG } from "@hyperframes/engine";
6+
import { createRenderJob } from "../../renderOrchestrator.js";
7+
import { formatCaptureFrameName } from "../../../utils/paths.js";
8+
9+
// Mock only the engine session primitives; `classifyCaptureFailure` stays
10+
// real so the retry decision is exercised.
11+
const createCaptureSession = vi.fn();
12+
const initializeSession = vi.fn(async () => {});
13+
const captureFrame = vi.fn();
14+
const closeCaptureSession = vi.fn(async () => {});
15+
const verifyDiskDrawElementSamples = vi.fn(async () => {});
16+
const getCapturePerfSummary = vi.fn(() => ({}));
17+
18+
vi.mock("@hyperframes/engine", async (importOriginal) => {
19+
const actual = await importOriginal<typeof import("@hyperframes/engine")>();
20+
return {
21+
...actual,
22+
createCaptureSession: (...args: unknown[]) => createCaptureSession(...args),
23+
initializeSession: (...args: unknown[]) => initializeSession(...args),
24+
captureFrame: (...args: unknown[]) => captureFrame(...args),
25+
closeCaptureSession: (...args: unknown[]) => closeCaptureSession(...args),
26+
verifyDiskDrawElementSamples: (...args: unknown[]) => verifyDiskDrawElementSamples(...args),
27+
getCapturePerfSummary: (...args: unknown[]) => getCapturePerfSummary(...args),
28+
};
29+
});
30+
231
import {
332
assertDiskCaptureHeadroom,
433
estimateDiskCaptureBytes,
534
inspectDiskCaptureHeadroom,
35+
runCaptureStage,
636
shouldAllowAdaptiveCaptureRetry,
737
} from "./captureStage.js";
838

@@ -96,3 +126,140 @@ describe("disk capture capacity", () => {
96126
expect(headroom.available).toBe(false);
97127
});
98128
});
129+
130+
describe("runCaptureStage — sequential path transient Target-closed single retry (integration)", () => {
131+
afterEach(() => {
132+
createCaptureSession.mockReset();
133+
initializeSession.mockReset().mockResolvedValue(undefined);
134+
captureFrame.mockReset();
135+
closeCaptureSession.mockReset().mockResolvedValue(undefined);
136+
verifyDiskDrawElementSamples.mockReset().mockResolvedValue(undefined);
137+
getCapturePerfSummary.mockReset().mockReturnValue({});
138+
});
139+
140+
it("retries once with a fresh session on a Target-closed and renders the remaining frames", async () => {
141+
const framesDir = mkdtempSync(join(tmpdir(), "hf-seq-transient-frames-"));
142+
const totalFrames = 4;
143+
let sessionCount = 0;
144+
createCaptureSession.mockImplementation(async () => {
145+
sessionCount++;
146+
return {
147+
isInitialized: false,
148+
browserConsoleBuffer: [],
149+
options: { captureBeyondViewport: false, format: "jpeg" },
150+
workerEncodeEnabled: false,
151+
outputDir: framesDir,
152+
};
153+
});
154+
captureFrame.mockImplementation(async (_session: unknown, frameIndex: number) => {
155+
// First session dies on frame 0 (zero forward progress).
156+
if (sessionCount === 1) {
157+
throw new Error("Protocol error (Page.captureScreenshot): Target closed");
158+
}
159+
writeFileSync(join(framesDir, formatCaptureFrameName(frameIndex, "jpg")), "captured-frame");
160+
});
161+
const log = { error: vi.fn(), warn: vi.fn(), info: vi.fn(), debug: vi.fn() };
162+
163+
try {
164+
const job = createRenderJob({ fps: { num: 30, den: 1 }, quality: "draft" });
165+
const result = await runCaptureStage({
166+
fileServer: { url: "http://localhost:0" } as never,
167+
workDir: framesDir,
168+
framesDir,
169+
job,
170+
totalFrames,
171+
cfg: DEFAULT_CONFIG,
172+
plan: { workerCount: 1, forceScreenshot: false },
173+
log: log as never,
174+
probeSession: null,
175+
captureAttempts: [],
176+
dedupPerfs: [],
177+
buildCaptureOptions: () => ({
178+
width: 64,
179+
height: 64,
180+
fps: { num: 30, den: 1 },
181+
format: "jpeg",
182+
}),
183+
createRenderVideoFrameInjector: () => null,
184+
abortSignal: undefined,
185+
assertNotAborted: () => {},
186+
});
187+
188+
expect(createCaptureSession).toHaveBeenCalledTimes(2);
189+
expect(captureFrame).toHaveBeenCalledTimes(1 + totalFrames);
190+
expect(closeCaptureSession).toHaveBeenCalledTimes(2);
191+
expect(result.workerCount).toBe(1);
192+
expect(log.warn).toHaveBeenCalledWith(
193+
expect.stringContaining("Transient browser failure during sequential capture"),
194+
expect.objectContaining({ transientRetriesUsed: 1, resumeFrom: 0 }),
195+
);
196+
} finally {
197+
rmSync(framesDir, { recursive: true, force: true });
198+
}
199+
});
200+
201+
it("resumes from the first missing frame inside a frameRange and fails on a second death", async () => {
202+
const framesDir = mkdtempSync(join(tmpdir(), "hf-seq-transient-range-"));
203+
let sessionCount = 0;
204+
createCaptureSession.mockImplementation(async () => {
205+
sessionCount++;
206+
return {
207+
isInitialized: false,
208+
browserConsoleBuffer: [],
209+
options: { captureBeyondViewport: false, format: "jpeg" },
210+
workerEncodeEnabled: false,
211+
outputDir: framesDir,
212+
};
213+
});
214+
const times: number[] = [];
215+
captureFrame.mockImplementation(async (_s: unknown, frameIndex: number, time: number) => {
216+
times.push(time);
217+
if (sessionCount === 1 && frameIndex === 2) {
218+
throw new Error("Protocol error (Runtime.callFunctionOn): Target closed");
219+
}
220+
if (sessionCount === 2 && frameIndex === 3) {
221+
throw new Error("Protocol error (Runtime.callFunctionOn): Target closed");
222+
}
223+
writeFileSync(join(framesDir, formatCaptureFrameName(frameIndex, "jpg")), "captured-frame");
224+
});
225+
const log = { error: vi.fn(), warn: vi.fn(), info: vi.fn(), debug: vi.fn() };
226+
const job = createRenderJob({ fps: { num: 10, den: 1 }, quality: "draft" });
227+
228+
try {
229+
await expect(
230+
runCaptureStage({
231+
fileServer: { url: "http://localhost:0" } as never,
232+
workDir: framesDir,
233+
framesDir,
234+
job,
235+
totalFrames: 4,
236+
cfg: DEFAULT_CONFIG,
237+
plan: { workerCount: 1, forceScreenshot: false },
238+
log: log as never,
239+
probeSession: null,
240+
captureAttempts: [],
241+
dedupPerfs: [],
242+
buildCaptureOptions: () => ({
243+
width: 64,
244+
height: 64,
245+
fps: { num: 10, den: 1 },
246+
format: "jpeg",
247+
}),
248+
createRenderVideoFrameInjector: () => null,
249+
abortSignal: undefined,
250+
assertNotAborted: () => {},
251+
frameRange: { startFrame: 10, endFrame: 14 },
252+
}),
253+
).rejects.toThrow(/Target closed/);
254+
255+
// Session 1: frames 0,1 ok, 2 dies. Session 2 resumes at 2 (absolute 12 -> 1.2s),
256+
// frame 2 ok, 3 dies; no third session.
257+
expect(createCaptureSession).toHaveBeenCalledTimes(2);
258+
expect(times).toEqual([1.0, 1.1, 1.2, 1.2, 1.3]);
259+
expect(log.warn).toHaveBeenCalledTimes(1);
260+
expect(closeCaptureSession).toHaveBeenCalledTimes(2);
261+
} finally {
262+
rmSync(framesDir, { recursive: true, force: true });
263+
}
264+
});
265+
});

0 commit comments

Comments
 (0)