Skip to content

Commit 04bc473

Browse files
fix(producer): retry with fewer workers when every page times out loading (#7171)
1 parent cd7b738 commit 04bc473

2 files changed

Lines changed: 67 additions & 27 deletions

File tree

‎packages/producer/src/services/renderOrchestrator.test.ts‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -428,6 +428,46 @@ describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single
428428
}
429429
});
430430

431+
it("halves the workers when every page times out loading, instead of reopening as many", async () => {
432+
const workDir = mkdtempSync(join(tmpdir(), "hf-navtimeout-work-"));
433+
const framesDir = mkdtempSync(join(tmpdir(), "hf-navtimeout-frames-"));
434+
const log = makeLog();
435+
let call = 0;
436+
vi.mocked(executeParallelCapture).mockImplementation(async () => {
437+
call++;
438+
if (call === 1) {
439+
throw new Error(
440+
"[Parallel] Capture failed: Worker 0: Navigation timeout of 60000 ms exceeded",
441+
);
442+
}
443+
writeAllFrames(framesDir, 16);
444+
return [];
445+
});
446+
vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined);
447+
448+
try {
449+
const attempts = await executeDiskCaptureWithAdaptiveRetry({
450+
serverUrl: "http://localhost:0",
451+
workDir,
452+
framesDir,
453+
totalFrames: 16,
454+
initialWorkerCount: 8,
455+
allowRetry: true,
456+
frameExt: "jpg",
457+
captureOptions: { width: 64, height: 64, fps: { num: 30, den: 1 } },
458+
createBeforeCaptureHook: () => null,
459+
cfg: DEFAULT_CONFIG,
460+
log,
461+
dedupPerfs: [],
462+
});
463+
464+
expect(attempts.map((a) => a.workers)).toEqual([8, 4]);
465+
} finally {
466+
rmSync(workDir, { recursive: true, force: true });
467+
rmSync(framesDir, { recursive: true, force: true });
468+
}
469+
});
470+
431471
it.each(["Target closed", "Network.enable timed out"])(
432472
"does NOT retry %s after cancellation",
433473
async (message) => {
@@ -475,7 +515,7 @@ describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single
475515
},
476516
);
477517

478-
it.each(["Session closed", "Network.enable timed out"])(
518+
it.each(["Session closed", "Network.enable timed out", "Navigation timeout of 9 ms exceeded"])(
479519
"bounds repeated %s failures",
480520
async (message) => {
481521
const workDir = mkdtempSync(join(tmpdir(), "hf-transient2-work-"));

‎packages/producer/src/services/renderOrchestrator.ts‎

Lines changed: 26 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1472,6 +1472,32 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
14721472
const remainingCount = countFrameRanges(remaining);
14731473
const madeProgress = captureAttemptMadeProgress(frameCount, remainingCount);
14741474

1475+
// A browser that times out starting up or loading its page before any frame exists had too many pages
1476+
// opening at once: retry with half the workers, down to one. Other zero-progress errors fail below.
1477+
// A lone worker gets one retry: here for Network.enable, as the transient retry below for a page load.
1478+
if (
1479+
options.allowRetry &&
1480+
!madeProgress &&
1481+
(currentWorkers > 1 ||
1482+
(initializationRetriesUsed === 0 && failure.kind === "protocol_timeout")) &&
1483+
/\bNetwork\.enable timed out|Navigation timeout of \d+ ms exceeded/i.test(failure.message)
1484+
) {
1485+
initializationRetriesUsed++;
1486+
const nextWorkers = getNextRetryWorkerCount(currentWorkers);
1487+
options.log.warn(
1488+
"[Render] Browser initialization timed out; retrying with fewer workers.",
1489+
{
1490+
fromWorkers: currentWorkers,
1491+
toWorkers: nextWorkers,
1492+
error: failure.message,
1493+
},
1494+
);
1495+
currentWorkers = nextWorkers;
1496+
missingRanges = remaining;
1497+
attempt++;
1498+
continue;
1499+
}
1500+
14751501
// Single bounded retry for a transient browser death (`Target closed` /
14761502
// `Page crashed` / `Session closed`): the tab died mid-capture, not the
14771503
// composition. Unlike the worker-halving retry below, this keeps the same
@@ -1500,32 +1526,6 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
15001526
continue;
15011527
}
15021528

1503-
// CDP initialization can time out before any frame exists. Give that
1504-
// specific startup failure one fresh attempt with less concurrency;
1505-
// arbitrary zero-progress authoring/capture errors still fail below.
1506-
if (
1507-
options.allowRetry &&
1508-
!madeProgress &&
1509-
initializationRetriesUsed === 0 &&
1510-
failure.kind === "protocol_timeout" &&
1511-
/\bNetwork\.enable timed out/i.test(failure.message)
1512-
) {
1513-
initializationRetriesUsed++;
1514-
const nextWorkers = getNextRetryWorkerCount(currentWorkers);
1515-
options.log.warn(
1516-
"[Render] Browser initialization timed out; retrying once with fresh sessions.",
1517-
{
1518-
fromWorkers: currentWorkers,
1519-
toWorkers: nextWorkers,
1520-
error: failure.message,
1521-
},
1522-
);
1523-
currentWorkers = nextWorkers;
1524-
missingRanges = remaining;
1525-
attempt++;
1526-
continue;
1527-
}
1528-
15291529
if (!madeProgress) {
15301530
options.log.warn(
15311531
"[Render] Capture attempt made no forward progress; composition is likely structurally broken, not retrying.",

0 commit comments

Comments
 (0)