Skip to content

Commit 7860d19

Browse files
fix(capture): fall back from networkidle2 to domcontentloaded on nav timeout (#3224)
* fix(cli): fall back from networkidle2 to domcontentloaded on capture nav timeout Sites like yahoo.com never reach network idle, so website capture hung for the full 120s navigation budget. Prefer a short networkidle2 attempt, then continue with domcontentloaded and the existing settle/scroll path. Co-authored-by: Cursor <[email protected]> * fix(capture): use remaining timeout budget for domcontentloaded fallback Keep total navigation time within the caller --timeout instead of re-applying a full 30s floor after networkidle2. Co-authored-by: Cursor <[email protected]> --------- Co-authored-by: Cursor <[email protected]>
1 parent 896bc33 commit 7860d19

3 files changed

Lines changed: 194 additions & 4 deletions

File tree

‎packages/cli/src/capture/index.ts‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ import {
4242
import type { VisionCaptionOutcome } from "./contentExtractor.js";
4343
import { loadEnvFile, generateProjectScaffold } from "./scaffolding.js";
4444
import { detectBlockedPage } from "./pageBlockDetection.js";
45+
import { navigateForCapture } from "./navigateForCapture.js";
4546
import type { CaptureOptions, CapturePhase, CapturePhaseProgress, CaptureResult } from "./types.js";
4647

4748
export type { CaptureOptions, CaptureResult } from "./types.js";
@@ -238,10 +239,17 @@ export async function captureWebsite(
238239
}
239240
});
240241

241-
// Use networkidle2 (allows 2 ongoing connections) instead of networkidle0 —
242-
// modern SPAs often have persistent WebSocket/analytics connections that
243-
// prevent networkidle0 from ever resolving.
244-
const navigationResponse = await page1.goto(url, { waitUntil: "networkidle2", timeout });
242+
const navigation = await navigateForCapture(page1, url, timeout);
243+
const navigationResponse = navigation.response;
244+
if (navigation.fellBackFromNetworkIdle) {
245+
warnings.push(
246+
`networkidle2 timed out after ${navigation.networkIdleTimeoutMs}ms; continued with domcontentloaded`,
247+
);
248+
progress(
249+
"warn",
250+
`networkidle2 timed out after ${navigation.networkIdleTimeoutMs}ms; continuing with domcontentloaded`,
251+
);
252+
}
245253
postNavigationDeadline = Date.now() + budgetMs;
246254
await new Promise((r) => setTimeout(r, settleTime));
247255

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
import { describe, expect, it, vi } from "vitest";
2+
import {
3+
isNavigationTimeoutError,
4+
navigateForCapture,
5+
networkIdleAttemptTimeoutMs,
6+
NETWORK_IDLE_ATTEMPT_MS,
7+
} from "./navigateForCapture.js";
8+
9+
describe("networkIdleAttemptTimeoutMs", () => {
10+
it("caps at the network-idle attempt budget", () => {
11+
expect(networkIdleAttemptTimeoutMs(120_000)).toBe(NETWORK_IDLE_ATTEMPT_MS);
12+
});
13+
14+
it("honors a caller timeout below the attempt budget", () => {
15+
expect(networkIdleAttemptTimeoutMs(10_000)).toBe(10_000);
16+
});
17+
});
18+
19+
describe("isNavigationTimeoutError", () => {
20+
it("matches Puppeteer navigation timeouts", () => {
21+
expect(isNavigationTimeoutError(new Error("Navigation timeout of 30000 ms exceeded"))).toBe(
22+
true,
23+
);
24+
});
25+
26+
it("ignores unrelated failures", () => {
27+
expect(isNavigationTimeoutError(new Error("net::ERR_NAME_NOT_RESOLVED"))).toBe(false);
28+
});
29+
});
30+
31+
describe("navigateForCapture", () => {
32+
it("returns after networkidle2 when the page settles", async () => {
33+
const response = { status: () => 200 };
34+
const goto = vi.fn().mockResolvedValue(response);
35+
const result = await navigateForCapture({ goto }, "https://example.com", 120_000);
36+
37+
expect(result).toEqual({
38+
response,
39+
waitUntil: "networkidle2",
40+
networkIdleTimeoutMs: NETWORK_IDLE_ATTEMPT_MS,
41+
fellBackFromNetworkIdle: false,
42+
});
43+
expect(goto).toHaveBeenCalledTimes(1);
44+
expect(goto).toHaveBeenCalledWith("https://example.com", {
45+
waitUntil: "networkidle2",
46+
timeout: NETWORK_IDLE_ATTEMPT_MS,
47+
});
48+
});
49+
50+
it("falls back to domcontentloaded after a networkidle2 navigation timeout", async () => {
51+
const response = { status: () => 200 };
52+
const goto = vi
53+
.fn()
54+
.mockRejectedValueOnce(new Error("Navigation timeout of 30000 ms exceeded"))
55+
.mockResolvedValueOnce(response);
56+
57+
const result = await navigateForCapture({ goto }, "https://www.yahoo.com/", 120_000);
58+
59+
expect(result).toEqual({
60+
response,
61+
waitUntil: "domcontentloaded",
62+
networkIdleTimeoutMs: NETWORK_IDLE_ATTEMPT_MS,
63+
fellBackFromNetworkIdle: true,
64+
});
65+
expect(goto).toHaveBeenNthCalledWith(1, "https://www.yahoo.com/", {
66+
waitUntil: "networkidle2",
67+
timeout: NETWORK_IDLE_ATTEMPT_MS,
68+
});
69+
expect(goto).toHaveBeenNthCalledWith(2, "https://www.yahoo.com/", {
70+
waitUntil: "domcontentloaded",
71+
timeout: 90_000,
72+
});
73+
});
74+
75+
it.each([
76+
{ totalTimeoutMs: 31_000, fallbackTimeoutMs: 1_000 },
77+
{ totalTimeoutMs: 45_000, fallbackTimeoutMs: 15_000 },
78+
])(
79+
"uses only the remaining budget for fallback when totalTimeoutMs=$totalTimeoutMs",
80+
async ({ totalTimeoutMs, fallbackTimeoutMs }) => {
81+
const response = { status: () => 200 };
82+
const goto = vi
83+
.fn()
84+
.mockRejectedValueOnce(new Error("Navigation timeout of 30000 ms exceeded"))
85+
.mockResolvedValueOnce(response);
86+
87+
await navigateForCapture({ goto }, "https://www.yahoo.com/", totalTimeoutMs);
88+
89+
expect(goto).toHaveBeenNthCalledWith(1, "https://www.yahoo.com/", {
90+
waitUntil: "networkidle2",
91+
timeout: NETWORK_IDLE_ATTEMPT_MS,
92+
});
93+
expect(goto).toHaveBeenNthCalledWith(2, "https://www.yahoo.com/", {
94+
waitUntil: "domcontentloaded",
95+
timeout: fallbackTimeoutMs,
96+
});
97+
},
98+
);
99+
100+
it("does not fall back for non-timeout navigation errors", async () => {
101+
const err = new Error("net::ERR_CONNECTION_REFUSED");
102+
const goto = vi.fn().mockRejectedValue(err);
103+
104+
await expect(navigateForCapture({ goto }, "https://example.com", 120_000)).rejects.toBe(err);
105+
expect(goto).toHaveBeenCalledTimes(1);
106+
});
107+
108+
it("does not fall back when the caller timeout already equals the idle attempt", async () => {
109+
const err = new Error("Navigation timeout of 10000 ms exceeded");
110+
const goto = vi.fn().mockRejectedValue(err);
111+
112+
await expect(navigateForCapture({ goto }, "https://example.com", 10_000)).rejects.toBe(err);
113+
expect(goto).toHaveBeenCalledWith("https://example.com", {
114+
waitUntil: "networkidle2",
115+
timeout: 10_000,
116+
});
117+
});
118+
});
Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
export const NETWORK_IDLE_ATTEMPT_MS = 30_000;
2+
3+
export type CaptureGotoWaitUntil = "networkidle2" | "domcontentloaded";
4+
5+
export interface CaptureGotoOptions {
6+
waitUntil: CaptureGotoWaitUntil;
7+
timeout: number;
8+
}
9+
10+
export interface CaptureGotoPage<TResponse = unknown> {
11+
goto(url: string, options: CaptureGotoOptions): Promise<TResponse>;
12+
}
13+
14+
export interface NavigateForCaptureResult<TResponse = unknown> {
15+
response: TResponse;
16+
waitUntil: CaptureGotoWaitUntil;
17+
networkIdleTimeoutMs: number;
18+
fellBackFromNetworkIdle: boolean;
19+
}
20+
21+
export function networkIdleAttemptTimeoutMs(totalTimeoutMs: number): number {
22+
return Math.min(NETWORK_IDLE_ATTEMPT_MS, Math.max(0, totalTimeoutMs));
23+
}
24+
25+
export function isNavigationTimeoutError(err: unknown): boolean {
26+
const msg = err instanceof Error ? err.message : String(err);
27+
return /navigation timeout/i.test(msg);
28+
}
29+
30+
export async function navigateForCapture<TResponse>(
31+
page: CaptureGotoPage<TResponse>,
32+
url: string,
33+
totalTimeoutMs: number,
34+
): Promise<NavigateForCaptureResult<TResponse>> {
35+
const networkIdleTimeoutMs = networkIdleAttemptTimeoutMs(totalTimeoutMs);
36+
try {
37+
const response = await page.goto(url, {
38+
waitUntil: "networkidle2",
39+
timeout: networkIdleTimeoutMs,
40+
});
41+
return {
42+
response,
43+
waitUntil: "networkidle2",
44+
networkIdleTimeoutMs,
45+
fellBackFromNetworkIdle: false,
46+
};
47+
} catch (err) {
48+
if (!isNavigationTimeoutError(err) || networkIdleTimeoutMs >= totalTimeoutMs) {
49+
throw err;
50+
}
51+
}
52+
53+
const fallbackTimeoutMs = totalTimeoutMs - networkIdleTimeoutMs;
54+
const response = await page.goto(url, {
55+
waitUntil: "domcontentloaded",
56+
timeout: fallbackTimeoutMs,
57+
});
58+
return {
59+
response,
60+
waitUntil: "domcontentloaded",
61+
networkIdleTimeoutMs,
62+
fellBackFromNetworkIdle: true,
63+
};
64+
}

0 commit comments

Comments
 (0)