Skip to content

Commit be3f177

Browse files
committed
fix(cli): bound consent-click settle by wall clock and skip it on banner-free pages
waitForPageReady now shares one deadline (timeoutMs, capped by the remaining capture budget) across its network-idle waits and readyState poll, uses waitForNetworkIdle concurrency: 2 (the networkidle2 semantic navigation already uses), races each readyState evaluate against the deadline, and rethrows degradable evaluate timeouts so the stage can degrade. captureScrollScreenshots only pays the settle when the consent sweep clicked a control or performance.timeOrigin shows the document was swapped; the catalog boundary call degrades to a budget-bounded readyState check (networkIdle: after-load).
1 parent 73ac3bb commit be3f177

5 files changed

Lines changed: 219 additions & 18 deletions

File tree

‎packages/cli/src/capture/coreExtractionPhase.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,13 @@ export async function runCoreExtraction(input: CoreExtractionInput): Promise<Cor
310310
await runAnimationCapture();
311311

312312
const runHtmlExtraction = async (): Promise<void> => {
313-
await waitForPageReady(page1);
313+
// The screenshot stage already settles consent-click reloads; this is a
314+
// bounded readyState guard — one evaluate on a parsed page — so the
315+
// catalog never reads a still-loading or mid-navigation document.
316+
await waitForPageReady(page1, {
317+
timeoutMs: Math.min(15_000, remainingMs()),
318+
networkIdle: "after-load",
319+
});
314320

315321
// Catalog all assets (must run before extractHtml which converts img src to data URLs)
316322
progress("design", "Cataloging assets...");

‎packages/cli/src/capture/navigateForCapture.test.ts‎

Lines changed: 83 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,10 @@ describe("waitForPageReady", () => {
126126

127127
await waitForPageReady({ waitForNetworkIdle, evaluate });
128128

129+
// concurrency: 2 — the networkidle2 semantic navigation already uses, so
130+
// a single held-open request cannot block the wait until timeout.
129131
expect(waitForNetworkIdle).toHaveBeenCalledTimes(1);
132+
expect(waitForNetworkIdle).toHaveBeenCalledWith(expect.objectContaining({ concurrency: 2 }));
130133
});
131134

132135
it("re-waits for idle once a still-parsing document finishes loading", async () => {
@@ -157,20 +160,96 @@ describe("waitForPageReady", () => {
157160
expect(waitForNetworkIdle).toHaveBeenCalledTimes(2);
158161
});
159162

160-
it("stays bounded when the document never reports a parsed state", async () => {
163+
it("rethrows degradable evaluate timeouts instead of polling through them", async () => {
164+
const waitForNetworkIdle = vi.fn(async () => undefined);
165+
const evaluate = vi.fn(async () => {
166+
throw new TimeoutError(
167+
"Runtime.evaluate timed out. Increase the 'protocolTimeout' setting in launch/connect calls for a higher timeout if needed.",
168+
);
169+
});
170+
171+
await expect(waitForPageReady({ waitForNetworkIdle, evaluate })).rejects.toBeInstanceOf(
172+
TimeoutError,
173+
);
174+
// The stage degrades on the first timeout — it is not retried as
175+
// "still loading" for the full deadline.
176+
expect(evaluate).toHaveBeenCalledTimes(1);
177+
});
178+
179+
it("stays bounded by wall clock when the document never reports a parsed state", async () => {
161180
vi.useFakeTimers();
162181
try {
163182
const waitForNetworkIdle = vi.fn(async () => undefined);
164183
const evaluate = vi.fn(async () => true);
165184

166-
const done = waitForPageReady({ waitForNetworkIdle, evaluate });
185+
const start = Date.now();
186+
const done = waitForPageReady({ waitForNetworkIdle, evaluate }, { timeoutMs: 15_000 });
167187
await vi.runAllTimersAsync();
168188
await done;
169189

170-
expect(evaluate).toHaveBeenCalledTimes(30);
171-
expect(waitForNetworkIdle).toHaveBeenCalledTimes(2);
190+
expect(Date.now() - start).toBeLessThanOrEqual(15_000);
191+
// ~500ms polls over the deadline — not unbounded.
192+
expect(evaluate.mock.calls.length).toBeLessThanOrEqual(31);
172193
} finally {
173194
vi.useRealTimers();
174195
}
175196
});
197+
198+
it("bounds a hung evaluate to the shared deadline, not per-iteration latency", async () => {
199+
vi.useFakeTimers();
200+
try {
201+
// Each probe hangs for a protocol-timeout-scale 30s; the settle must
202+
// still finish at the deadline rather than paying 30s per poll.
203+
const evaluate = vi.fn(
204+
() =>
205+
new Promise<unknown>((resolve) => {
206+
setTimeout(() => resolve(true), 30_000);
207+
}),
208+
);
209+
const waitForNetworkIdle = vi.fn(async () => undefined);
210+
211+
const start = Date.now();
212+
const done = waitForPageReady({ waitForNetworkIdle, evaluate }, { timeoutMs: 15_000 });
213+
await vi.advanceTimersByTimeAsync(15_000);
214+
await done;
215+
216+
expect(Date.now() - start).toBeLessThanOrEqual(15_000);
217+
expect(evaluate).toHaveBeenCalledTimes(1);
218+
} finally {
219+
vi.useRealTimers();
220+
}
221+
});
222+
223+
it("skips the leading idle wait in after-load mode on a parsed document", async () => {
224+
const waitForNetworkIdle = vi.fn(async () => undefined);
225+
const evaluate = vi.fn(async () => false);
226+
227+
await waitForPageReady({ waitForNetworkIdle, evaluate }, { networkIdle: "after-load" });
228+
229+
expect(waitForNetworkIdle).not.toHaveBeenCalled();
230+
});
231+
232+
it("still idles in after-load mode once a loading document finishes", async () => {
233+
const waitForNetworkIdle = vi.fn(async () => undefined);
234+
let loading = true;
235+
const evaluate = vi.fn(async () => {
236+
const was = loading;
237+
loading = false;
238+
return was;
239+
});
240+
241+
await waitForPageReady({ waitForNetworkIdle, evaluate }, { networkIdle: "after-load" });
242+
243+
expect(waitForNetworkIdle).toHaveBeenCalledTimes(1);
244+
});
245+
246+
it("is a no-op when the deadline has already passed", async () => {
247+
const waitForNetworkIdle = vi.fn(async () => undefined);
248+
const evaluate = vi.fn(async () => false);
249+
250+
await waitForPageReady({ waitForNetworkIdle, evaluate }, { timeoutMs: 0 });
251+
252+
expect(evaluate).not.toHaveBeenCalled();
253+
expect(waitForNetworkIdle).not.toHaveBeenCalled();
254+
});
176255
});

‎packages/cli/src/capture/navigateForCapture.ts‎

Lines changed: 65 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { isNavigationTimeoutError } from "./captureTimeout.js";
1+
import { isDegradableEvaluateTimeoutError, isNavigationTimeoutError } from "./captureTimeout.js";
22

33
export { isNavigationTimeoutError };
44

@@ -63,19 +63,75 @@ export async function navigateForCapture<TResponse>(
6363
}
6464

6565
export interface PageReadyPage {
66-
waitForNetworkIdle(options: { idleTime: number; timeout: number }): Promise<unknown>;
66+
waitForNetworkIdle(options: {
67+
idleTime: number;
68+
timeout: number;
69+
concurrency?: number;
70+
}): Promise<unknown>;
6771
evaluate(fn: () => unknown): Promise<unknown>;
6872
}
6973

70-
export async function waitForPageReady(page: PageReadyPage): Promise<void> {
71-
await page.waitForNetworkIdle({ idleTime: 500, timeout: 15_000 }).catch(() => {});
74+
export interface WaitForPageReadyOptions {
75+
/**
76+
* Wall-clock ceiling shared by the idle waits and the readyState poll —
77+
* each wait gets only the time still left. Pass the remaining capture
78+
* budget (min'd with 15_000) so the settle cannot outrun the stage.
79+
*/
80+
timeoutMs?: number;
81+
/**
82+
* "always" pays one bounded idle wait up front — the post-click settle,
83+
* where a navigation or consent XHR is expected. "after-load" skips that
84+
* wait and idles only when the poll actually saw a loading document — a
85+
* cheap guard for stages that already settled the page.
86+
*/
87+
networkIdle?: "always" | "after-load";
88+
}
89+
90+
export async function waitForPageReady(
91+
page: PageReadyPage,
92+
options: WaitForPageReadyOptions = {},
93+
): Promise<void> {
94+
const { timeoutMs = 15_000, networkIdle = "always" } = options;
95+
const deadline = Date.now() + timeoutMs;
96+
const remaining = () => Math.max(0, deadline - Date.now());
97+
const idle = () =>
98+
page
99+
// concurrency: 2 matches the networkidle2 semantic navigation already
100+
// uses, so one held-open request (long-poll, beacon) cannot block it.
101+
.waitForNetworkIdle({ idleTime: 500, timeout: remaining(), concurrency: 2 })
102+
.catch(() => {});
103+
104+
if (networkIdle === "always" && remaining() > 0) await idle();
105+
106+
// Poll the document out of a still-loading or mid-navigation state. The
107+
// bound is wall clock, not iteration count: each evaluate races the
108+
// remaining deadline, so a hung execution-context wait (~30s, or the
109+
// protocol timeout) cannot stretch the settle. Degradable protocol/stage
110+
// timeouts rethrow so the stage can degrade like the rest of capture.
72111
let waitedOutLoad = false;
73-
for (let i = 0; i < 30; i++) {
74-
const loading = await page.evaluate(() => document.readyState === "loading").catch(() => true);
112+
while (remaining() > 0) {
113+
let timer: ReturnType<typeof setTimeout> | undefined;
114+
const loading = await Promise.race([
115+
page
116+
.evaluate(() => document.readyState === "loading")
117+
.catch((err: unknown) => {
118+
if (isDegradableEvaluateTimeoutError(err)) throw err;
119+
return true;
120+
}),
121+
new Promise<true>((resolve) => {
122+
timer = setTimeout(() => resolve(true), Math.max(1, remaining()));
123+
}),
124+
]).finally(() => {
125+
if (timer !== undefined) clearTimeout(timer);
126+
});
75127
if (!loading) break;
76128
waitedOutLoad = true;
77-
await new Promise((resolve) => setTimeout(resolve, 500));
129+
if (remaining() > 0) {
130+
await new Promise((resolve) => setTimeout(resolve, Math.min(500, remaining())));
131+
}
78132
}
79-
if (!waitedOutLoad) return;
80-
await page.waitForNetworkIdle({ idleTime: 500, timeout: 15_000 }).catch(() => {});
133+
134+
// A document we had to wait out keeps fetching once parsed — settle those
135+
// requests too, inside the same deadline.
136+
if (waitedOutLoad && remaining() > 0) await idle();
81137
}

‎packages/cli/src/capture/screenshotCapture.test.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,42 @@ describe("captureScrollScreenshots — consent-click page reload (#4923)", () =>
367367
expect(waitForNetworkIdle).toHaveBeenCalledTimes(1);
368368
});
369369

370+
it("skips the settle when nothing was clicked and the document is unchanged", async () => {
371+
const dir = tempDir("hf-scroll-no-settle-");
372+
// A stable timeOrigin proves the same document before and after the
373+
// sweep, so a banner-free page pays the flat 400ms and no idle wait.
374+
const evaluate = vi.fn(async (script?: unknown) => {
375+
const src = String(script);
376+
if (src.includes("performance.timeOrigin")) return 12_345;
377+
if (src.includes("el.click()")) return 0;
378+
return settledSize(src);
379+
});
380+
const waitForNetworkIdle = vi.fn(async () => undefined);
381+
const { page } = scrollablePage(evaluate, waitForNetworkIdle);
382+
383+
const files = await captureScrollScreenshots(page, dir);
384+
385+
expect(files.length).toBeGreaterThan(0);
386+
expect(waitForNetworkIdle).not.toHaveBeenCalled();
387+
});
388+
389+
it("settles when the sweep clicked a control even without a reload", async () => {
390+
const dir = tempDir("hf-scroll-clicked-");
391+
const evaluate = vi.fn(async (script?: unknown) => {
392+
const src = String(script);
393+
if (src.includes("performance.timeOrigin")) return 12_345;
394+
if (src.includes("el.click()")) return 1;
395+
return settledSize(src);
396+
});
397+
const waitForNetworkIdle = vi.fn(async () => undefined);
398+
const { page } = scrollablePage(evaluate, waitForNetworkIdle);
399+
400+
const files = await captureScrollScreenshots(page, dir);
401+
402+
expect(files.length).toBeGreaterThan(0);
403+
expect(waitForNetworkIdle).toHaveBeenCalledTimes(1);
404+
});
405+
370406
// Puppeteer's waitForNetworkIdle counts a request only until its response
371407
// headers arrive, so it resolves while a reloaded document is still
372408
// parsing. waitForNavigation cannot help there — it deadlocks when armed

‎packages/cli/src/capture/screenshotCapture.ts‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,19 @@ export async function captureScrollScreenshots(
130130
// These overlay content and contaminate screenshots with UI that doesn't
131131
// belong in video compositions (cookie popups, newsletter modals, etc.)
132132
// A consent click can reload the page (some CMPs reload once consent is
133-
// stored); wait out the reloading document before reading the DOM again.
134-
await page
133+
// stored); a swapped document reads as a new performance.timeOrigin. Read
134+
// it around the sweep so a banner-free page skips the settle and keeps
135+
// the old flat-400ms cost — an unreadable (NaN) pair counts as "cannot
136+
// prove unchanged" and settles on the safe side.
137+
const readTimeOrigin = async (): Promise<number> => {
138+
const value: unknown = await page.evaluate(`performance.timeOrigin`).catch((err: unknown) => {
139+
if (isDegradableEvaluateTimeoutError(err)) throw err;
140+
return undefined;
141+
});
142+
return typeof value === "number" ? value : Number.NaN;
143+
};
144+
const originBefore = await readTimeOrigin();
145+
const clickedCount = await page
135146
.evaluate(() => {
136147
// Click common dismiss/accept buttons
137148
const selectors = [
@@ -161,22 +172,35 @@ export async function captureScrollScreenshots(
161172
'button[class*="not-now"]',
162173
'button[class*="no-thanks"]',
163174
];
175+
let clicked = 0;
164176
for (const sel of selectors) {
165177
try {
166178
const el = document.querySelector<HTMLElement>(sel);
167-
if (el) el.click();
179+
if (el) {
180+
el.click();
181+
clicked++;
182+
}
168183
} catch {
169184
/* ignore */
170185
}
171186
}
187+
return clicked;
172188
})
173189
.catch((err: unknown) => {
174190
if (isDegradableEvaluateTimeoutError(err)) {
175191
throw err;
176192
}
193+
return 0;
177194
});
178195
await new Promise((r) => setTimeout(r, 400));
179-
await waitForPageReady(page);
196+
// Settle only when the sweep clicked a control or the document was
197+
// swapped under us — the click reload is the case that needs it (#4923),
198+
// and the wait shares the stage's remaining budget.
199+
if (clickedCount > 0 || (await readTimeOrigin()) !== originBefore) {
200+
await waitForPageReady(page, {
201+
timeoutMs: Math.min(15_000, budget.remainingMs?.() ?? 15_000),
202+
});
203+
}
180204
await page
181205
// fallow-ignore-next-line complexity
182206
.evaluate(() => {

0 commit comments

Comments
 (0)