@@ -43,7 +43,11 @@ import type { VisionCaptionOutcome } from "./contentExtractor.js";
4343import { loadEnvFile , generateProjectScaffold } from "./scaffolding.js" ;
4444import { detectBlockedPage } from "./pageBlockDetection.js" ;
4545import { navigateForCapture } from "./navigateForCapture.js" ;
46- import { captureProtocolTimeoutMs , isProtocolEvaluateTimeoutError } from "./captureTimeout.js" ;
46+ import {
47+ captureProtocolTimeoutMs ,
48+ isDegradableEvaluateTimeoutError ,
49+ withRemainingBudget ,
50+ } from "./captureTimeout.js" ;
4751import { lazyScrollForCapture } from "./lazyScrollForCapture.js" ;
4852import type { CaptureOptions , CapturePhase , CapturePhaseProgress , CaptureResult } from "./types.js" ;
4953
@@ -269,43 +273,42 @@ export async function captureWebsite(
269273 } ;
270274 let contentCheckTimedOut = false ;
271275 try {
272- pageContentCheck = ( await page1 . evaluate ( `(() => {
276+ pageContentCheck = ( await withRemainingBudget (
277+ page1 . evaluate ( `(() => {
278+ var text = (document.body && document.body.innerText || "").trim();
273279 var title = document.title || "";
274- var body = document.body;
275- var bodyChildCount = body ? body.children.length : 0;
276- var text = "";
277- if (body) {
278- var walk = document.createTreeWalker(body, NodeFilter.SHOW_TEXT, null);
279- var node;
280- var guard = 0;
281- while ((node = walk.nextNode()) && text.length < 800 && guard < 300) {
282- guard++;
283- text += node.nodeValue || "";
284- }
285- }
286280 var hasCfTurnstile = !!document.querySelector('.cf-turnstile, [data-sitekey], iframe[src*="challenges.cloudflare.com"], #challenge-running, #challenge-form');
287- return { textLength: text.trim().length, title: title, hasChallengeElement: hasCfTurnstile, bodyChildCount: bodyChildCount };
288- })()` ) ) as typeof pageContentCheck ;
281+ var bodyChildCount = document.body ? document.body.children.length : 0;
282+ return { textLength: text.length, title: title, hasChallengeElement: hasCfTurnstile, bodyChildCount: bodyChildCount };
283+ })()` ) ,
284+ Math . min ( 5_000 , remainingMs ( ) ) ,
285+ "content-check" ,
286+ ) ) as typeof pageContentCheck ;
289287 } catch ( err ) {
290- if ( ! isProtocolEvaluateTimeoutError ( err ) ) {
288+ if ( ! isDegradableEvaluateTimeoutError ( err ) ) {
291289 throw err ;
292290 }
293291 contentCheckTimedOut = true ;
294292 const message =
295- "post-navigation content check timed out; skipping blocked-page detection and continuing " ;
293+ "post-navigation content check timed out; continuing with HTTP-status blocked-page detection only " ;
296294 warnings . push ( message ) ;
297295 progress ( "warn" , message ) ;
298296 }
299297
300- if ( ! contentCheckTimedOut ) {
301- const blockedReason = detectBlockedPage ( {
302- httpStatus : navigationResponse ?. status ( ) ?? null ,
303- ...pageContentCheck ,
304- } ) ;
305- if ( blockedReason ) {
306- phase ( "navigation" , "degraded" , "blocked" ) ;
307- throw new Error ( blockedReason ) ;
308- }
298+ const blockedReason = detectBlockedPage ( {
299+ httpStatus : navigationResponse ?. status ( ) ?? null ,
300+ ...( contentCheckTimedOut
301+ ? {
302+ title : "" ,
303+ textLength : 0 ,
304+ bodyChildCount : 0 ,
305+ hasChallengeElement : false ,
306+ }
307+ : pageContentCheck ) ,
308+ } ) ;
309+ if ( blockedReason ) {
310+ phase ( "navigation" , "degraded" , "blocked" ) ;
311+ throw new Error ( blockedReason ) ;
309312 }
310313
311314 phase ( "navigation" , "completed" ) ;
@@ -440,11 +443,19 @@ export async function captureWebsite(
440443
441444 progress ( "animations" , "Cataloging animations..." ) ;
442445 try {
443- animationCatalog = await collectAnimationCatalog ( page1 , cdpAnims , cdp , {
446+ const animationOutcome = await collectAnimationCatalog ( page1 , cdpAnims , cdp , {
444447 scrollBudgetMs : Math . min ( 8_000 , remainingMs ( ) ) ,
448+ evaluateBudgetMs : Math . min ( 15_000 , remainingMs ( ) ) ,
445449 } ) ;
450+ animationCatalog = animationOutcome . catalog ;
451+ if ( animationOutcome . timedOut ) {
452+ const message =
453+ "animation catalog evaluate timed out; continuing without animation catalog" ;
454+ warnings . push ( message ) ;
455+ progress ( "warn" , message ) ;
456+ }
446457 } catch ( err ) {
447- if ( ! isProtocolEvaluateTimeoutError ( err ) ) {
458+ if ( ! isDegradableEvaluateTimeoutError ( err ) ) {
448459 throw err ;
449460 }
450461 const message = "animation catalog evaluate timed out; continuing without animation catalog" ;
@@ -464,7 +475,7 @@ export async function captureWebsite(
464475 screenshots = await captureScrollScreenshots ( page1 , outputDir , { remainingMs } ) ;
465476 progress ( "screenshots" , `${ screenshots . length } scroll screenshots captured` ) ;
466477 } catch ( err ) {
467- if ( ! isProtocolEvaluateTimeoutError ( err ) ) {
478+ if ( ! isDegradableEvaluateTimeoutError ( err ) ) {
468479 throw err ;
469480 }
470481 const message = "scroll screenshots timed out; continuing without screenshots" ;
0 commit comments