Skip to content

Commit 7e76f20

Browse files
committed
fix(deep-crawl): cap each BFS level to the remaining max_pages budget
Siblings in a level queue links before the level's own pages are all counted, so the next level could still exceed max_pages in a real batch crawl (83 pages for max_pages=50). Cut each level to the budget left before fetching it, in both batch and stream mode. Add tests that run the real batch and stream loops against a fake crawler.
1 parent 860072e commit 7e76f20

2 files changed

Lines changed: 97 additions & 1 deletion

File tree

‎crawl4ai/deep_crawling/bfs_strategy.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,12 @@ async def _arun_batch(
247247
break
248248

249249
next_level: List[Tuple[str, Optional[str]]] = []
250+
# Siblings in a level queue links before the level's own pages are all
251+
# counted, so cap the level to the budget left before fetching it.
252+
# max_pages defaults to infinity, which is not a valid slice index.
253+
remaining = self.max_pages - self._pages_crawled
254+
if remaining < len(current_level):
255+
current_level = current_level[:max(int(remaining), 0)]
250256
urls = [url for url, _ in current_level]
251257
# reversed() keeps first-parent-wins if a resumed level repeats a URL
252258
parents = dict(reversed(current_level))
@@ -337,6 +343,15 @@ async def _arun_stream(
337343
break
338344

339345
next_level: List[Tuple[str, Optional[str]]] = []
346+
# Siblings in a level queue links before the level's own pages are all
347+
# counted, so cap the level to the budget left before fetching it.
348+
# max_pages defaults to infinity, which is not a valid slice index.
349+
remaining = self.max_pages - self._pages_crawled
350+
if remaining < len(current_level):
351+
current_level = current_level[:max(int(remaining), 0)]
352+
if not current_level:
353+
self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
354+
break
340355
urls = [url for url, _ in current_level]
341356
# reversed() keeps first-parent-wins if a resumed level repeats a URL
342357
parents = dict(reversed(current_level))

‎tests/deep_crawling/test_bfs_max_pages_next_level.py‎

Lines changed: 82 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
"""max_pages must count URLs already queued for the next level (#2338)."""
1+
"""BFS deep crawl must not fetch more than max_pages pages (#2338)."""
2+
3+
from collections import Counter
24

35
import pytest
46

7+
from crawl4ai import CrawlerRunConfig
58
from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
69
from crawl4ai.models import CrawlResult
710

@@ -28,3 +31,81 @@ async def test_link_discovery_budget_counts_next_level():
2831
)
2932

3033
assert len(next_level) == 14
34+
35+
36+
# A home page that links to 35 pages, each of which links to 30 more.
37+
ROOT = "https://example.com/"
38+
39+
40+
def _tree_links(url):
41+
if url == ROOT:
42+
return [f"https://example.com/p{i}" for i in range(35)]
43+
if url.count("/") == 3:
44+
return [f"{url}/c{j}" for j in range(30)]
45+
return []
46+
47+
48+
# Every page links to a shared 10-link menu plus 3 pages of its own.
49+
MENU = [f"https://example.com/m{i}" for i in range(10)]
50+
51+
52+
def _menu_links(url):
53+
return MENU + [f"{url.rstrip('/')}/n{j}" for j in range(3)]
54+
55+
56+
class FakeCrawler:
57+
def __init__(self, links=_tree_links):
58+
self.links = links
59+
self.fetched = 0
60+
61+
def _results(self, urls):
62+
self.fetched += len(urls)
63+
return [_result(u, self.links(u)) for u in urls]
64+
65+
async def arun_many(self, urls, config):
66+
results = self._results(urls)
67+
if not config.stream:
68+
return results
69+
70+
async def gen():
71+
for r in results:
72+
yield r
73+
74+
return gen()
75+
76+
77+
@pytest.mark.asyncio
78+
async def test_batch_crawl_stops_at_max_pages():
79+
strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=50)
80+
crawler = FakeCrawler()
81+
results = await strategy._arun_batch(ROOT, crawler, CrawlerRunConfig())
82+
assert crawler.fetched == 50
83+
assert len(results) == 50
84+
depths = Counter(r.metadata["depth"] for r in results)
85+
assert depths == {0: 1, 1: 35, 2: 14}
86+
87+
88+
@pytest.mark.asyncio
89+
async def test_batch_crawl_stops_at_max_pages_shared_menu():
90+
strategy = BFSDeepCrawlStrategy(max_depth=3, max_pages=50)
91+
crawler = FakeCrawler(_menu_links)
92+
results = await strategy._arun_batch(ROOT, crawler, CrawlerRunConfig())
93+
assert crawler.fetched == 50
94+
assert len(results) == 50
95+
96+
97+
@pytest.mark.asyncio
98+
async def test_stream_crawl_fetches_at_most_max_pages():
99+
strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=50)
100+
crawler = FakeCrawler()
101+
config = CrawlerRunConfig(stream=True)
102+
[r async for r in strategy._arun_stream(ROOT, crawler, config)]
103+
assert crawler.fetched == 50
104+
105+
106+
@pytest.mark.asyncio
107+
async def test_batch_crawl_without_max_pages_crawls_everything():
108+
strategy = BFSDeepCrawlStrategy(max_depth=2)
109+
crawler = FakeCrawler()
110+
results = await strategy._arun_batch(ROOT, crawler, CrawlerRunConfig())
111+
assert len(results) == 1 + 35 + 35 * 30

0 commit comments

Comments
 (0)