Skip to content

Commit 8520fb2

Browse files
authored
Merge pull request #5304 from modelcontextprotocol/v2/fix/5158-fetch-empty-tree-indexerror
fix(fetch): report an empty-tree page as unsimplifiable under beautifulsoup4 4.13
2 parents 2f8c794 + b4c0c80 commit 8520fb2

2 files changed

Lines changed: 128 additions & 3 deletions

File tree

‎src/fetch/src/mcp_server_fetch/server.py‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -176,10 +176,19 @@ def extract_content_from_html(html: str) -> str:
176176
Returns:
177177
Simplified markdown version of the content
178178
"""
179-
ret = readabilipy.simple_json.simple_json_from_html_string(
180-
html, use_readability=True
181-
)
182179
failed = "<error>Page failed to be simplified from HTML</error>"
180+
try:
181+
ret = readabilipy.simple_json.simple_json_from_html_string(
182+
html, use_readability=True
183+
)
184+
except IndexError:
185+
# readabilipy's pure-Python extractor wraps its simplified tree with
186+
# `Tag.append`, which beautifulsoup4 4.13 implements as
187+
# `self.insert(...)[0]`. A page that simplifies to nothing (an SPA
188+
# shell, an empty body) inserts nothing, so that index raises (#5158).
189+
# Under beautifulsoup4 <= 4.12 the same page comes back as an empty
190+
# tree, reported below as this same failure.
191+
return failed
183192
if not ret["content"]:
184193
return failed
185194
content = markdownify.markdownify(

‎src/fetch/tests/test_empty_tree.py‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
"""Regression tests for #5158: pages that simplify to an empty tree under beautifulsoup4 4.13.
2+
3+
readabilipy's pure-Python extractor (the path taken without Node) finishes by
4+
wrapping its simplified tree in a ``<div>`` with ``Tag.append``. beautifulsoup4
5+
4.13 implements ``append`` as ``self.insert(...)[0]``, and appending an empty
6+
tree inserts nothing, so a page with no extractable content (an SPA shell, an
7+
empty body) raised ``IndexError`` out of the tool call instead of returning the
8+
server's "Page failed to be simplified" error.
9+
10+
The lockfile pins beautifulsoup4 4.12, where ``append`` returns nothing and the
11+
crash cannot happen, and pyproject.toml is not where this is fixed: users who
12+
install the server unpinned get 4.13. So ``bs4_413_append`` reproduces 4.13's
13+
behavior on the installed version, and the first test proves the emulation
14+
raises the crash the issue reports before the others rely on it.
15+
"""
16+
17+
from __future__ import annotations
18+
19+
from typing import Any
20+
21+
import httpx
22+
import pytest
23+
import readabilipy.simple_json
24+
from bs4 import BeautifulSoup
25+
from bs4.element import Tag
26+
from mcp.types import CallToolResult
27+
28+
from .harness import FakeWeb, connect
29+
30+
PAGE = "https://example.com/app"
31+
FAILED = "<error>Page failed to be simplified from HTML</error>"
32+
33+
# The shape of the page in the issue's reproduction: client-rendered, nothing
34+
# to extract. (A <title> would survive simplification as text, so it has none.)
35+
SPA_SHELL = """<!doctype html><html><head><meta charset="utf-8">
36+
<script type="module" src="/assets/index.js"></script></head>
37+
<body><div id="root"></div></body></html>"""
38+
39+
ARTICLE = """<html><head><title>Article</title></head><body><article>
40+
<h1>Heading</h1><p>A paragraph with enough words to be the page's content.</p>
41+
</article></body></html>"""
42+
43+
44+
@pytest.fixture
45+
def bs4_413_append(monkeypatch: pytest.MonkeyPatch) -> None:
46+
"""Make ``Tag.append`` index the elements it inserted, as beautifulsoup4 4.13 does."""
47+
original = Tag.append
48+
49+
def append(self: Tag, tag: Any) -> Any:
50+
# Appending a BeautifulSoup object inserts its children, so an empty
51+
# one inserts nothing, and 4.13's `insert(...)[0]` has nothing to index.
52+
inserted = list(tag.contents) if isinstance(tag, BeautifulSoup) else [tag]
53+
original(self, tag)
54+
return inserted[0]
55+
56+
monkeypatch.setattr(Tag, "append", append)
57+
58+
59+
def wire(result: CallToolResult) -> dict[str, Any]:
60+
# The result as it crosses the wire (2025-11-25 fields only).
61+
return result.model_dump(
62+
by_alias=True, mode="json", exclude_none=True, exclude_unset=True
63+
)
64+
65+
66+
def html(body: str) -> httpx.Response:
67+
return httpx.Response(
68+
200, content=body.encode(), headers={"content-type": "text/html"}
69+
)
70+
71+
72+
@pytest.mark.usefixtures("python_readability", "bs4_413_append")
73+
@pytest.mark.parametrize("page", [SPA_SHELL, "<html><body></body></html>", ""])
74+
def test_emulation_reproduces_the_reported_crash(page: str) -> None:
75+
with pytest.raises(IndexError, match="list index out of range"):
76+
readabilipy.simple_json.simple_json_from_html_string(page, use_readability=True)
77+
78+
79+
@pytest.mark.usefixtures("python_readability", "bs4_413_append")
80+
async def test_empty_tree_tool_call_reports_simplification_failure(
81+
web: FakeWeb,
82+
) -> None:
83+
web.add(PAGE, html(SPA_SHELL))
84+
async with connect(ignore_robots_txt=True) as (session, _):
85+
result = await session.call_tool("fetch", {"url": PAGE})
86+
assert wire(result) == {
87+
"content": [{"type": "text", "text": f"Contents of {PAGE}:\n{FAILED}"}],
88+
"isError": False,
89+
}
90+
91+
92+
@pytest.mark.usefixtures("python_readability", "bs4_413_append")
93+
async def test_empty_tree_prompt_reports_simplification_failure(
94+
web: FakeWeb,
95+
) -> None:
96+
web.add(PAGE, html(SPA_SHELL))
97+
async with connect() as (session, _):
98+
result = await session.get_prompt("fetch", {"url": PAGE})
99+
assert len(result.messages) == 1
100+
content = result.messages[0].content
101+
assert content.type == "text"
102+
assert content.text == FAILED
103+
104+
105+
@pytest.mark.usefixtures("python_readability", "bs4_413_append")
106+
async def test_page_with_content_still_extracts_under_bs4_413(web: FakeWeb) -> None:
107+
# Only the empty tree trips the index; a page with content is unaffected.
108+
web.add(PAGE, html(ARTICLE))
109+
async with connect(ignore_robots_txt=True) as (session, _):
110+
result = await session.call_tool("fetch", {"url": PAGE})
111+
data = wire(result)
112+
assert data["isError"] is False
113+
[content] = data["content"]
114+
assert content["type"] == "text"
115+
assert "# Heading" in content["text"]
116+
assert "A paragraph with enough words" in content["text"]

0 commit comments

Comments
 (0)