Skip to content

Commit fb31aa5

Browse files
cliffhallclaude
andcommitted
fix(fetch): declare beautifulsoup4 and name a missing Content-Type in the raw prefix
server.py imports bs4 directly since #5308, but beautifulsoup4 reached the install only through readabilipy. Declare it (>=4.7.1, readabilipy's own floor); uv.lock gains the metadata only, no version moves. A raw response with no Content-Type was prefixed "Content type cannot be simplified", an empty slot with a double space. It now reads "Content type (not declared) ...". Closes #5321 Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent 755b188 commit fb31aa5

4 files changed

Lines changed: 73 additions & 1 deletion

File tree

‎src/fetch/pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ classifiers = [
1616
"Programming Language :: Python :: 3.10",
1717
]
1818
dependencies = [
19+
"beautifulsoup4>=4.7.1",
1920
"httpx[socks]>=0.27",
2021
"markdownify>=0.13.1",
2122
"mcp>=2.3,<3",

‎src/fetch/src/mcp_server_fetch/server.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -584,9 +584,11 @@ async def _check_redirect_robots_txt(request: httpx.Request) -> None:
584584
if is_page_html and not force_raw:
585585
return extract_content_from_html(page_raw, use_readability_js), ""
586586

587+
# A response with no Content-Type would otherwise leave an empty slot here.
588+
declared_type = content_type or "(not declared)"
587589
return (
588590
page_raw,
589-
f"Content type {content_type} cannot be simplified to markdown, but here is the raw content:\n",
591+
f"Content type {declared_type} cannot be simplified to markdown, but here is the raw content:\n",
590592
)
591593

592594

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
"""Tests for the raw-content prefix and the declared dependencies (#5321).
2+
3+
A body fetch returns raw names its Content-Type in the prefix it adds. A
4+
response with no Content-Type left that slot empty ("Content type cannot be
5+
simplified"), so the prefix now says the type was not declared. The prefix tests
6+
drive ``serve()`` through an MCP client with HTTP served by ``harness.FakeWeb``.
7+
8+
``server.py`` imports ``bs4`` directly, so ``beautifulsoup4`` must be a declared
9+
dependency of the package rather than one that only arrives through
10+
readabilipy; the last test reads the installed package's metadata to pin that.
11+
"""
12+
13+
from __future__ import annotations
14+
15+
import re
16+
from importlib.metadata import requires
17+
18+
import httpx
19+
20+
from .harness import FakeWeb, connect
21+
22+
PAGE = "https://example.com/data"
23+
24+
25+
async def fetch_text(web: FakeWeb, response: httpx.Response) -> str:
26+
web.add(PAGE, response)
27+
async with connect(ignore_robots_txt=True) as (session, _):
28+
result = await session.call_tool("fetch", {"url": PAGE})
29+
data = result.model_dump(by_alias=True, mode="json", exclude_none=True)
30+
assert data["isError"] is False
31+
return data["content"][0]["text"]
32+
33+
34+
async def test_a_missing_content_type_is_named_as_not_declared(
35+
web: FakeWeb,
36+
) -> None:
37+
response = httpx.Response(200, content=b'{"key": "value"}')
38+
assert "content-type" not in response.headers
39+
40+
text = await fetch_text(web, response)
41+
42+
assert text.startswith(
43+
"Content type (not declared) cannot be simplified to markdown, "
44+
"but here is the raw content:\n"
45+
)
46+
assert "Content type " not in text
47+
48+
49+
async def test_a_declared_content_type_is_still_named(web: FakeWeb) -> None:
50+
response = httpx.Response(
51+
200, content=b"a,b\n1,2\n", headers={"content-type": "text/csv"}
52+
)
53+
54+
text = await fetch_text(web, response)
55+
56+
assert text.startswith(
57+
"Content type text/csv cannot be simplified to markdown, "
58+
"but here is the raw content:\n"
59+
)
60+
61+
62+
def test_beautifulsoup4_is_a_declared_dependency() -> None:
63+
names = {
64+
re.split(r"[\s\[<>=!~;]", requirement, maxsplit=1)[0].lower()
65+
for requirement in requires("mcp-server-fetch") or []
66+
}
67+
assert "beautifulsoup4" in names

‎src/fetch/uv.lock‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)