Skip to content

Commit adc62f7

Browse files
cliffhallclaude
andcommitted
fix(fetch): treat a header-less response as HTML only when its body is
A response with no Content-Type was always treated as HTML, so JSON, plain text and other payloads went through readability and came back altered or as a simplification error. A header-less body is now simplified only when it opens with an HTML doctype or <html> tag (after any BOM and whitespace). application/xhtml+xml is recognized as HTML, and the media type is matched case-insensitively. Closes modelcontextprotocol#5134 Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent f72fdbd commit adc62f7

2 files changed

Lines changed: 154 additions & 1 deletion

File tree

‎src/fetch/src/mcp_server_fetch/server.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,18 @@ async def check_may_autonomously_fetch_url(
306306
)
307307

308308

309+
def _body_looks_like_html(page_raw: str) -> bool:
310+
"""Whether a body served with no Content-Type is an HTML document.
311+
312+
A missing header says nothing about the payload, so only a body that opens
313+
(after any byte-order mark and whitespace) with an HTML doctype or an
314+
``<html`` tag is simplified; JSON, plain text and the like come back raw
315+
rather than being forced through readability (#5134).
316+
"""
317+
start = page_raw.lstrip("\ufeff").lstrip()[:20].lower()
318+
return start.startswith(("<!doctype html", "<html"))
319+
320+
309321
async def fetch_url(
310322
url: str,
311323
user_agent: str,
@@ -361,7 +373,10 @@ async def _check_redirect_robots_txt(request: httpx.Request) -> None:
361373

362374
content_type = response.headers.get("content-type", "")
363375
is_page_html = (
364-
"<html" in page_raw[:100] or "text/html" in content_type or not content_type
376+
"<html" in page_raw[:100]
377+
or "text/html" in content_type.lower()
378+
or "application/xhtml+xml" in content_type.lower()
379+
or (not content_type and _body_looks_like_html(page_raw))
365380
)
366381

367382
if is_page_html and not force_raw:
Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
"""Protocol-level tests for responses served without a Content-Type (#5134).
2+
3+
A missing Content-Type header used to make every body count as HTML, so JSON,
4+
plain text and other payloads were pushed through readability and came back
5+
altered or as a simplification error. Now a header-less body is simplified
6+
only when it itself opens like an HTML document (an HTML doctype or an
7+
``<html`` tag, after any byte-order mark and whitespace); anything else comes
8+
back raw. These tests drive ``serve()`` through an MCP client with HTTP served
9+
by ``harness.FakeWeb``, and also pin the HTML media types that are still
10+
simplified whatever the body looks like.
11+
"""
12+
13+
from __future__ import annotations
14+
15+
import httpx
16+
import pytest
17+
from mcp.types import CallToolResult
18+
19+
from .harness import FakeWeb, connect
20+
21+
PAGE = "https://example.com/page"
22+
23+
ARTICLE_BODY = """<head><title>Test Page</title></head><body>
24+
<article>
25+
<h1>Hello World</h1>
26+
<p>This is a <a href="https://example.com/x">link</a> in a test paragraph
27+
with enough words to count as the main content of the page.</p>
28+
<h2>Sub heading</h2>
29+
<p>More text here.</p>
30+
</article>
31+
</body>"""
32+
33+
34+
def text_of(result: CallToolResult) -> str:
35+
data = result.model_dump(by_alias=True, mode="json", exclude_none=True)
36+
assert data["isError"] is False
37+
assert len(data["content"]) == 1
38+
assert data["content"][0]["type"] == "text"
39+
return data["content"][0]["text"]
40+
41+
42+
def headerless(body: str) -> httpx.Response:
43+
response = httpx.Response(200, content=body.encode())
44+
assert "content-type" not in response.headers
45+
return response
46+
47+
48+
async def fetch(web: FakeWeb, response: httpx.Response) -> str:
49+
web.add(PAGE, response)
50+
async with connect(ignore_robots_txt=True) as (session, _):
51+
result = await session.call_tool("fetch", {"url": PAGE})
52+
return text_of(result)
53+
54+
55+
def assert_raw(text: str, body: str) -> None:
56+
assert "Page failed to be simplified" not in text
57+
assert text.endswith(f"Contents of {PAGE}:\n{body}")
58+
assert "cannot be simplified to markdown, but here is the raw content:" in text
59+
60+
61+
def assert_simplified(text: str) -> None:
62+
assert text.startswith(f"Contents of {PAGE}:\n")
63+
assert "[link](https://example.com/x)" in text
64+
assert "## Sub heading" in text
65+
assert "<article>" not in text
66+
67+
68+
@pytest.mark.parametrize(
69+
"body",
70+
[
71+
'{"key": "value", "snake_case": "*starred*"}',
72+
"[1, 2, 3]",
73+
"just some plain_text with *asterisks*\nand a second line",
74+
"a,b,c\n1,2,3\n",
75+
],
76+
ids=["json-object", "json-array", "plain-text", "csv"],
77+
)
78+
async def test_non_html_without_content_type_is_returned_raw(
79+
web: FakeWeb, body: str
80+
) -> None:
81+
assert_raw(await fetch(web, headerless(body)), body)
82+
83+
84+
async def test_empty_body_without_content_type_is_not_simplified(
85+
web: FakeWeb,
86+
) -> None:
87+
# An empty body used to reach readability and fail to simplify; now it is
88+
# raw content with nothing in it, which the tool reports as such.
89+
text = await fetch(web, headerless(""))
90+
assert "Page failed to be simplified" not in text
91+
assert "cannot be simplified to markdown, but here is the raw content:" in text
92+
assert text.endswith("<error>No more content available.</error>")
93+
94+
95+
@pytest.mark.usefixtures("node_readability")
96+
@pytest.mark.parametrize(
97+
"body",
98+
[
99+
f"<!DOCTYPE html>\n{ARTICLE_BODY}",
100+
f"<!doctype html>{ARTICLE_BODY}",
101+
f"\n <!DocType HTML>\n{ARTICLE_BODY}",
102+
# More leading whitespace than the 100-character "<html" window.
103+
f"{' ' * 120}<HTML>{ARTICLE_BODY}</HTML>",
104+
],
105+
ids=["doctype-upper", "doctype-lower", "bom-and-whitespace", "long-indent"],
106+
)
107+
async def test_html_without_content_type_is_still_simplified(
108+
web: FakeWeb, body: str
109+
) -> None:
110+
assert_simplified(await fetch(web, headerless(body)))
111+
112+
113+
@pytest.mark.usefixtures("node_readability")
114+
@pytest.mark.parametrize(
115+
"content_type",
116+
[
117+
"application/xhtml+xml",
118+
"application/xhtml+xml; charset=utf-8",
119+
"Text/HTML; charset=UTF-8",
120+
],
121+
)
122+
async def test_html_media_types_are_simplified(web: FakeWeb, content_type: str) -> None:
123+
# The body carries no "<html" tag, so only the header can make it HTML.
124+
response = httpx.Response(
125+
200, content=ARTICLE_BODY.encode(), headers={"content-type": content_type}
126+
)
127+
assert_simplified(await fetch(web, response))
128+
129+
130+
async def test_doctype_alone_does_not_override_a_declared_content_type(
131+
web: FakeWeb,
132+
) -> None:
133+
# Sniffing the doctype applies only when the header is missing.
134+
body = f"<!doctype html>{ARTICLE_BODY}"
135+
response = httpx.Response(
136+
200, content=body.encode(), headers={"content-type": "text/plain"}
137+
)
138+
assert_raw(await fetch(web, response), body)

0 commit comments

Comments
 (0)