Skip to content

Commit 0f09091

Browse files
authored
Merge pull request #5302 from modelcontextprotocol/v2/fix/5199-fetch-html-charset
fix(fetch): decode a page by the charset its HTML declares
2 parents b8260a0 + 72891bd commit 0f09091

2 files changed

Lines changed: 327 additions & 1 deletion

File tree

‎src/fetch/src/mcp_server_fetch/server.py‎

Lines changed: 88 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,93 @@ def make_client(proxy_url: str | None) -> httpx.AsyncClient:
271271
)
272272

273273

274+
def decode_page(response: httpx.Response) -> str:
275+
"""Decode a fetched body, honoring a charset the HTML declares (#5199).
276+
277+
httpx's ``response.text`` uses the Content-Type header's charset, and
278+
UTF-8 without one, so a page that declares a legacy encoding only in a
279+
``<meta charset>`` or ``<meta http-equiv="Content-Type">`` tag came back
280+
garbled. Following the HTML encoding-sniffing order, kept simple: a byte
281+
order mark or a header charset wins (httpx already honors the header), and
282+
otherwise the first 1024 bytes of a ``text/html`` or
283+
``application/xhtml+xml`` (or untyped) body are parsed for the first
284+
``<meta>`` that declares one: a ``charset`` attribute, or a ``content``
285+
attribute's ``charset=`` when ``http-equiv`` is ``Content-Type``. The
286+
stdlib HTML parser does the tokenizing, so a commented-out tag or an
287+
unrelated attribute (``data-charset``) is not mistaken for a declaration.
288+
A declaration Python does not know, or one that names a codec that is not
289+
a web text encoding, is ignored, and the body is decoded as before.
290+
"""
291+
# Imported here so the helper stays one self-contained block of this file.
292+
import codecs
293+
import re
294+
from html.parser import HTMLParser
295+
296+
content = response.content
297+
media_type = (
298+
response.headers.get("content-type", "").split(";", 1)[0].strip().lower()
299+
)
300+
if (
301+
response.charset_encoding is not None
302+
or media_type not in ("", "text/html", "application/xhtml+xml")
303+
or content.startswith(
304+
(codecs.BOM_UTF8, codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)
305+
)
306+
):
307+
return response.text
308+
309+
class MetaCharset(HTMLParser):
310+
declared: str | None = None
311+
312+
def handle_starttag(
313+
self, tag: str, attrs: list[tuple[str, str | None]]
314+
) -> None:
315+
if tag != "meta" or self.declared is not None:
316+
return
317+
values = {name: (value or "").strip() for name, value in attrs}
318+
if values.get("charset"):
319+
self.declared = values["charset"]
320+
elif values.get("http-equiv", "").lower() == "content-type":
321+
found = re.search(
322+
r"""charset\s*=\s*["']?\s*([^\s"';]+)""",
323+
values.get("content", ""),
324+
re.IGNORECASE,
325+
)
326+
if found:
327+
self.declared = found.group(1)
328+
329+
# Latin-1 maps every byte to one character, so an ASCII declaration reads
330+
# the same whatever the page's real encoding. A tag cut off at byte 1024
331+
# is left unparsed, since the parser is never closed.
332+
scan = MetaCharset()
333+
scan.feed(content[:1024].decode("latin-1"))
334+
if scan.declared is None:
335+
return response.text
336+
try:
337+
name = codecs.lookup(scan.declared).name
338+
# A UTF-16/32 declaration in an ASCII-compatible document cannot be
339+
# true, so the HTML standard reads it as UTF-8.
340+
if name.startswith(("utf-16", "utf-32")):
341+
name = "utf-8"
342+
# Python codecs that are not web text encodings: decoding a page with
343+
# them would mangle it rather than fix it.
344+
if name in ("idna", "punycode", "unicode-escape", "raw-unicode-escape"):
345+
return response.text
346+
# A declaration found in ASCII bytes is only believable for an
347+
# ASCII-compatible encoding. UTF-7, HZ and the EBCDIC code pages
348+
# (cp037, cp500, …) would rewrite the markup itself, so they fail this
349+
# probe (by a different result or a decode error) and are ignored.
350+
probe = b"\t\n\r" + bytes(range(0x20, 0x7F))
351+
if probe.decode(name) != probe.decode("ascii"):
352+
return response.text
353+
return content.decode(name, errors="replace")
354+
except (LookupError, ValueError): # UnicodeError is a ValueError
355+
# An unknown charset, a name Python refuses outright (one holding a
356+
# NUL), a non-text codec (base64, rot13, …), one that refuses to
357+
# decode at all (``undefined``), or one the probe cannot decode.
358+
return response.text
359+
360+
274361
async def check_may_autonomously_fetch_url(
275362
url: str,
276363
user_agent: str,
@@ -386,7 +473,7 @@ async def _check_redirect_robots_txt(request: httpx.Request) -> None:
386473
f"Failed to fetch {url} - status code {response.status_code}"
387474
)
388475

389-
page_raw = response.text
476+
page_raw = decode_page(response)
390477

391478
content_type = response.headers.get("content-type", "")
392479
is_page_html = (
Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
"""Protocol-level tests for decoding a page by the charset its HTML declares (#5199).
2+
3+
httpx decodes a body by the Content-Type header's charset and falls back to
4+
UTF-8 without one, so a legacy-encoded page that names its charset only in a
5+
``<meta charset>`` or ``<meta http-equiv="Content-Type">`` tag came back
6+
garbled. ``decode_page`` now scans the head of an HTML (or untyped) body for
7+
that declaration when the header names none. These tests drive the real
8+
server through an MCP client with HTTP served by ``harness.FakeWeb``, and pin
9+
both the fix and its edges: the header and a byte order mark still win, a
10+
declaration on a non-HTML body or past the first 1024 bytes is ignored, and an
11+
unknown or unusable declared charset falls back to the old decoding.
12+
13+
Most cases fetch with ``raw: true`` so the decoded text reaches the client
14+
untouched by HTML extraction. The one case that goes through extraction, to
15+
show the Markdown is right too, forces readabilipy's pure-Python path: the
16+
Readability.js path writes the page to a temp file in the locale encoding,
17+
which cannot hold Japanese on Windows.
18+
"""
19+
20+
from __future__ import annotations
21+
22+
from typing import Any
23+
24+
import httpx
25+
import pytest
26+
from mcp import ClientSession
27+
from mcp.types import CallToolResult
28+
29+
from .harness import FakeWeb, connect
30+
31+
PAGE = "https://example.com/page"
32+
ROBOTS = "https://example.com/robots.txt"
33+
34+
JAPANESE = "日本語のページです"
35+
RUSSIAN = "Привет, мир"
36+
KOREAN = "안녕하세요 세계"
37+
CHINESE = "你好,世界"
38+
39+
40+
def text_of(result: CallToolResult) -> str:
41+
assert not result.is_error, result
42+
data = result.model_dump(by_alias=True, mode="json", exclude_none=True)
43+
assert len(data["content"]) == 1
44+
return data["content"][0]["text"]
45+
46+
47+
def page(body: bytes, content_type: str | None = "text/html") -> httpx.Response:
48+
headers = {} if content_type is None else {"content-type": content_type}
49+
return httpx.Response(200, content=body, headers=headers)
50+
51+
52+
async def fetch(web: FakeWeb, response: httpx.Response, **arguments: Any) -> str:
53+
web.add(ROBOTS, httpx.Response(404))
54+
web.add(PAGE, response)
55+
async with connect() as (session, _):
56+
assert isinstance(session, ClientSession)
57+
result = await session.call_tool("fetch", {"url": PAGE, **arguments})
58+
return text_of(result)
59+
60+
61+
def document(meta: str, text: str) -> str:
62+
return f"<html><head>{meta}<title>t</title></head><body><p>{text}</p></body></html>"
63+
64+
65+
@pytest.mark.parametrize(
66+
("meta", "encoding", "text"),
67+
[
68+
('<meta charset="Shift_JIS">', "shift_jis", JAPANESE),
69+
("<meta charset=windows-1251>", "cp1251", RUSSIAN),
70+
("<META CHARSET='euc-kr'>", "euc_kr", KOREAN),
71+
('<meta charset="GBK" />', "gbk", CHINESE),
72+
(
73+
'<meta http-equiv="Content-Type" content="text/html; charset=Shift_JIS">',
74+
"shift_jis",
75+
JAPANESE,
76+
),
77+
(
78+
"<meta http-equiv='content-type' content='text/html;charset=windows-1251'>",
79+
"cp1251",
80+
RUSSIAN,
81+
),
82+
],
83+
)
84+
async def test_a_charset_declared_only_in_the_html_is_honored(
85+
web: FakeWeb, meta: str, encoding: str, text: str
86+
) -> None:
87+
body = document(meta, text).encode(encoding)
88+
assert text in await fetch(web, page(body), raw=True)
89+
90+
91+
async def test_a_declared_charset_is_honored_without_a_content_type_header(
92+
web: FakeWeb,
93+
) -> None:
94+
body = document('<meta charset="Shift_JIS">', JAPANESE).encode("shift_jis")
95+
assert JAPANESE in await fetch(web, page(body, content_type=None), raw=True)
96+
97+
98+
async def test_xhtml_content_type_is_scanned(web: FakeWeb) -> None:
99+
body = document('<meta charset="windows-1251">', RUSSIAN).encode("cp1251")
100+
response = page(body, content_type="application/xhtml+xml")
101+
assert RUSSIAN in await fetch(web, response, raw=True)
102+
103+
104+
async def test_the_extracted_markdown_is_decoded_too(
105+
web: FakeWeb, python_readability: None
106+
) -> None:
107+
body = document('<meta charset="Shift_JIS">', JAPANESE).encode("shift_jis")
108+
assert JAPANESE in await fetch(web, page(body))
109+
110+
111+
async def test_a_header_charset_wins_over_the_html_declaration(web: FakeWeb) -> None:
112+
# The page lies in its meta tag; the header is right, and wins.
113+
body = document('<meta charset="Shift_JIS">', RUSSIAN).encode("cp1251")
114+
response = page(body, content_type="text/html; charset=windows-1251")
115+
assert RUSSIAN in await fetch(web, response, raw=True)
116+
117+
118+
async def test_a_byte_order_mark_wins_over_the_html_declaration(
119+
web: FakeWeb,
120+
) -> None:
121+
body = b"\xef\xbb\xbf" + document('<meta charset="Shift_JIS">', JAPANESE).encode(
122+
"utf-8"
123+
)
124+
assert JAPANESE in await fetch(web, page(body), raw=True)
125+
126+
127+
async def test_a_declaration_on_a_non_html_body_is_ignored(web: FakeWeb) -> None:
128+
# A text/plain body is UTF-8 by default; a meta-like string in it is text.
129+
body = document('<meta charset="windows-1251">', JAPANESE).encode("utf-8")
130+
# raw: the body opens with "<html", so without it the server would hand it
131+
# to Readability.js, whose temp-file write fails on non-cp1252 text on
132+
# Windows (pre-existing, not what this test is about).
133+
response = page(body, content_type="text/plain")
134+
assert JAPANESE in await fetch(web, response, raw=True)
135+
136+
137+
async def test_a_text_plain_type_with_an_html_parameter_is_not_scanned(
138+
web: FakeWeb,
139+
) -> None:
140+
# #5302 review: only the media type counts, not "html" in a parameter.
141+
body = document('<meta charset="windows-1251">', JAPANESE).encode("utf-8")
142+
response = page(body, content_type='text/plain; name="page.html"')
143+
assert JAPANESE in await fetch(web, response, raw=True)
144+
145+
146+
@pytest.mark.parametrize(
147+
"head",
148+
[
149+
# #5302 review: a commented-out tag declares nothing.
150+
'<!-- <meta charset="windows-1251"> --><meta charset="utf-8">',
151+
"<!-- <meta charset=windows-1251> -->",
152+
# Nor does an attribute that only contains the word.
153+
'<meta data-charset="windows-1251" charset="utf-8">',
154+
'<meta data-charset="windows-1251">',
155+
# A content attribute counts only under http-equiv="Content-Type".
156+
'<meta name="description" content="text/html; charset=windows-1251">',
157+
'<meta http-equiv="refresh" content="5; charset=windows-1251">',
158+
# An empty declaration, and one with no charset in it.
159+
'<meta charset="">',
160+
'<meta http-equiv="Content-Type" content="text/html">',
161+
# A declaration Python refuses outright.
162+
'<meta charset="utf\x008">',
163+
# The first declaration wins.
164+
'<meta charset="utf-8"><meta charset="windows-1251">',
165+
],
166+
)
167+
async def test_text_that_is_not_a_declaration_leaves_utf8_alone(
168+
web: FakeWeb, head: str
169+
) -> None:
170+
body = document(head, JAPANESE).encode("utf-8")
171+
assert JAPANESE in await fetch(web, page(body), raw=True)
172+
173+
174+
async def test_a_declaration_after_other_tags_and_comments_is_found(
175+
web: FakeWeb,
176+
) -> None:
177+
head = (
178+
'<!-- <meta charset="utf-8"> --><link rel="icon" href="/i.png">'
179+
'<meta name="viewport" content="width=device-width"><meta charset=Shift_JIS>'
180+
)
181+
body = document(head, JAPANESE).encode("shift_jis")
182+
assert JAPANESE in await fetch(web, page(body), raw=True)
183+
184+
185+
async def test_a_declaration_past_the_first_1024_bytes_is_ignored(
186+
web: FakeWeb,
187+
) -> None:
188+
padding = "<!-- " + "x" * 1100 + " -->"
189+
body = (
190+
f"<html><head>{padding}<meta charset=windows-1251></head>"
191+
f"<body><p>{JAPANESE}</p></body></html>"
192+
).encode("utf-8")
193+
assert JAPANESE in await fetch(web, page(body), raw=True)
194+
195+
196+
async def test_a_page_with_no_declaration_is_decoded_as_utf8(web: FakeWeb) -> None:
197+
body = document("", JAPANESE).encode("utf-8")
198+
assert JAPANESE in await fetch(web, page(body), raw=True)
199+
200+
201+
@pytest.mark.parametrize(
202+
"declared",
203+
[
204+
# Unknown to Python.
205+
"no-such-charset",
206+
"x-user-defined",
207+
# Known codecs that are not text encodings, or refuse to decode.
208+
"base64",
209+
"rot13",
210+
"undefined",
211+
# Text codecs that are not web encodings.
212+
"unicode_escape",
213+
"raw_unicode_escape",
214+
"idna",
215+
"punycode",
216+
# #5302 review: encodings that are not ASCII-compatible, so could not
217+
# have been declared in ASCII. UTF-7 would turn "+AEI-" into "B";
218+
# the EBCDIC pages would rewrite the markup.
219+
"utf-7",
220+
"cp037",
221+
"cp1140",
222+
"hz",
223+
],
224+
)
225+
async def test_an_unusable_declared_charset_falls_back_to_the_old_decoding(
226+
web: FakeWeb, declared: str
227+
) -> None:
228+
marker = JAPANESE + " \\u0041 +AEI-"
229+
body = document(f'<meta charset="{declared}">', marker).encode("utf-8")
230+
text = await fetch(web, page(body), raw=True)
231+
assert f"<p>{marker}</p>" in text
232+
233+
234+
@pytest.mark.parametrize("declared", ["utf-16", "UTF-16LE", "utf-32"])
235+
async def test_a_utf16_or_utf32_declaration_is_read_as_utf8(
236+
web: FakeWeb, declared: str
237+
) -> None:
238+
body = document(f'<meta charset="{declared}">', JAPANESE).encode("utf-8")
239+
assert JAPANESE in await fetch(web, page(body), raw=True)

0 commit comments

Comments
 (0)