Skip to content

Commit 5bbe6f2

Browse files
cliffhallclaude
andcommitted
fix(fetch): require readabilipy 0.3.0 so extraction works on non-cp1252 pages on Windows
readabilipy 0.2.0's Readability.js path wrote the page to a temp file in the locale encoding, so on Windows (cp1252) any page with CJK text, a BOM or another character outside the code page crashed with UnicodeEncodeError. 0.3.0 writes and reads its temp files as UTF-8. Raise the floor to >=0.3.0 and relock (only readabilipy changes), drop the win32 skip on the BOM case in test_missing_content_type.py, and add a protocol-level test with Japanese text and a leading BOM. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent 0c8c564 commit 5bbe6f2

4 files changed

Lines changed: 87 additions & 19 deletions

File tree

‎src/fetch/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ dependencies = [
2121
"mcp>=2.3,<3",
2222
"protego>=0.6.2",
2323
"pydantic>=2.0.0",
24-
"readabilipy>=0.2.0",
24+
"readabilipy>=0.3.0",
2525
"requests>=2.32.3",
2626
]
2727

‎src/fetch/tests/test_missing_content_type.py‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@
1212

1313
from __future__ import annotations
1414

15-
import sys
16-
1715
import httpx
1816
import pytest
1917
from mcp.types import CallToolResult
@@ -104,16 +102,7 @@ async def test_empty_body_without_content_type_is_not_simplified(
104102
pytest.param(f"<!DOCTYPE html>\n{ARTICLE_BODY}", id="doctype-upper"),
105103
pytest.param(f"<!doctype html>{ARTICLE_BODY}", id="doctype-lower"),
106104
pytest.param(
107-
f"{BOM}\n <!DocType HTML>\n{ARTICLE_BODY}",
108-
id="bom-and-whitespace",
109-
# readabilipy 0.2.0 writes the page to a temp file in the locale
110-
# encoding, which cannot hold a BOM on Windows (cp1252). The
111-
# detection itself is covered on every platform by
112-
# test_body_detection_skips_bom_and_whitespace below.
113-
marks=pytest.mark.skipif(
114-
sys.platform == "win32",
115-
reason="readabilipy 0.2.0 cannot extract non-cp1252 pages on Windows (#5303)",
116-
),
105+
f"{BOM}\n <!DocType HTML>\n{ARTICLE_BODY}", id="bom-and-whitespace"
117106
),
118107
# More leading whitespace than the 100-character "<html" window.
119108
pytest.param(f"{' ' * 120}<HTML>{ARTICLE_BODY}</HTML>", id="long-indent"),
@@ -173,6 +162,5 @@ async def test_doctype_alone_does_not_override_a_declared_content_type(
173162
],
174163
)
175164
def test_body_detection_skips_bom_and_whitespace(body: str, expected: bool) -> None:
176-
# Runs on every platform: what #5134 decides, without readabilipy's
177-
# extraction (which cannot handle a BOM on Windows, #5303).
165+
# What #5134 decides, without readabilipy's extraction.
178166
assert _body_looks_like_html(body) is expected
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
"""Protocol-level tests for pages whose text has no cp1252 encoding (#5303).
2+
3+
readabilipy 0.2.0's Readability.js path wrote the page to a temp file opened
4+
in the locale encoding, so on a Western-locale Windows (cp1252) any page
5+
holding a character outside that code page (CJK text, a leading byte-order
6+
mark, U+2010 HYPHEN) crashed extraction with ``UnicodeEncodeError``. readabilipy
7+
0.3.0 writes and reads its temp files as UTF-8, and pyproject.toml now requires
8+
it. On Linux and macOS the locale is already UTF-8, so these tests pass there
9+
either way; the "Test fetch (Windows)" CI leg is where they prove the fix.
10+
11+
Each test drives ``serve()`` through an MCP client, with the page served by
12+
``harness.FakeWeb``, on the Readability.js path (the one that crashed).
13+
"""
14+
15+
from __future__ import annotations
16+
17+
import httpx
18+
import pytest
19+
from mcp.types import CallToolResult
20+
21+
from .harness import FakeWeb, connect
22+
23+
PAGE = "https://example.jp/article"
24+
BOM = ""
25+
26+
# None of these characters exists in cp1252.
27+
HEADING = "日本語の見出し"
28+
PARAGRAPH = (
29+
"これは本文の段落です。読みやすさの抽出が主な内容として選ぶのに"
30+
"十分な長さの文章をここに書いています。"
31+
)
32+
HYPHENATED = "non‐breaking"
33+
34+
ARTICLE = f"""<!doctype html>
35+
<html lang="ja"><head><meta charset="utf-8"><title>{HEADING}</title></head>
36+
<body><article>
37+
<h1>{HEADING}</h1>
38+
<p>{PARAGRAPH}</p>
39+
<p>A <a href="https://example.jp/x">link</a> and a {HYPHENATED} word.</p>
40+
</article></body></html>"""
41+
42+
43+
def text_of(result: CallToolResult) -> str:
44+
data = result.model_dump(by_alias=True, mode="json", exclude_none=True)
45+
assert data["isError"] is False
46+
assert len(data["content"]) == 1
47+
assert data["content"][0]["type"] == "text"
48+
return data["content"][0]["text"]
49+
50+
51+
async def fetch(web: FakeWeb, response: httpx.Response) -> str:
52+
web.add(PAGE, response)
53+
async with connect(ignore_robots_txt=True) as (session, _):
54+
result = await session.call_tool("fetch", {"url": PAGE})
55+
return text_of(result)
56+
57+
58+
@pytest.mark.usefixtures("node_readability")
59+
@pytest.mark.parametrize(
60+
"body",
61+
[
62+
pytest.param(ARTICLE, id="japanese"),
63+
pytest.param(f"{BOM}{ARTICLE}", id="japanese-with-bom"),
64+
],
65+
)
66+
async def test_non_cp1252_page_is_simplified(web: FakeWeb, body: str) -> None:
67+
response = httpx.Response(
68+
200,
69+
content=body.encode("utf-8"),
70+
headers={"content-type": "text/html; charset=utf-8"},
71+
)
72+
text = await fetch(web, response)
73+
74+
assert "Page failed to be simplified" not in text
75+
assert text.startswith(f"Contents of {PAGE}:\n")
76+
# The non-cp1252 text came through Readability.js intact, as Markdown.
77+
assert PARAGRAPH in text
78+
assert HYPHENATED in text
79+
assert "[link](https://example.jp/x)" in text
80+
assert "<article>" not in text

‎src/fetch/uv.lock‎

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)