|
| 1 | +"""Protocol-level tests for decoding a page by the charset its HTML declares (#5199). |
| 2 | +
|
| 3 | +httpx decodes a body by the Content-Type header's charset and falls back to |
| 4 | +UTF-8 without one, so a legacy-encoded page that names its charset only in a |
| 5 | +``<meta charset>`` or ``<meta http-equiv="Content-Type">`` tag came back |
| 6 | +garbled. ``decode_page`` now scans the head of an HTML (or untyped) body for |
| 7 | +that declaration when the header names none. These tests drive the real |
| 8 | +server through an MCP client with HTTP served by ``harness.FakeWeb``, and pin |
| 9 | +both the fix and its edges: the header and a byte order mark still win, a |
| 10 | +declaration on a non-HTML body or past the first 1024 bytes is ignored, and an |
| 11 | +unknown or unusable declared charset falls back to the old decoding. |
| 12 | +
|
| 13 | +Most cases fetch with ``raw: true`` so the decoded text reaches the client |
| 14 | +untouched by HTML extraction. The one case that goes through extraction, to |
| 15 | +show the Markdown is right too, forces readabilipy's pure-Python path: the |
| 16 | +Readability.js path writes the page to a temp file in the locale encoding, |
| 17 | +which cannot hold Japanese on Windows. |
| 18 | +""" |
| 19 | + |
| 20 | +from __future__ import annotations |
| 21 | + |
| 22 | +from typing import Any |
| 23 | + |
| 24 | +import httpx |
| 25 | +import pytest |
| 26 | +from mcp import ClientSession |
| 27 | +from mcp.types import CallToolResult |
| 28 | + |
| 29 | +from .harness import FakeWeb, connect |
| 30 | + |
| 31 | +PAGE = "https://example.com/page" |
| 32 | +ROBOTS = "https://example.com/robots.txt" |
| 33 | + |
| 34 | +JAPANESE = "日本語のページです" |
| 35 | +RUSSIAN = "Привет, мир" |
| 36 | +KOREAN = "안녕하세요 세계" |
| 37 | +CHINESE = "你好,世界" |
| 38 | + |
| 39 | + |
| 40 | +def text_of(result: CallToolResult) -> str: |
| 41 | + assert not result.is_error, result |
| 42 | + data = result.model_dump(by_alias=True, mode="json", exclude_none=True) |
| 43 | + assert len(data["content"]) == 1 |
| 44 | + return data["content"][0]["text"] |
| 45 | + |
| 46 | + |
| 47 | +def page(body: bytes, content_type: str | None = "text/html") -> httpx.Response: |
| 48 | + headers = {} if content_type is None else {"content-type": content_type} |
| 49 | + return httpx.Response(200, content=body, headers=headers) |
| 50 | + |
| 51 | + |
| 52 | +async def fetch(web: FakeWeb, response: httpx.Response, **arguments: Any) -> str: |
| 53 | + web.add(ROBOTS, httpx.Response(404)) |
| 54 | + web.add(PAGE, response) |
| 55 | + async with connect() as (session, _): |
| 56 | + assert isinstance(session, ClientSession) |
| 57 | + result = await session.call_tool("fetch", {"url": PAGE, **arguments}) |
| 58 | + return text_of(result) |
| 59 | + |
| 60 | + |
| 61 | +def document(meta: str, text: str) -> str: |
| 62 | + return f"<html><head>{meta}<title>t</title></head><body><p>{text}</p></body></html>" |
| 63 | + |
| 64 | + |
| 65 | +@pytest.mark.parametrize( |
| 66 | + ("meta", "encoding", "text"), |
| 67 | + [ |
| 68 | + ('<meta charset="Shift_JIS">', "shift_jis", JAPANESE), |
| 69 | + ("<meta charset=windows-1251>", "cp1251", RUSSIAN), |
| 70 | + ("<META CHARSET='euc-kr'>", "euc_kr", KOREAN), |
| 71 | + ('<meta charset="GBK" />', "gbk", CHINESE), |
| 72 | + ( |
| 73 | + '<meta http-equiv="Content-Type" content="text/html; charset=Shift_JIS">', |
| 74 | + "shift_jis", |
| 75 | + JAPANESE, |
| 76 | + ), |
| 77 | + ( |
| 78 | + "<meta http-equiv='content-type' content='text/html;charset=windows-1251'>", |
| 79 | + "cp1251", |
| 80 | + RUSSIAN, |
| 81 | + ), |
| 82 | + ], |
| 83 | +) |
| 84 | +async def test_a_charset_declared_only_in_the_html_is_honored( |
| 85 | + web: FakeWeb, meta: str, encoding: str, text: str |
| 86 | +) -> None: |
| 87 | + body = document(meta, text).encode(encoding) |
| 88 | + assert text in await fetch(web, page(body), raw=True) |
| 89 | + |
| 90 | + |
| 91 | +async def test_a_declared_charset_is_honored_without_a_content_type_header( |
| 92 | + web: FakeWeb, |
| 93 | +) -> None: |
| 94 | + body = document('<meta charset="Shift_JIS">', JAPANESE).encode("shift_jis") |
| 95 | + assert JAPANESE in await fetch(web, page(body, content_type=None), raw=True) |
| 96 | + |
| 97 | + |
| 98 | +async def test_xhtml_content_type_is_scanned(web: FakeWeb) -> None: |
| 99 | + body = document('<meta charset="windows-1251">', RUSSIAN).encode("cp1251") |
| 100 | + response = page(body, content_type="application/xhtml+xml") |
| 101 | + assert RUSSIAN in await fetch(web, response, raw=True) |
| 102 | + |
| 103 | + |
| 104 | +async def test_the_extracted_markdown_is_decoded_too( |
| 105 | + web: FakeWeb, python_readability: None |
| 106 | +) -> None: |
| 107 | + body = document('<meta charset="Shift_JIS">', JAPANESE).encode("shift_jis") |
| 108 | + assert JAPANESE in await fetch(web, page(body)) |
| 109 | + |
| 110 | + |
| 111 | +async def test_a_header_charset_wins_over_the_html_declaration(web: FakeWeb) -> None: |
| 112 | + # The page lies in its meta tag; the header is right, and wins. |
| 113 | + body = document('<meta charset="Shift_JIS">', RUSSIAN).encode("cp1251") |
| 114 | + response = page(body, content_type="text/html; charset=windows-1251") |
| 115 | + assert RUSSIAN in await fetch(web, response, raw=True) |
| 116 | + |
| 117 | + |
| 118 | +async def test_a_byte_order_mark_wins_over_the_html_declaration( |
| 119 | + web: FakeWeb, |
| 120 | +) -> None: |
| 121 | + body = b"\xef\xbb\xbf" + document('<meta charset="Shift_JIS">', JAPANESE).encode( |
| 122 | + "utf-8" |
| 123 | + ) |
| 124 | + assert JAPANESE in await fetch(web, page(body), raw=True) |
| 125 | + |
| 126 | + |
| 127 | +async def test_a_declaration_on_a_non_html_body_is_ignored(web: FakeWeb) -> None: |
| 128 | + # A text/plain body is UTF-8 by default; a meta-like string in it is text. |
| 129 | + body = document('<meta charset="windows-1251">', JAPANESE).encode("utf-8") |
| 130 | + # raw: the body opens with "<html", so without it the server would hand it |
| 131 | + # to Readability.js, whose temp-file write fails on non-cp1252 text on |
| 132 | + # Windows (pre-existing, not what this test is about). |
| 133 | + response = page(body, content_type="text/plain") |
| 134 | + assert JAPANESE in await fetch(web, response, raw=True) |
| 135 | + |
| 136 | + |
| 137 | +async def test_a_text_plain_type_with_an_html_parameter_is_not_scanned( |
| 138 | + web: FakeWeb, |
| 139 | +) -> None: |
| 140 | + # #5302 review: only the media type counts, not "html" in a parameter. |
| 141 | + body = document('<meta charset="windows-1251">', JAPANESE).encode("utf-8") |
| 142 | + response = page(body, content_type='text/plain; name="page.html"') |
| 143 | + assert JAPANESE in await fetch(web, response, raw=True) |
| 144 | + |
| 145 | + |
| 146 | +@pytest.mark.parametrize( |
| 147 | + "head", |
| 148 | + [ |
| 149 | + # #5302 review: a commented-out tag declares nothing. |
| 150 | + '<!-- <meta charset="windows-1251"> --><meta charset="utf-8">', |
| 151 | + "<!-- <meta charset=windows-1251> -->", |
| 152 | + # Nor does an attribute that only contains the word. |
| 153 | + '<meta data-charset="windows-1251" charset="utf-8">', |
| 154 | + '<meta data-charset="windows-1251">', |
| 155 | + # A content attribute counts only under http-equiv="Content-Type". |
| 156 | + '<meta name="description" content="text/html; charset=windows-1251">', |
| 157 | + '<meta http-equiv="refresh" content="5; charset=windows-1251">', |
| 158 | + # An empty declaration, and one with no charset in it. |
| 159 | + '<meta charset="">', |
| 160 | + '<meta http-equiv="Content-Type" content="text/html">', |
| 161 | + # A declaration Python refuses outright. |
| 162 | + '<meta charset="utf\x008">', |
| 163 | + # The first declaration wins. |
| 164 | + '<meta charset="utf-8"><meta charset="windows-1251">', |
| 165 | + ], |
| 166 | +) |
| 167 | +async def test_text_that_is_not_a_declaration_leaves_utf8_alone( |
| 168 | + web: FakeWeb, head: str |
| 169 | +) -> None: |
| 170 | + body = document(head, JAPANESE).encode("utf-8") |
| 171 | + assert JAPANESE in await fetch(web, page(body), raw=True) |
| 172 | + |
| 173 | + |
| 174 | +async def test_a_declaration_after_other_tags_and_comments_is_found( |
| 175 | + web: FakeWeb, |
| 176 | +) -> None: |
| 177 | + head = ( |
| 178 | + '<!-- <meta charset="utf-8"> --><link rel="icon" href="/i.png">' |
| 179 | + '<meta name="viewport" content="width=device-width"><meta charset=Shift_JIS>' |
| 180 | + ) |
| 181 | + body = document(head, JAPANESE).encode("shift_jis") |
| 182 | + assert JAPANESE in await fetch(web, page(body), raw=True) |
| 183 | + |
| 184 | + |
| 185 | +async def test_a_declaration_past_the_first_1024_bytes_is_ignored( |
| 186 | + web: FakeWeb, |
| 187 | +) -> None: |
| 188 | + padding = "<!-- " + "x" * 1100 + " -->" |
| 189 | + body = ( |
| 190 | + f"<html><head>{padding}<meta charset=windows-1251></head>" |
| 191 | + f"<body><p>{JAPANESE}</p></body></html>" |
| 192 | + ).encode("utf-8") |
| 193 | + assert JAPANESE in await fetch(web, page(body), raw=True) |
| 194 | + |
| 195 | + |
| 196 | +async def test_a_page_with_no_declaration_is_decoded_as_utf8(web: FakeWeb) -> None: |
| 197 | + body = document("", JAPANESE).encode("utf-8") |
| 198 | + assert JAPANESE in await fetch(web, page(body), raw=True) |
| 199 | + |
| 200 | + |
| 201 | +@pytest.mark.parametrize( |
| 202 | + "declared", |
| 203 | + [ |
| 204 | + # Unknown to Python. |
| 205 | + "no-such-charset", |
| 206 | + "x-user-defined", |
| 207 | + # Known codecs that are not text encodings, or refuse to decode. |
| 208 | + "base64", |
| 209 | + "rot13", |
| 210 | + "undefined", |
| 211 | + # Text codecs that are not web encodings. |
| 212 | + "unicode_escape", |
| 213 | + "raw_unicode_escape", |
| 214 | + "idna", |
| 215 | + "punycode", |
| 216 | + # #5302 review: encodings that are not ASCII-compatible, so could not |
| 217 | + # have been declared in ASCII. UTF-7 would turn "+AEI-" into "B"; |
| 218 | + # the EBCDIC pages would rewrite the markup. |
| 219 | + "utf-7", |
| 220 | + "cp037", |
| 221 | + "cp1140", |
| 222 | + "hz", |
| 223 | + ], |
| 224 | +) |
| 225 | +async def test_an_unusable_declared_charset_falls_back_to_the_old_decoding( |
| 226 | + web: FakeWeb, declared: str |
| 227 | +) -> None: |
| 228 | + marker = JAPANESE + " \\u0041 +AEI-" |
| 229 | + body = document(f'<meta charset="{declared}">', marker).encode("utf-8") |
| 230 | + text = await fetch(web, page(body), raw=True) |
| 231 | + assert f"<p>{marker}</p>" in text |
| 232 | + |
| 233 | + |
| 234 | +@pytest.mark.parametrize("declared", ["utf-16", "UTF-16LE", "utf-32"]) |
| 235 | +async def test_a_utf16_or_utf32_declaration_is_read_as_utf8( |
| 236 | + web: FakeWeb, declared: str |
| 237 | +) -> None: |
| 238 | + body = document(f'<meta charset="{declared}">', JAPANESE).encode("utf-8") |
| 239 | + assert JAPANESE in await fetch(web, page(body), raw=True) |
0 commit comments