|
12 | 12 |
|
13 | 13 | from __future__ import annotations |
14 | 14 |
|
| 15 | +import sys |
| 16 | + |
15 | 17 | import httpx |
16 | 18 | import pytest |
17 | 19 | from mcp.types import CallToolResult |
18 | 20 |
|
| 21 | +from mcp_server_fetch.server import _body_looks_like_html |
| 22 | + |
19 | 23 | from .harness import FakeWeb, connect |
20 | 24 |
|
21 | 25 | PAGE = "https://example.com/page" |
| 26 | +BOM = "\ufeff" |
22 | 27 |
|
23 | 28 | ARTICLE_BODY = """<head><title>Test Page</title></head><body> |
24 | 29 | <article> |
@@ -96,13 +101,23 @@ async def test_empty_body_without_content_type_is_not_simplified( |
96 | 101 | @pytest.mark.parametrize( |
97 | 102 | "body", |
98 | 103 | [ |
99 | | - f"<!DOCTYPE html>\n{ARTICLE_BODY}", |
100 | | - f"<!doctype html>{ARTICLE_BODY}", |
101 | | - f"\n <!DocType HTML>\n{ARTICLE_BODY}", |
| 104 | + pytest.param(f"<!DOCTYPE html>\n{ARTICLE_BODY}", id="doctype-upper"), |
| 105 | + pytest.param(f"<!doctype html>{ARTICLE_BODY}", id="doctype-lower"), |
| 106 | + pytest.param( |
| 107 | + f"{BOM}\n <!DocType HTML>\n{ARTICLE_BODY}", |
| 108 | + id="bom-and-whitespace", |
| 109 | + # readabilipy 0.2.0 writes the page to a temp file in the locale |
| 110 | + # encoding, which cannot hold a BOM on Windows (cp1252). The |
| 111 | + # detection itself is covered on every platform by |
| 112 | + # test_body_detection_skips_bom_and_whitespace below. |
| 113 | + marks=pytest.mark.skipif( |
| 114 | + sys.platform == "win32", |
| 115 | + reason="readabilipy 0.2.0 cannot extract non-cp1252 pages on Windows (#5303)", |
| 116 | + ), |
| 117 | + ), |
102 | 118 | # More leading whitespace than the 100-character "<html" window. |
103 | | - f"{' ' * 120}<HTML>{ARTICLE_BODY}</HTML>", |
| 119 | + pytest.param(f"{' ' * 120}<HTML>{ARTICLE_BODY}</HTML>", id="long-indent"), |
104 | 120 | ], |
105 | | - ids=["doctype-upper", "doctype-lower", "bom-and-whitespace", "long-indent"], |
106 | 121 | ) |
107 | 122 | async def test_html_without_content_type_is_still_simplified( |
108 | 123 | web: FakeWeb, body: str |
@@ -136,3 +151,28 @@ async def test_doctype_alone_does_not_override_a_declared_content_type( |
136 | 151 | 200, content=body.encode(), headers={"content-type": "text/plain"} |
137 | 152 | ) |
138 | 153 | assert_raw(await fetch(web, response), body) |
| 154 | + |
| 155 | + |
| 156 | +@pytest.mark.parametrize( |
| 157 | + "body, expected", |
| 158 | + [ |
| 159 | + (f"{BOM}<!doctype html><p>x</p>", True), |
| 160 | + (f"{BOM}\n <!DocType HTML>\n<p>x</p>", True), |
| 161 | + (f"{BOM} \t\r\n<HTML><p>x</p></HTML>", True), |
| 162 | + (f'{BOM}{{"key": "value"}}', False), |
| 163 | + (f"{BOM}plain text", False), |
| 164 | + ("", False), |
| 165 | + ], |
| 166 | + ids=[ |
| 167 | + "bom-doctype", |
| 168 | + "bom-whitespace-doctype", |
| 169 | + "bom-html", |
| 170 | + "bom-json", |
| 171 | + "bom-text", |
| 172 | + "empty", |
| 173 | + ], |
| 174 | +) |
| 175 | +def test_body_detection_skips_bom_and_whitespace(body: str, expected: bool) -> None: |
| 176 | + # Runs on every platform: what #5134 decides, without readabilipy's |
| 177 | + # extraction (which cannot handle a BOM on Windows, #5303). |
| 178 | + assert _body_looks_like_html(body) is expected |
0 commit comments