|
| 1 | +"""Protocol-level tests for responses served without a Content-Type (#5134). |
| 2 | +
|
| 3 | +A missing Content-Type header used to make every body count as HTML, so JSON, |
| 4 | +plain text and other payloads were pushed through readability and came back |
| 5 | +altered or as a simplification error. Now a header-less body is simplified |
| 6 | +only when it itself opens like an HTML document (an HTML doctype or an |
| 7 | +``<html`` tag, after any byte-order mark and whitespace); anything else comes |
| 8 | +back raw. These tests drive ``serve()`` through an MCP client with HTTP served |
| 9 | +by ``harness.FakeWeb``, and also pin the HTML media types that are still |
| 10 | +simplified whatever the body looks like. |
| 11 | +""" |
| 12 | + |
| 13 | +from __future__ import annotations |
| 14 | + |
| 15 | +import httpx |
| 16 | +import pytest |
| 17 | +from mcp.types import CallToolResult |
| 18 | + |
| 19 | +from .harness import FakeWeb, connect |
| 20 | + |
| 21 | +PAGE = "https://example.com/page" |
| 22 | + |
| 23 | +ARTICLE_BODY = """<head><title>Test Page</title></head><body> |
| 24 | +<article> |
| 25 | +<h1>Hello World</h1> |
| 26 | +<p>This is a <a href="https://example.com/x">link</a> in a test paragraph |
| 27 | +with enough words to count as the main content of the page.</p> |
| 28 | +<h2>Sub heading</h2> |
| 29 | +<p>More text here.</p> |
| 30 | +</article> |
| 31 | +</body>""" |
| 32 | + |
| 33 | + |
| 34 | +def text_of(result: CallToolResult) -> str: |
| 35 | + data = result.model_dump(by_alias=True, mode="json", exclude_none=True) |
| 36 | + assert data["isError"] is False |
| 37 | + assert len(data["content"]) == 1 |
| 38 | + assert data["content"][0]["type"] == "text" |
| 39 | + return data["content"][0]["text"] |
| 40 | + |
| 41 | + |
| 42 | +def headerless(body: str) -> httpx.Response: |
| 43 | + response = httpx.Response(200, content=body.encode()) |
| 44 | + assert "content-type" not in response.headers |
| 45 | + return response |
| 46 | + |
| 47 | + |
| 48 | +async def fetch(web: FakeWeb, response: httpx.Response) -> str: |
| 49 | + web.add(PAGE, response) |
| 50 | + async with connect(ignore_robots_txt=True) as (session, _): |
| 51 | + result = await session.call_tool("fetch", {"url": PAGE}) |
| 52 | + return text_of(result) |
| 53 | + |
| 54 | + |
| 55 | +def assert_raw(text: str, body: str) -> None: |
| 56 | + assert "Page failed to be simplified" not in text |
| 57 | + assert text.endswith(f"Contents of {PAGE}:\n{body}") |
| 58 | + assert "cannot be simplified to markdown, but here is the raw content:" in text |
| 59 | + |
| 60 | + |
| 61 | +def assert_simplified(text: str) -> None: |
| 62 | + assert text.startswith(f"Contents of {PAGE}:\n") |
| 63 | + assert "[link](https://example.com/x)" in text |
| 64 | + assert "## Sub heading" in text |
| 65 | + assert "<article>" not in text |
| 66 | + |
| 67 | + |
| 68 | +@pytest.mark.parametrize( |
| 69 | + "body", |
| 70 | + [ |
| 71 | + '{"key": "value", "snake_case": "*starred*"}', |
| 72 | + "[1, 2, 3]", |
| 73 | + "just some plain_text with *asterisks*\nand a second line", |
| 74 | + "a,b,c\n1,2,3\n", |
| 75 | + ], |
| 76 | + ids=["json-object", "json-array", "plain-text", "csv"], |
| 77 | +) |
| 78 | +async def test_non_html_without_content_type_is_returned_raw( |
| 79 | + web: FakeWeb, body: str |
| 80 | +) -> None: |
| 81 | + assert_raw(await fetch(web, headerless(body)), body) |
| 82 | + |
| 83 | + |
| 84 | +async def test_empty_body_without_content_type_is_not_simplified( |
| 85 | + web: FakeWeb, |
| 86 | +) -> None: |
| 87 | + # An empty body used to reach readability and fail to simplify; now it is |
| 88 | + # raw content with nothing in it, which the tool reports as such. |
| 89 | + text = await fetch(web, headerless("")) |
| 90 | + assert "Page failed to be simplified" not in text |
| 91 | + assert "cannot be simplified to markdown, but here is the raw content:" in text |
| 92 | + assert text.endswith("<error>No more content available.</error>") |
| 93 | + |
| 94 | + |
| 95 | +@pytest.mark.usefixtures("node_readability") |
| 96 | +@pytest.mark.parametrize( |
| 97 | + "body", |
| 98 | + [ |
| 99 | + f"<!DOCTYPE html>\n{ARTICLE_BODY}", |
| 100 | + f"<!doctype html>{ARTICLE_BODY}", |
| 101 | + f"\n <!DocType HTML>\n{ARTICLE_BODY}", |
| 102 | + # More leading whitespace than the 100-character "<html" window. |
| 103 | + f"{' ' * 120}<HTML>{ARTICLE_BODY}</HTML>", |
| 104 | + ], |
| 105 | + ids=["doctype-upper", "doctype-lower", "bom-and-whitespace", "long-indent"], |
| 106 | +) |
| 107 | +async def test_html_without_content_type_is_still_simplified( |
| 108 | + web: FakeWeb, body: str |
| 109 | +) -> None: |
| 110 | + assert_simplified(await fetch(web, headerless(body))) |
| 111 | + |
| 112 | + |
| 113 | +@pytest.mark.usefixtures("node_readability") |
| 114 | +@pytest.mark.parametrize( |
| 115 | + "content_type", |
| 116 | + [ |
| 117 | + "application/xhtml+xml", |
| 118 | + "application/xhtml+xml; charset=utf-8", |
| 119 | + "Text/HTML; charset=UTF-8", |
| 120 | + ], |
| 121 | +) |
| 122 | +async def test_html_media_types_are_simplified(web: FakeWeb, content_type: str) -> None: |
| 123 | + # The body carries no "<html" tag, so only the header can make it HTML. |
| 124 | + response = httpx.Response( |
| 125 | + 200, content=ARTICLE_BODY.encode(), headers={"content-type": content_type} |
| 126 | + ) |
| 127 | + assert_simplified(await fetch(web, response)) |
| 128 | + |
| 129 | + |
| 130 | +async def test_doctype_alone_does_not_override_a_declared_content_type( |
| 131 | + web: FakeWeb, |
| 132 | +) -> None: |
| 133 | + # Sniffing the doctype applies only when the header is missing. |
| 134 | + body = f"<!doctype html>{ARTICLE_BODY}" |
| 135 | + response = httpx.Response( |
| 136 | + 200, content=body.encode(), headers={"content-type": "text/plain"} |
| 137 | + ) |
| 138 | + assert_raw(await fetch(web, response), body) |
0 commit comments