|
| 1 | +"""Protocol-level tests for how a robots.txt server error gates a tool fetch (#5197). |
| 2 | +
|
| 3 | +RFC 9309 section 2.3.1.4 says a robots.txt that is unreachable because of a |
| 4 | +server error (5xx) means complete disallow. The fetch tool used to fall |
| 5 | +through on a 5xx and parse the error page as rules, and an error page holds no |
| 6 | +rules, so the fetch went ahead. These tests drive the real server through an |
| 7 | +MCP client (``harness.connect``) with HTTP served by ``harness.FakeWeb``, and |
| 8 | +pin that a 5xx refuses with the same guidance as 401 and 403, without reading |
| 9 | +the body and without requesting the page. |
| 10 | +""" |
| 11 | + |
| 12 | +from __future__ import annotations |
| 13 | + |
| 14 | +import httpx |
| 15 | +import pytest |
| 16 | + |
| 17 | +from .harness import FakeWeb, connect |
| 18 | + |
| 19 | +PAGE = "https://example.com/page" |
| 20 | +ROBOTS = "https://example.com/robots.txt" |
| 21 | +TOOL_ERROR = "Error executing tool fetch: " |
| 22 | + |
| 23 | + |
| 24 | +def refusal(status: int, robots_url: str = ROBOTS) -> str: |
| 25 | + return ( |
| 26 | + f"{TOOL_ERROR}When fetching robots.txt ({robots_url}), received status " |
| 27 | + f"{status} so assuming that autonomous fetching is not allowed, the user " |
| 28 | + "can try manually fetching by using the fetch prompt" |
| 29 | + ) |
| 30 | + |
| 31 | + |
| 32 | +def page(body: str, status: int = 200) -> httpx.Response: |
| 33 | + return httpx.Response( |
| 34 | + status, content=body.encode(), headers={"content-type": "text/plain"} |
| 35 | + ) |
| 36 | + |
| 37 | + |
| 38 | +@pytest.mark.parametrize("status", [500, 502, 503, 504, 599]) |
| 39 | +async def test_robots_5xx_refuses_without_parsing_the_body( |
| 40 | + web: FakeWeb, status: int |
| 41 | +) -> None: |
| 42 | + # The body allows everything: if it were parsed as rules, the fetch would |
| 43 | + # go ahead. A 5xx must refuse whatever the error page says. |
| 44 | + web.add(ROBOTS, page("User-agent: *\nAllow: /", status=status)) |
| 45 | + web.add(PAGE, page("should not be fetched")) |
| 46 | + async with connect() as (session, _): |
| 47 | + result = await session.call_tool("fetch", {"url": PAGE}) |
| 48 | + data = result.model_dump(by_alias=True, mode="json", exclude_none=True) |
| 49 | + assert data["isError"] is True |
| 50 | + assert [c["text"] for c in data["content"]] == [refusal(status)] |
| 51 | + assert web.urls() == [ROBOTS] |
| 52 | + |
| 53 | + |
| 54 | +async def test_robots_5xx_after_a_redirect_refuses(web: FakeWeb) -> None: |
| 55 | + moved = "https://www.example.com/robots.txt" |
| 56 | + web.add(ROBOTS, httpx.Response(301, headers={"location": moved})) |
| 57 | + web.add(moved, page("", status=503)) |
| 58 | + async with connect() as (session, _): |
| 59 | + result = await session.call_tool("fetch", {"url": PAGE}) |
| 60 | + data = result.model_dump(by_alias=True, mode="json", exclude_none=True) |
| 61 | + assert data["isError"] is True |
| 62 | + assert [c["text"] for c in data["content"]] == [refusal(503)] |
| 63 | + assert web.urls() == [ROBOTS, moved] |
| 64 | + |
| 65 | + |
| 66 | +async def test_fetch_prompt_ignores_a_5xx_robots(web: FakeWeb) -> None: |
| 67 | + # The refusal points the user at the fetch prompt; that path must still |
| 68 | + # work, since a user-initiated fetch does not consult robots.txt. |
| 69 | + web.add(ROBOTS, page("", status=503)) |
| 70 | + web.add(PAGE, page("manual ok")) |
| 71 | + async with connect() as (session, _): |
| 72 | + result = await session.get_prompt("fetch", {"url": PAGE}) |
| 73 | + assert "manual ok" in str(result.model_dump(mode="json")) |
| 74 | + assert web.urls() == [PAGE] |
0 commit comments