Skip to content

Commit 50f1be9

Browse files
cliffhallclaude
andcommitted
fix(fetch): refuse an autonomous fetch when robots.txt answers 5xx
A 5xx robots.txt response fell through to parsing the error page as rules, and an error page holds no rules, so the fetch went ahead. RFC 9309 section 2.3.1.4 treats a server-error robots.txt as complete disallow, so a 5xx now refuses with the same guidance as 401 and 403. Closes #5197 Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent f72fdbd commit 50f1be9

4 files changed

Lines changed: 81 additions & 14 deletions

File tree

‎src/fetch/README.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -169,8 +169,10 @@ Optionally, you can add it to a file called `.vscode/mcp.json` in your workspace
169169

170170
By default, the server will obey a websites robots.txt file if the request came from the model (via a tool), but not if
171171
the request was user initiated (via a prompt). Every redirect hop of a tool fetch is checked against the robots.txt of
172-
the host it leads to, the same way as the requested URL. This can be disabled by adding the argument
173-
`--ignore-robots-txt` to the `args` list in the configuration.
172+
the host it leads to, the same way as the requested URL. If the robots.txt request fails to connect, or is answered
173+
with 401, 403 or a server error (5xx), the tool refuses the fetch rather than reading the response as rules (RFC 9309
174+
treats a 5xx robots.txt as complete disallow); any other 4xx is treated as no rules. This can be disabled by adding the
175+
argument `--ignore-robots-txt` to the `args` list in the configuration.
174176

175177
### Customization - Private addresses
176178

‎src/fetch/src/mcp_server_fetch/server.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,9 @@ async def check_may_autonomously_fetch_url(
284284
raise ToolError(
285285
f"Failed to fetch robots.txt {robot_txt_url} due to a connection issue"
286286
)
287-
if response.status_code in (401, 403):
287+
# A 5xx robots.txt is "unreachable" (RFC 9309 section 2.3.1.4), which
288+
# means complete disallow: the error page is never parsed as rules.
289+
if response.status_code in (401, 403) or response.status_code >= 500:
288290
raise ToolError(
289291
f"When fetching robots.txt ({robot_txt_url}), received status {response.status_code} so assuming that autonomous fetching is not allowed, the user can try manually fetching by using the fetch prompt"
290292
)

‎src/fetch/tests/test_protocol.py‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -419,17 +419,6 @@ async def test_robots_comment_lines_are_ignored(web: FakeWeb) -> None:
419419
assert wire(result)["isError"] is False
420420

421421

422-
async def test_robots_5xx_body_is_parsed_as_rules(web: FakeWeb) -> None:
423-
# Only 4xx short-circuits; a 5xx response body is still parsed as rules.
424-
web.add(ROBOTS, plain("User-agent: *\nDisallow: /", status=503))
425-
async with connect() as (session, _):
426-
result = await call(session, {"url": PAGE})
427-
assert wire(result)["isError"] is True
428-
assert "specifies that autonomous fetching of this page is not allowed" in (
429-
text_of(result)
430-
)
431-
432-
433422
async def test_robots_redirect_is_followed(web: FakeWeb) -> None:
434423
web.add(
435424
ROBOTS,
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
"""Protocol-level tests for how a robots.txt server error gates a tool fetch (#5197).
2+
3+
RFC 9309 section 2.3.1.4 says a robots.txt that is unreachable because of a
4+
server error (5xx) means complete disallow. The fetch tool used to fall
5+
through on a 5xx and parse the error page as rules, and an error page holds no
6+
rules, so the fetch went ahead. These tests drive the real server through an
7+
MCP client (``harness.connect``) with HTTP served by ``harness.FakeWeb``, and
8+
pin that a 5xx refuses with the same guidance as 401 and 403, without reading
9+
the body and without requesting the page.
10+
"""
11+
12+
from __future__ import annotations
13+
14+
import httpx
15+
import pytest
16+
17+
from .harness import FakeWeb, connect
18+
19+
PAGE = "https://example.com/page"
20+
ROBOTS = "https://example.com/robots.txt"
21+
TOOL_ERROR = "Error executing tool fetch: "
22+
23+
24+
def refusal(status: int, robots_url: str = ROBOTS) -> str:
25+
return (
26+
f"{TOOL_ERROR}When fetching robots.txt ({robots_url}), received status "
27+
f"{status} so assuming that autonomous fetching is not allowed, the user "
28+
"can try manually fetching by using the fetch prompt"
29+
)
30+
31+
32+
def page(body: str, status: int = 200) -> httpx.Response:
33+
return httpx.Response(
34+
status, content=body.encode(), headers={"content-type": "text/plain"}
35+
)
36+
37+
38+
@pytest.mark.parametrize("status", [500, 502, 503, 504, 599])
39+
async def test_robots_5xx_refuses_without_parsing_the_body(
40+
web: FakeWeb, status: int
41+
) -> None:
42+
# The body allows everything: if it were parsed as rules, the fetch would
43+
# go ahead. A 5xx must refuse whatever the error page says.
44+
web.add(ROBOTS, page("User-agent: *\nAllow: /", status=status))
45+
web.add(PAGE, page("should not be fetched"))
46+
async with connect() as (session, _):
47+
result = await session.call_tool("fetch", {"url": PAGE})
48+
data = result.model_dump(by_alias=True, mode="json", exclude_none=True)
49+
assert data["isError"] is True
50+
assert [c["text"] for c in data["content"]] == [refusal(status)]
51+
assert web.urls() == [ROBOTS]
52+
53+
54+
async def test_robots_5xx_after_a_redirect_refuses(web: FakeWeb) -> None:
55+
moved = "https://www.example.com/robots.txt"
56+
web.add(ROBOTS, httpx.Response(301, headers={"location": moved}))
57+
web.add(moved, page("", status=503))
58+
async with connect() as (session, _):
59+
result = await session.call_tool("fetch", {"url": PAGE})
60+
data = result.model_dump(by_alias=True, mode="json", exclude_none=True)
61+
assert data["isError"] is True
62+
assert [c["text"] for c in data["content"]] == [refusal(503)]
63+
assert web.urls() == [ROBOTS, moved]
64+
65+
66+
async def test_fetch_prompt_ignores_a_5xx_robots(web: FakeWeb) -> None:
67+
# The refusal points the user at the fetch prompt; that path must still
68+
# work, since a user-initiated fetch does not consult robots.txt.
69+
web.add(ROBOTS, page("", status=503))
70+
web.add(PAGE, page("manual ok"))
71+
async with connect() as (session, _):
72+
result = await session.get_prompt("fetch", {"url": PAGE})
73+
assert "manual ok" in str(result.model_dump(mode="json"))
74+
assert web.urls() == [PAGE]

0 commit comments

Comments
 (0)