Skip to content

Commit f85cb8f

Browse files
cliffhallclaude
andcommitted
fix(fetch): harden robots.txt handling
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent 69b53e9 commit f85cb8f

3 files changed

Lines changed: 123 additions & 6 deletions

File tree

‎src/fetch/README.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -168,8 +168,9 @@ Optionally, you can add it to a file called `.vscode/mcp.json` in your workspace
168168
### Customization - robots.txt
169169

170170
By default, the server will obey a websites robots.txt file if the request came from the model (via a tool), but not if
171-
the request was user initiated (via a prompt). This can be disabled by adding the argument `--ignore-robots-txt` to the
172-
`args` list in the configuration.
171+
the request was user initiated (via a prompt). Every redirect hop of a tool fetch is checked against the robots.txt of
172+
the host it leads to, the same way as the requested URL. This can be disabled by adding the argument
173+
`--ignore-robots-txt` to the `args` list in the configuration.
173174

174175
### Customization - Private addresses
175176

‎src/fetch/src/mcp_server_fetch/server.py‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
import ipaddress
2424
import os
2525
import socket
26-
from collections.abc import Mapping
26+
from collections.abc import Awaitable, Callable, Mapping
2727
from typing import Annotated, Any, Tuple
2828
from importlib.metadata import version
2929
from urllib.parse import urlparse, urlunparse
@@ -299,15 +299,37 @@ async def fetch_url(
299299
force_raw: bool = False,
300300
proxy_url: str | None = None,
301301
allow_private_ips: bool = False,
302+
check_robots_txt: bool = False,
302303
) -> Tuple[str, str]:
303304
"""
304305
Fetch the URL and return the content in a form ready for the LLM, as well as a prefix string with status information.
306+
307+
With ``check_robots_txt``, every redirect hop is checked against the
308+
robots.txt of its own origin before it is requested. The caller has
309+
already checked the requested URL itself.
305310
"""
306311
from httpx import HTTPError
307312

313+
seen_first_request = False
314+
315+
async def _check_redirect_robots_txt(request: httpx.Request) -> None:
316+
# httpx runs request hooks before each request, redirect hops included.
317+
nonlocal seen_first_request
318+
if not seen_first_request:
319+
seen_first_request = True
320+
return
321+
await check_may_autonomously_fetch_url(
322+
str(request.url), user_agent, proxy_url, allow_private_ips
323+
)
324+
308325
async with make_client(proxy_url) as client:
326+
hooks: list[Callable[[httpx.Request], Awaitable[None]]] = []
309327
if not allow_private_ips:
310-
client.event_hooks = {"request": [_refuse_private_destination]}
328+
hooks.append(_refuse_private_destination)
329+
if check_robots_txt:
330+
hooks.append(_check_redirect_robots_txt)
331+
if hooks:
332+
client.event_hooks = {"request": hooks}
311333
try:
312334
response = await client.get(
313335
url,
@@ -462,6 +484,7 @@ async def fetch(
462484
force_raw=raw,
463485
proxy_url=proxy_url,
464486
allow_private_ips=allow_private_ips,
487+
check_robots_txt=not ignore_robots_txt,
465488
)
466489
original_length = len(content)
467490
if start_index >= original_length:

‎src/fetch/tests/test_protocol.py‎

Lines changed: 95 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -460,6 +460,98 @@ async def test_ignore_robots_txt_skips_the_check(web: FakeWeb) -> None:
460460
assert web.urls() == [PAGE]
461461

462462

463+
OTHER_ROBOTS = "https://other.example.org/robots.txt"
464+
OTHER_PAGE = "https://other.example.org/secret"
465+
466+
467+
async def test_redirect_to_host_whose_robots_disallows_is_refused(
468+
web: FakeWeb,
469+
) -> None:
470+
# robots.txt belongs to one origin, so a redirect to another host is
471+
# checked against that host's robots.txt before it is requested.
472+
other_robots = "User-agent: *\nDisallow: /"
473+
web.add(ROBOTS, plain("", status=404))
474+
web.add(PAGE, httpx.Response(302, headers={"location": OTHER_PAGE}))
475+
web.add(OTHER_ROBOTS, plain(other_robots))
476+
web.add(OTHER_PAGE, plain("other-secret"))
477+
async with connect() as (session, _):
478+
result = await call(session, {"url": PAGE})
479+
assert wire(result)["isError"] is True
480+
text = text_of(result)
481+
assert text.startswith(
482+
f"{TOOL_ERROR}The sites robots.txt ({OTHER_ROBOTS}), specifies that "
483+
"autonomous fetching of this page is not allowed, "
484+
)
485+
assert f"<url>{OTHER_PAGE}</url>" in text
486+
assert "other-secret" not in text
487+
# The redirect target itself is never requested.
488+
assert web.urls() == [ROBOTS, PAGE, OTHER_ROBOTS]
489+
490+
491+
async def test_redirect_to_a_disallowed_path_on_the_same_host_is_refused(
492+
web: FakeWeb,
493+
) -> None:
494+
private = "https://example.com/private/report"
495+
web.add(ROBOTS, plain("User-agent: *\nDisallow: /private"))
496+
web.add(PAGE, httpx.Response(302, headers={"location": "/private/report"}))
497+
web.add(private, plain("private-report"))
498+
async with connect() as (session, _):
499+
result = await call(session, {"url": PAGE})
500+
assert wire(result)["isError"] is True
501+
assert f"<url>{private}</url>" in text_of(result)
502+
assert web.urls() == [ROBOTS, PAGE, ROBOTS]
503+
504+
505+
async def test_redirect_to_host_answering_robots_403_is_refused(
506+
web: FakeWeb,
507+
) -> None:
508+
web.add(ROBOTS, plain("", status=404))
509+
web.add(PAGE, httpx.Response(302, headers={"location": OTHER_PAGE}))
510+
web.add(OTHER_ROBOTS, plain("", status=403))
511+
web.add(OTHER_PAGE, plain("other-secret"))
512+
async with connect() as (session, _):
513+
result = await call(session, {"url": PAGE})
514+
assert wire(result)["isError"] is True
515+
assert text_of(result).startswith(
516+
f"{TOOL_ERROR}When fetching robots.txt ({OTHER_ROBOTS}), received status 403"
517+
)
518+
assert web.urls() == [ROBOTS, PAGE, OTHER_ROBOTS]
519+
520+
521+
async def test_redirect_to_an_allowed_target_is_followed(web: FakeWeb) -> None:
522+
web.add(ROBOTS, plain("", status=404))
523+
web.add(PAGE, httpx.Response(302, headers={"location": OTHER_PAGE}))
524+
web.add(OTHER_ROBOTS, plain("User-agent: *\nDisallow: /private"))
525+
web.add(OTHER_PAGE, plain("other-page"))
526+
async with connect() as (session, _):
527+
result = await call(session, {"url": PAGE})
528+
assert wire(result)["isError"] is False
529+
assert text_of(result) == f"{RAW_PREFIX_PLAIN}Contents of {PAGE}:\nother-page"
530+
assert web.urls() == [ROBOTS, PAGE, OTHER_ROBOTS, OTHER_PAGE]
531+
for request in web.requests:
532+
assert request.headers["user-agent"] == DEFAULT_USER_AGENT_AUTONOMOUS
533+
534+
535+
async def test_redirects_skip_robots_with_ignore_robots_txt_and_on_the_prompt(
536+
web: FakeWeb,
537+
) -> None:
538+
# --ignore-robots-txt and the user-initiated prompt never consult
539+
# robots.txt, for the requested URL or for any redirect hop.
540+
web.add(ROBOTS, plain("User-agent: *\nDisallow: /"))
541+
web.add(PAGE, httpx.Response(302, headers={"location": OTHER_PAGE}))
542+
web.add(OTHER_ROBOTS, plain("User-agent: *\nDisallow: /"))
543+
web.add(OTHER_PAGE, plain("other-page"))
544+
async with connect(ignore_robots_txt=True) as (session, _):
545+
result = await call(session, {"url": PAGE})
546+
assert wire(result)["isError"] is False
547+
assert web.urls() == [PAGE, OTHER_PAGE]
548+
web.requests.clear()
549+
async with connect() as (session, _):
550+
prompt = await session.get_prompt("fetch", {"url": PAGE})
551+
assert "other-page" in wire(prompt)["messages"][0]["content"]["text"]
552+
assert web.urls() == [PAGE, OTHER_PAGE]
553+
554+
463555
# --------------------------------------------------------------------------
464556
# tools/call: fetching and content handling
465557
# --------------------------------------------------------------------------
@@ -855,8 +947,9 @@ async def test_allow_private_ips_follows_redirect_to_private_address(
855947
assert wire(result)["isError"] is False
856948
assert text_of(result) == f"{RAW_PREFIX_PLAIN}Contents of {PAGE}:\ninstance-secret"
857949
assert prompt.description == PROMPT_DESCRIPTION
858-
# robots.txt is only consulted for the original host, never the target.
859-
assert web.urls() == [ROBOTS, PAGE, metadata, metadata]
950+
# The redirect target's own robots.txt (a 404 here) is checked before it.
951+
metadata_robots = "http://169.254.169.254/robots.txt"
952+
assert web.urls() == [ROBOTS, PAGE, metadata_robots, metadata, metadata]
860953
# With the guard off, nothing is resolved by the server.
861954
assert web.lookups == []
862955

0 commit comments

Comments
 (0)