Skip to content

Commit 7924213

Browse files
cliffhallclaude
andcommitted
fix(fetch): address GHSA-3rxj-6jhw-qfv7 and GHSA-f9hh-cmx9-2f23
Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent b71b3da commit 7924213

4 files changed

Lines changed: 224 additions & 13 deletions

File tree

‎src/fetch/README.md‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,10 @@ Optionally, you can add it to a file called `.vscode/mcp.json` in your workspace
182182

183183
By default, the server will obey a websites robots.txt file if the request came from the model (via a tool), but not if
184184
the request was user initiated (via a prompt). Every redirect hop of a tool fetch is checked against the robots.txt of
185-
the host it leads to, the same way as the requested URL. If the robots.txt request fails to connect, or is answered
185+
the host it leads to, the same way as the requested URL. A path is matched against the rules both as it is spelled and
186+
as an origin typically resolves it (`%2F` and `%2E` decoded, repeated slashes merged, `.` and `..` segments removed),
187+
and the fetch is refused if either is disallowed, so `//private` or `/x/../private` cannot get past `Disallow: /private`.
188+
Matching stays case-sensitive, as RFC 9309 specifies. If the robots.txt request fails to connect, or is answered
186189
with 401, 403 or a server error (5xx), the tool refuses the fetch rather than reading the response as rules (RFC 9309
187190
treats a 5xx robots.txt as complete disallow); any other 4xx is treated as no rules. This can be disabled by adding the
188191
argument `--ignore-robots-txt` to the `args` list in the configuration.
@@ -192,9 +195,11 @@ argument `--ignore-robots-txt` to the `args` list in the configuration.
192195
By default, the server refuses any request, including a redirect hop or a robots.txt fetch, whose host resolves to an
193196
address that is not globally routable: loopback (`127.0.0.0/8`, `::1`), private (`10.0.0.0/8`, `172.16.0.0/12`,
194197
`192.168.0.0/16`, `fc00::/7`), link-local (`169.254.0.0/16`, `fe80::/10`), shared address space (`100.64.0.0/10`),
195-
multicast, unspecified, documentation and the other IANA special-purpose ranges. An IPv4-mapped (`::ffff:0:0/96`) or
196-
well-known NAT64 (`64:ff9b::/96`) address is judged by the IPv4 address it carries; a deprecated IPv4-compatible
197-
address (`::/96`, such as `::127.0.0.1`) is refused whatever IPv4 it carries. A hostname is refused if any one of its addresses is in those ranges.
198+
multicast, unspecified, documentation and the other IANA special-purpose ranges, including deprecated site-local
199+
(`fec0::/10`). An IPv6 address outside global unicast (`2000::/3`) is refused unless it carries an IPv4 address: an
200+
IPv4-mapped (`::ffff:0:0/96`), IPv4-translated (`::ffff:0:0:0/96`) or well-known NAT64 (`64:ff9b::/96`) address is
201+
judged by the IPv4 address it carries; a deprecated IPv4-compatible address (`::/96`, such as `::127.0.0.1`) is
202+
refused whatever IPv4 it carries. A hostname is refused if any one of its addresses is in those ranges.
198203
The client receives an error naming the refused address.
199204

200205
To fetch from local or internal hosts (a development server, an intranet page), add the argument

‎src/fetch/src/mcp_server_fetch/server.py‎

Lines changed: 75 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,14 @@
2525
import asyncio
2626
import ipaddress
2727
import os
28+
import re
2829
import socket
2930
import zlib
3031
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence
3132
from contextlib import asynccontextmanager
3233
from importlib.metadata import version
3334
from typing import Annotated, Any
34-
from urllib.parse import urlparse, urlunparse
35+
from urllib.parse import urlparse, urlsplit, urlunparse, urlunsplit
3536

3637
import httpx
3738
import markdownify
@@ -104,8 +105,13 @@ async def _resolve_host(host: str) -> list[str]:
104105
"2001::/23",
105106
"2001:db8::/32",
106107
"2002::/16",
108+
# IPv6 documentation (RFC 9637), absent from older is_global tables.
109+
"3fff::/20",
107110
"fc00::/7",
108111
"fe80::/10",
112+
# Deprecated site-local (RFC 3879), which is_global reads as global.
113+
# Outside 2000::/3 as well, but listed so it is refused by name.
114+
"fec0::/10",
109115
)
110116
)
111117

@@ -131,6 +137,17 @@ async def _resolve_host(host: str) -> list[str]:
131137
# in it to the IPv4 held in its low 32 bits, so that IPv4 is the destination.
132138
_NAT64_WELL_KNOWN = ipaddress.IPv6Network("64:ff9b::/96")
133139

140+
# IPv4-translated addresses (RFC 2765, RFC 6145), which a stateless translator
141+
# forwards to the IPv4 held in their low 32 bits, like NAT64.
142+
_IPV4_TRANSLATED = ipaddress.IPv6Network("::ffff:0:0:0/96")
143+
144+
# IPv6 global unicast (RFC 4291 2.4, IANA's IPv6 address space registry). The
145+
# rest of the IPv6 space is reserved by the IETF, unique or site-local,
146+
# link-local or multicast, none of it a public destination, while is_global
147+
# reads most of it as global. Deny-by-default there means a range missing from
148+
# the list above cannot pass as public.
149+
_IPV6_GLOBAL_UNICAST = ipaddress.IPv6Network("2000::/3")
150+
134151

135152
def _is_public_address(address: str) -> bool:
136153
"""True when ``address`` is a globally routable unicast address.
@@ -139,18 +156,24 @@ def _is_public_address(address: str) -> bool:
139156
cloud metadata endpoints), shared address space (100.64.0.0/10, which holds
140157
Alibaba Cloud's 100.100.100.200), unique local IPv6 (AWS's fd00:ec2::254),
141158
multicast, unspecified, documentation and other special-purpose ranges.
142-
An IPv4-mapped or well-known NAT64 address is judged by the IPv4 it wraps;
143-
a deprecated IPv4-compatible address (``::/96``) is refused whatever it holds.
159+
An IPv4-mapped, IPv4-translated or well-known NAT64 address is judged by
160+
the IPv4 it wraps; a deprecated IPv4-compatible address (``::/96``) is
161+
refused whatever it holds. Any other IPv6 address outside global unicast
162+
(``2000::/3``) is refused.
144163
"""
145164
ip = ipaddress.ip_address(address)
146165
if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
147166
ip = ip.ipv4_mapped
148-
elif isinstance(ip, ipaddress.IPv6Address) and ip in _NAT64_WELL_KNOWN:
167+
elif isinstance(ip, ipaddress.IPv6Address) and (
168+
ip in _NAT64_WELL_KNOWN or ip in _IPV4_TRANSLATED
169+
):
149170
ip = ipaddress.IPv4Address(int(ip) & 0xFFFFFFFF)
150171
if any(ip in network for network in _PUBLIC_EXCEPTIONS):
151172
return True
152173
if any(ip in network for network in _NON_PUBLIC_NETWORKS):
153174
return False
175+
if isinstance(ip, ipaddress.IPv6Address) and ip not in _IPV6_GLOBAL_UNICAST:
176+
return False
154177
return ip.is_global and not ip.is_multicast
155178

156179

@@ -660,6 +683,48 @@ def handle_starttag(
660683
return decode_text(response, content)
661684

662685

686+
# Escapes of the path separator and of the dot, which an origin may decode
687+
# before it resolves the path. Other escapes (such as %20) are left alone.
688+
_ENCODED_SLASH = re.compile("%2f", re.IGNORECASE)
689+
_ENCODED_DOT = re.compile("%2e", re.IGNORECASE)
690+
_REPEATED_SLASHES = re.compile("/{2,}")
691+
692+
693+
def _remove_dot_segments(path: str) -> str:
694+
"""``path`` with its ``.`` and ``..`` segments resolved (RFC 3986 5.2.4).
695+
696+
The result is always absolute, as robots.txt rules are: an empty path
697+
becomes ``/``.
698+
"""
699+
segments = path.split("/")
700+
resolved: list[str] = []
701+
for segment in segments[1:]:
702+
if segment == "..":
703+
if resolved:
704+
resolved.pop()
705+
elif segment != ".":
706+
resolved.append(segment)
707+
# A path ending in a dot segment names a directory: "/a/b/.." is "/a/".
708+
trailing = "/" if resolved and segments[-1] in (".", "..") else ""
709+
return "/" + "/".join(resolved) + trailing
710+
711+
712+
def _resolved_path_url(url: str) -> str:
713+
"""``url`` with its path spelled the way a typical origin resolves it.
714+
715+
robots.txt rules match the path a crawler requests, but an origin such as
716+
nginx or Apache serves ``//private``, ``/x/../private`` and ``/%2Fprivate``
717+
as ``/private``. So the path's separator and dot escapes are decoded,
718+
repeated slashes merged and dot segments removed before rules are matched
719+
against it, as well as against the path as spelled. The query is kept as
720+
it is, and matching stays case-sensitive (RFC 9309 2.2.2).
721+
"""
722+
parts = urlsplit(url)
723+
path = _ENCODED_DOT.sub(".", _ENCODED_SLASH.sub("/", parts.path))
724+
path = _remove_dot_segments(_REPEATED_SLASHES.sub("/", path))
725+
return urlunsplit(parts._replace(path=path))
726+
727+
663728
async def check_may_autonomously_fetch_url(
664729
url: str,
665730
user_agent: str,
@@ -706,7 +771,12 @@ async def check_may_autonomously_fetch_url(
706771
line for line in robot_txt.splitlines() if not line.strip().startswith("#")
707772
)
708773
robot_parser = Protego.parse(processed_robot_txt)
709-
if not robot_parser.can_fetch(str(url), user_agent):
774+
# Refused when either spelling of the path is disallowed: resolving the
775+
# path only ever adds refusals.
776+
if not (
777+
robot_parser.can_fetch(url, user_agent)
778+
and robot_parser.can_fetch(_resolved_path_url(url), user_agent)
779+
):
710780
raise ToolError(
711781
f"The sites robots.txt ({robot_txt_url}), specifies that autonomous fetching of this page is not allowed, "
712782
f"<useragent>{user_agent}</useragent>\n"

‎src/fetch/tests/test_protocol.py‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -457,6 +457,108 @@ async def test_ignore_robots_txt_skips_the_check(web: FakeWeb) -> None:
457457
assert web.urls() == [PAGE]
458458

459459

460+
PRIVATE_ROBOTS = "User-agent: *\nDisallow: /private"
461+
462+
463+
@pytest.mark.parametrize(
464+
"path",
465+
[
466+
# An origin that merges slashes, decodes %2F, or removes dot segments
467+
# serves /private for each of these spellings.
468+
"//private",
469+
"///private",
470+
"/./private",
471+
"/x/../private",
472+
"/%2Fprivate",
473+
"/%2fprivate",
474+
"/a%2f..%2fprivate",
475+
"/%2e/private",
476+
"/x/%2E%2E/private",
477+
"//private//report",
478+
],
479+
)
480+
async def test_robots_disallow_matches_the_resolved_path(
481+
web: FakeWeb, path: str
482+
) -> None:
483+
url = f"https://example.com{path}"
484+
web.add(ROBOTS, plain(PRIVATE_ROBOTS))
485+
web.add(url, plain("private-page"))
486+
async with connect() as (session, _):
487+
result = await call(session, {"url": url})
488+
assert wire(result)["isError"] is True
489+
text = text_of(result)
490+
assert text.startswith(
491+
f"{TOOL_ERROR}The sites robots.txt ({ROBOTS}), specifies that "
492+
"autonomous fetching of this page is not allowed, "
493+
)
494+
assert "private-page" not in text
495+
assert web.urls() == [ROBOTS]
496+
497+
498+
@pytest.mark.parametrize(
499+
"location",
500+
[
501+
"/%2e%2e/private",
502+
"/%2E/private",
503+
"/x/..%2Fprivate",
504+
"/x/%2e%2e//private",
505+
"/x//%2e%2e/private",
506+
],
507+
)
508+
async def test_redirect_to_a_disallowed_path_spelled_otherwise_is_refused(
509+
web: FakeWeb, location: str
510+
) -> None:
511+
web.add(ROBOTS, plain(PRIVATE_ROBOTS))
512+
web.add(PAGE, httpx.Response(302, headers={"location": location}))
513+
web.add(f"https://example.com{location}", plain("private-page"))
514+
async with connect() as (session, _):
515+
result = await call(session, {"url": PAGE})
516+
assert wire(result)["isError"] is True
517+
assert "private-page" not in text_of(result)
518+
assert web.urls() == [ROBOTS, PAGE, ROBOTS]
519+
520+
521+
@pytest.mark.parametrize(
522+
"path",
523+
[
524+
# Resolving never widens a rule: these resolve to allowed paths.
525+
"//public",
526+
"/%2Fpublic",
527+
# Only separator and dot escapes are decoded; %20 stays as it is.
528+
"/pri%20vate",
529+
# Matching stays case-sensitive (RFC 9309 2.2.2).
530+
"//Private",
531+
],
532+
)
533+
async def test_robots_allows_a_path_that_resolves_outside_the_rule(
534+
web: FakeWeb, path: str
535+
) -> None:
536+
url = f"https://example.com{path}"
537+
web.add(ROBOTS, plain(PRIVATE_ROBOTS))
538+
web.add(url, plain("public-page"))
539+
async with connect() as (session, _):
540+
result = await call(session, {"url": url})
541+
assert wire(result)["isError"] is False
542+
assert "public-page" in text_of(result)
543+
assert web.urls() == [ROBOTS, url]
544+
545+
546+
async def test_robots_disallow_still_matches_the_literal_spelling(
547+
web: FakeWeb,
548+
) -> None:
549+
# A rule written for the unresolved spelling still refuses it, though the
550+
# path resolves somewhere the rule does not cover. A redirect carries the
551+
# spelling as it is; a tool argument would have its dots resolved first.
552+
location = "/a/%2e%2e/public"
553+
web.add(ROBOTS, plain("User-agent: *\nDisallow: /a/"))
554+
web.add(PAGE, httpx.Response(302, headers={"location": location}))
555+
web.add(f"https://example.com{location}", plain("public-page"))
556+
async with connect() as (session, _):
557+
result = await call(session, {"url": PAGE})
558+
assert wire(result)["isError"] is True
559+
assert web.urls() == [ROBOTS, PAGE, ROBOTS]
560+
561+
460562
OTHER_ROBOTS = "https://other.example.org/robots.txt"
461563
OTHER_PAGE = "https://other.example.org/secret"
462564

@@ -858,6 +960,14 @@ async def test_redirect_to_private_address_is_refused(web: FakeWeb) -> None:
858960
"2001:2::1",
859961
# Multicast, which is_global does not exclude.
860962
"224.0.0.251",
963+
# Deprecated site-local (RFC 3879), which is_global reads as global.
964+
"fec0::1",
965+
# IPv6 documentation (RFC 9637), missing from older is_global tables.
966+
"3fff::1",
967+
# Outside IPv6 global unicast (2000::/3): SRv6 SIDs, IETF-reserved.
968+
"5f00::1",
969+
"feff::1",
970+
"4000::1",
861971
],
862972
)
863973
async def test_private_address_is_refused_before_any_request(

‎src/fetch/tests/test_translated_addresses.py‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
"""The private-address guard on IPv6 forms that carry an IPv4 address (#5181).
22
3-
Two /96 prefixes embed an IPv4 in their low 32 bits, and Python's
4-
``is_global`` reads both as global whatever that IPv4 is:
3+
Three /96 prefixes embed an IPv4 in their low 32 bits, and Python's
4+
``is_global`` reads each as global whatever that IPv4 is:
55
66
- the well-known NAT64 prefix ``64:ff9b::/96`` (RFC 6052), which a NAT64
77
gateway forwards to the embedded IPv4, so the guard judges it by that IPv4
88
(fixed by #5238; pinned here so #5181 is covered end to end);
9+
- the IPv4-translated prefix ``::ffff:0:0:0/96`` (RFC 2765, RFC 6145), which a
10+
stateless translator forwards to the embedded IPv4, so it is judged by that
11+
IPv4 the same way;
912
- the IPv4-compatible prefix ``::/96`` (RFC 4291 2.5.5.1), deprecated and
1013
never routed, so the guard refuses it outright, even around a public IPv4.
1114
@@ -33,6 +36,9 @@
3336
# Well-known NAT64 (64:ff9b::/96) addresses wrapping non-public IPv4s.
3437
NAT64_NON_PUBLIC = ["64:ff9b::7f00:1", "64:ff9b::a9fe:a9fe", "64:ff9b::a00:1"]
3538

39+
# IPv4-translated (::ffff:0:0:0/96, RFC 2765) addresses wrapping non-public IPv4s.
40+
TRANSLATED_NON_PUBLIC = ["::ffff:0:7f00:1", "::ffff:0:a9fe:a9fe", "::ffff:0:a00:1"]
41+
3642

3743
def wire(result: CallToolResult) -> dict[str, Any]:
3844
return result.model_dump(
@@ -46,7 +52,9 @@ def text_of(result: CallToolResult) -> str:
4652
return data["content"][0]["text"]
4753

4854

49-
@pytest.mark.parametrize("address", IPV4_COMPATIBLE + NAT64_NON_PUBLIC)
55+
@pytest.mark.parametrize(
56+
"address", IPV4_COMPATIBLE + NAT64_NON_PUBLIC + TRANSLATED_NON_PUBLIC
57+
)
5058
async def test_ip_literal_is_refused_before_any_request(
5159
web: FakeWeb, address: str
5260
) -> None:
@@ -62,7 +70,9 @@ async def test_ip_literal_is_refused_before_any_request(
6270
assert web.requests == []
6371

6472

65-
@pytest.mark.parametrize("address", IPV4_COMPATIBLE + NAT64_NON_PUBLIC)
73+
@pytest.mark.parametrize(
74+
"address", IPV4_COMPATIBLE + NAT64_NON_PUBLIC + TRANSLATED_NON_PUBLIC
75+
)
6676
async def test_hostname_resolving_to_the_address_is_refused(
6777
web: FakeWeb, address: str
6878
) -> None:
@@ -94,6 +104,22 @@ async def test_nat64_address_wrapping_a_public_ipv4_is_allowed(
94104
assert web.urls() == [url]
95105

96106

107+
async def test_translated_address_wrapping_a_public_ipv4_is_allowed(
108+
web: FakeWeb,
109+
) -> None:
110+
# Judged by its IPv4 like NAT64: ::ffff:0:808:808 stands for 8.8.8.8.
111+
web.dns["translated.example.com"] = ["::ffff:0:808:808"]
112+
url = "https://translated.example.com/"
113+
web.add(
114+
url,
115+
httpx.Response(200, content=b"ok", headers={"content-type": "text/plain"}),
116+
)
117+
async with connect(ignore_robots_txt=True) as (session, _):
118+
result = await session.call_tool("fetch", {"url": url})
119+
assert wire(result)["isError"] is False
120+
assert web.urls() == [url]
121+
122+
97123
@pytest.mark.parametrize("address", IPV4_COMPATIBLE)
98124
async def test_allow_private_ips_lets_ipv4_compatible_addresses_through(
99125
web: FakeWeb, address: str

0 commit comments

Comments
 (0)