Skip to content

Commit 7464deb

Browse files
committed
feat(fetch): add opt-in host allowlisting via --allowed-hosts
Add a host allowlist for the fetch server, addressing the SSRF surface tracked in modelcontextprotocol#2317 without changing default behavior: - New --allowed-hosts flag (exact hosts, case-insensitive; *.example.com wildcards covering the bare domain and any subdomain; IP literals). - Enforced on the initial request, the robots.txt pre-check, and every redirect hop: redirects are now followed manually (bounded at 20, same as httpx's default) and each hop is re-validated, so a redirect from an allowed host can no longer bounce a fetch to a disallowed host. - URL validation parses with httpx.URL — the same parser used to connect — so the validated host is always the host being connected. - Denials fail closed with an error that does not echo the allowlist. Default-IP-blocking is intentionally left to the separate default-deny proposal; with no flag set, behavior is identical to before.
1 parent d73f99e commit 7464deb

4 files changed

Lines changed: 565 additions & 14 deletions

File tree

‎src/fetch/README.md‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,30 @@ This can be customized by adding the argument `--user-agent=YourUserAgent` to th
174174

175175
The server can be configured to use a proxy by using the `--proxy-url` argument.
176176

177+
### Customization - Allowed hosts
178+
179+
By default the server can fetch any host (see the security caution above). To restrict which hosts the server may connect to, add the `--allowed-hosts` argument with one or more entries to the `args` list in the configuration:
180+
181+
- `example.com` allows exactly `example.com` (matching is case-insensitive)
182+
- `*.example.com` allows `example.com` itself and any subdomain, e.g. `api.example.com`
183+
184+
Matching is by hostname only: an entry allows that host on any port, and IPv6 addresses are listed without brackets (e.g. `--allowed-hosts ::1`).
185+
186+
The allowlist is enforced on the initial request, on the `robots.txt` pre-check, and on every redirect hop, so a redirect from an allowed host cannot bounce the fetch to a disallowed host. Requests to any other host fail with an error explaining that the host is not allowlisted.
187+
188+
Note that the allowlist matches hostnames, not the IP addresses they resolve to: an allowlisted domain whose DNS records point at internal addresses can still be fetched (this is what makes it possible to deliberately allowlist internal hosts).
189+
190+
```json
191+
{
192+
"mcpServers": {
193+
"fetch": {
194+
"command": "uvx",
195+
"args": ["mcp-server-fetch", "--allowed-hosts", "example.com", "*.github.com"]
196+
}
197+
}
198+
}
199+
```
200+
177201
## Windows Configuration
178202

179203
If you're experiencing timeout issues on Windows, you may need to set the `PYTHONIOENCODING` environment variable to ensure proper character encoding:

‎src/fetch/src/mcp_server_fetch/__init__.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,16 @@ def main():
1616
help="Ignore robots.txt restrictions",
1717
)
1818
parser.add_argument("--proxy-url", type=str, help="Proxy URL to use for requests")
19+
parser.add_argument(
20+
"--allowed-hosts",
21+
type=str,
22+
nargs="+",
23+
metavar="HOST",
24+
help="Only allow fetching these hosts (exact names like example.com or wildcards like *.example.com, which also covers example.com itself). Applies to the initial URL and every redirect hop. If omitted, all hosts are allowed.",
25+
)
1926

2027
args = parser.parse_args()
21-
asyncio.run(serve(args.user_agent, args.ignore_robots_txt, args.proxy_url))
28+
asyncio.run(serve(args.user_agent, args.ignore_robots_txt, args.proxy_url, allowed_hosts=args.allowed_hosts))
2229

2330

2431
if __name__ == "__main__":

‎src/fetch/src/mcp_server_fetch/server.py‎

Lines changed: 145 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
from typing import Annotated, Tuple
2-
from urllib.parse import urlparse, urlunparse
1+
from typing import TYPE_CHECKING, Annotated, Any, Tuple
2+
from urllib.parse import urljoin, urlparse, urlunparse
33

44
import markdownify
55
import readabilipy.simple_json
@@ -20,9 +20,133 @@
2020
from protego import Protego
2121
from pydantic import BaseModel, Field, AnyUrl
2222

23+
if TYPE_CHECKING:
24+
from httpx import AsyncClient, Response
25+
2326
DEFAULT_USER_AGENT_AUTONOMOUS = "ModelContextProtocol/1.0 (Autonomous; +https://github.com/modelcontextprotocol/servers)"
2427
DEFAULT_USER_AGENT_MANUAL = "ModelContextProtocol/1.0 (User-Specified; +https://github.com/modelcontextprotocol/servers)"
2528

29+
REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})
30+
MAX_REDIRECTS = 20
31+
32+
33+
def is_host_allowed(hostname: str | None, allowed_hosts: list[str] | None) -> bool:
34+
"""Check whether a hostname is permitted by the configured allowlist.
35+
36+
Args:
37+
hostname: Hostname taken from the request URL
38+
allowed_hosts: Allowlist entries. ``None`` disables the allowlist (every
39+
host is allowed). An entry is either an exact host (``example.com``)
40+
or a wildcard (``*.example.com``, which matches ``example.com`` itself
41+
and any subdomain). Matching is case-insensitive.
42+
43+
Returns:
44+
True if the host may be fetched, False otherwise
45+
"""
46+
if allowed_hosts is None:
47+
return True
48+
if not hostname:
49+
return False
50+
hostname = hostname.lower().rstrip(".")
51+
for entry in allowed_hosts:
52+
entry = entry.strip().lower().rstrip(".").strip("[]")
53+
if entry.startswith("*."):
54+
suffix = entry[2:]
55+
if hostname == suffix or hostname.endswith("." + suffix):
56+
return True
57+
elif hostname == entry:
58+
return True
59+
return False
60+
61+
62+
def validate_url_allowed(url: str, allowed_hosts: list[str] | None) -> None:
63+
"""Validate a URL's host against the allowlist.
64+
65+
The URL is parsed with httpx — the same parser that will be used to
66+
connect — so the validated host is always the host that gets connected.
67+
68+
Raises:
69+
McpError: If the URL has no hostname or its host is not allowlisted
70+
"""
71+
if allowed_hosts is None:
72+
return
73+
from httpx import URL, InvalidURL
74+
75+
try:
76+
hostname = URL(url).host
77+
except InvalidURL:
78+
hostname = None
79+
if not hostname:
80+
raise McpError(ErrorData(
81+
code=INVALID_PARAMS,
82+
message=f"Invalid URL: could not determine a hostname for {url}",
83+
))
84+
if not is_host_allowed(hostname, allowed_hosts):
85+
raise McpError(ErrorData(
86+
code=INTERNAL_ERROR,
87+
message=f"Fetching '{hostname}' is not allowed: this server is configured with a host allowlist (--allowed-hosts) and this host is not on it. The user can adjust the server configuration if this host should be accessible.",
88+
))
89+
90+
91+
async def _get_following_redirects(
92+
client: "AsyncClient",
93+
url: str,
94+
*,
95+
user_agent: str,
96+
allowed_hosts: list[str] | None,
97+
timeout: float | None = None,
98+
) -> "Response":
99+
"""GET a URL, following redirects manually and re-validating every hop.
100+
101+
Redirects are followed by hand (instead of httpx's follow_redirects) so that
102+
each redirect target is checked against the allowlist before connecting;
103+
otherwise a 302 from an allowed host could bounce the fetch to any host.
104+
105+
Args:
106+
client: httpx.AsyncClient to use
107+
url: Initial URL to fetch
108+
user_agent: User-Agent header value
109+
allowed_hosts: Allowlist applied to the initial URL and every redirect hop
110+
timeout: Optional per-request timeout in seconds (httpx default if None)
111+
112+
Returns:
113+
The final (non-redirect) httpx.Response
114+
115+
Raises:
116+
McpError: If a hop is not allowlisted or the redirect limit is exceeded
117+
"""
118+
request_kwargs: dict[str, Any] = {"follow_redirects": False, "headers": {"User-Agent": user_agent}}
119+
if timeout is not None:
120+
request_kwargs["timeout"] = timeout
121+
122+
current_url = url
123+
redirects_remaining = MAX_REDIRECTS
124+
while True:
125+
validate_url_allowed(current_url, allowed_hosts)
126+
response = await client.get(current_url, **request_kwargs)
127+
if response.status_code not in REDIRECT_STATUS_CODES:
128+
return response
129+
location = response.headers.get("location")
130+
if location is None:
131+
# A redirect status without a Location header is not followable;
132+
# the response is used as-is (matches httpx's follow_redirects).
133+
return response
134+
if redirects_remaining <= 0:
135+
raise McpError(ErrorData(
136+
code=INTERNAL_ERROR,
137+
message=f"Failed to fetch {url}: exceeded the limit of {MAX_REDIRECTS} redirects",
138+
))
139+
redirects_remaining -= 1
140+
# An empty Location redirects to the same URL (matching httpx), so a
141+
# redirect loop — self-inflicted or otherwise — hits the limit above.
142+
try:
143+
current_url = urljoin(str(response.url), location)
144+
except ValueError:
145+
raise McpError(ErrorData(
146+
code=INTERNAL_ERROR,
147+
message=f"Failed to fetch {url}: redirect target {location!r} is not a valid URL",
148+
))
149+
26150

27151
def extract_content_from_html(html: str) -> str:
28152
"""Extract and convert HTML content to Markdown format.
@@ -63,21 +187,23 @@ def get_robots_txt_url(url: str) -> str:
63187
return robots_url
64188

65189

66-
async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: str | None = None) -> None:
190+
async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: str | None = None, allowed_hosts: list[str] | None = None) -> None:
67191
"""
68192
Check if the URL can be fetched by the user agent according to the robots.txt file.
69193
Raises a McpError if not.
70194
"""
71195
from httpx import AsyncClient, HTTPError
72196

197+
validate_url_allowed(url, allowed_hosts)
73198
robot_txt_url = get_robots_txt_url(url)
74199

75200
async with AsyncClient(proxy=proxy_url) as client:
76201
try:
77-
response = await client.get(
202+
response = await _get_following_redirects(
203+
client,
78204
robot_txt_url,
79-
follow_redirects=True,
80-
headers={"User-Agent": user_agent},
205+
user_agent=user_agent,
206+
allowed_hosts=allowed_hosts,
81207
)
82208
except HTTPError:
83209
raise McpError(ErrorData(
@@ -109,19 +235,22 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url:
109235

110236

111237
async def fetch_url(
112-
url: str, user_agent: str, force_raw: bool = False, proxy_url: str | None = None
238+
url: str, user_agent: str, force_raw: bool = False, proxy_url: str | None = None, allowed_hosts: list[str] | None = None
113239
) -> Tuple[str, str]:
114240
"""
115241
Fetch the URL and return the content in a form ready for the LLM, as well as a prefix string with status information.
116242
"""
117243
from httpx import AsyncClient, HTTPError
118244

245+
validate_url_allowed(url, allowed_hosts)
246+
119247
async with AsyncClient(proxy=proxy_url) as client:
120248
try:
121-
response = await client.get(
249+
response = await _get_following_redirects(
250+
client,
122251
url,
123-
follow_redirects=True,
124-
headers={"User-Agent": user_agent},
252+
user_agent=user_agent,
253+
allowed_hosts=allowed_hosts,
125254
timeout=30,
126255
)
127256
except HTTPError as e:
@@ -182,13 +311,16 @@ async def serve(
182311
custom_user_agent: str | None = None,
183312
ignore_robots_txt: bool = False,
184313
proxy_url: str | None = None,
314+
allowed_hosts: list[str] | None = None,
185315
) -> None:
186316
"""Run the fetch MCP server.
187317
188318
Args:
189319
custom_user_agent: Optional custom User-Agent string to use for requests
190320
ignore_robots_txt: Whether to ignore robots.txt restrictions
191321
proxy_url: Optional proxy URL to use for requests
322+
allowed_hosts: Optional host allowlist; when set, only these hosts
323+
(exact names or *.example.com wildcards) may be fetched
192324
"""
193325
server = Server("mcp-fetch")
194326
user_agent_autonomous = custom_user_agent or DEFAULT_USER_AGENT_AUTONOMOUS
@@ -232,10 +364,10 @@ async def call_tool(name, arguments: dict) -> list[TextContent]:
232364
raise McpError(ErrorData(code=INVALID_PARAMS, message="URL is required"))
233365

234366
if not ignore_robots_txt:
235-
await check_may_autonomously_fetch_url(url, user_agent_autonomous, proxy_url)
367+
await check_may_autonomously_fetch_url(url, user_agent_autonomous, proxy_url, allowed_hosts=allowed_hosts)
236368

237369
content, prefix = await fetch_url(
238-
url, user_agent_autonomous, force_raw=args.raw, proxy_url=proxy_url
370+
url, user_agent_autonomous, force_raw=args.raw, proxy_url=proxy_url, allowed_hosts=allowed_hosts
239371
)
240372
original_length = len(content)
241373
if args.start_index >= original_length:
@@ -262,7 +394,7 @@ async def get_prompt(name: str, arguments: dict | None) -> GetPromptResult:
262394
url = arguments["url"]
263395

264396
try:
265-
content, prefix = await fetch_url(url, user_agent_manual, proxy_url=proxy_url)
397+
content, prefix = await fetch_url(url, user_agent_manual, proxy_url=proxy_url, allowed_hosts=allowed_hosts)
266398
# TODO: after SDK bug is addressed, don't catch the exception
267399
except McpError as e:
268400
return GetPromptResult(

0 commit comments

Comments
 (0)