Skip to content

Commit defc050

Browse files
cliffhallhe-yufengclaude
committed
fix(fetch): support SOCKS proxies and report bad proxy settings as tool errors
- Depend on httpx[socks], so a SOCKS5 proxy works without installing socksio by hand (modelcontextprotocol#1401). - Treat the socks:// alias, which httpx rejects, as socks5://, both for --proxy-url and for the HTTP(S)_PROXY / ALL_PROXY environment variables (modelcontextprotocol#767). The environment is normalized in place, so httpx still applies NO_PROXY. - Build the httpx client through make_client(), which turns a proxy setting httpx cannot use into an McpError naming the proxy settings, instead of httpx's bare exception text. The pinning test in test_protocol.py now asserts the new error text, and its KNOWN BUG modelcontextprotocol#767, modelcontextprotocol#1401 marker is gone. Ported from modelcontextprotocol#4340 by Yufeng He (socks:// alias and httpx[socks]), adapted to v2/main. Closes modelcontextprotocol#767 Closes modelcontextprotocol#1401 Co-authored-by: Yufeng He <[email protected]> Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: cliffhall <[email protected]>
1 parent 142bb84 commit defc050

6 files changed

Lines changed: 151 additions & 15 deletions

File tree

‎README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,11 @@ This can be customized by adding the argument `--user-agent=YourUserAgent` to th
174174

175175
The server can be configured to use a proxy by using the `--proxy-url` argument.
176176

177+
Without `--proxy-url`, the server uses the proxy in the standard `HTTP_PROXY`, `HTTPS_PROXY` and `ALL_PROXY`
178+
environment variables (honoring `NO_PROXY`). Both `http(s)://` and SOCKS5 proxies are supported (SOCKS support is
179+
installed with the server). A `socks://` proxy URL, as many desktop proxy settings export it, is treated as
180+
`socks5://`. A proxy setting the server cannot use makes each fetch fail with a tool error that says so.
181+
177182
## Windows Configuration
178183

179184
If you're experiencing timeout issues on Windows, you may need to set the `PYTHONIOENCODING` environment variable to ensure proper character encoding:

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ classifiers = [
1616
"Programming Language :: Python :: 3.10",
1717
]
1818
dependencies = [
19-
"httpx>=0.27",
19+
"httpx[socks]>=0.27",
2020
"markdownify>=0.13.1",
2121
"mcp>=1.29.0,<2",
2222
"protego>=0.3.1",

‎src/mcp_server_fetch/server.py‎

Lines changed: 63 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
from typing import Annotated, Tuple
1+
import os
2+
from typing import TYPE_CHECKING, Annotated, Tuple
23
from urllib.parse import urlparse, urlunparse
34

45
import markdownify
@@ -20,6 +21,9 @@
2021
from protego import Protego
2122
from pydantic import BaseModel, Field, AnyUrl
2223

24+
if TYPE_CHECKING:
25+
from httpx import AsyncClient
26+
2327
DEFAULT_USER_AGENT_AUTONOMOUS = "ModelContextProtocol/1.0 (Autonomous; +https://github.com/modelcontextprotocol/servers)"
2428
DEFAULT_USER_AGENT_MANUAL = "ModelContextProtocol/1.0 (User-Specified; +https://github.com/modelcontextprotocol/servers)"
2529

@@ -63,18 +67,69 @@ def get_robots_txt_url(url: str) -> str:
6367
return robots_url
6468

6569

70+
PROXY_ENV_VARS = (
71+
"HTTP_PROXY",
72+
"http_proxy",
73+
"HTTPS_PROXY",
74+
"https_proxy",
75+
"ALL_PROXY",
76+
"all_proxy",
77+
)
78+
79+
80+
def normalize_proxy_url(proxy_url: str) -> str:
81+
"""Rewrite the ``socks://`` alias, which httpx rejects, to ``socks5://``.
82+
83+
Desktop proxy settings often export ``socks://host:port``; httpx only
84+
accepts ``socks5://``. Every other value is returned unchanged.
85+
"""
86+
if proxy_url.lower().startswith("socks://"):
87+
return "socks5://" + proxy_url[len("socks://") :]
88+
return proxy_url
89+
90+
91+
def normalize_proxy_env() -> None:
92+
"""Apply ``normalize_proxy_url`` to the proxy variables httpx reads."""
93+
for name in PROXY_ENV_VARS:
94+
value = os.environ.get(name)
95+
if value:
96+
os.environ[name] = normalize_proxy_url(value)
97+
98+
99+
def make_client(proxy_url: str | None) -> "AsyncClient":
100+
"""Build the httpx client, turning a bad proxy setting into a tool error.
101+
102+
httpx validates the proxy (``--proxy-url`` or the proxy environment
103+
variables) when the client is constructed, before any request, so this
104+
failure is not an ``HTTPError``.
105+
"""
106+
from httpx import AsyncClient
107+
108+
try:
109+
return AsyncClient(proxy=proxy_url)
110+
except (ValueError, ImportError) as e:
111+
raise McpError(
112+
ErrorData(
113+
code=INTERNAL_ERROR,
114+
message=f"Failed to set up the HTTP client, check the proxy "
115+
f"configuration (--proxy-url or the HTTP_PROXY, HTTPS_PROXY and "
116+
f"ALL_PROXY environment variables): {e}",
117+
)
118+
)
119+
120+
66121
async def check_may_autonomously_fetch_url(
67122
url: str, user_agent: str, proxy_url: str | None = None
68123
) -> None:
69124
"""
70125
Check if the URL can be fetched by the user agent according to the robots.txt file.
71126
Raises a McpError if not.
72127
"""
73-
from httpx import AsyncClient, HTTPError
128+
from httpx import HTTPError
74129

75130
robot_txt_url = get_robots_txt_url(url)
76131

77-
async with AsyncClient(proxy=proxy_url) as client:
132+
async with make_client(proxy_url) as client:
78133
try:
79134
response = await client.get(
80135
robot_txt_url,
@@ -122,9 +177,9 @@ async def fetch_url(
122177
"""
123178
Fetch the URL and return the content in a form ready for the LLM, as well as a prefix string with status information.
124179
"""
125-
from httpx import AsyncClient, HTTPError
180+
from httpx import HTTPError
126181

127-
async with AsyncClient(proxy=proxy_url) as client:
182+
async with make_client(proxy_url) as client:
128183
try:
129184
response = await client.get(
130185
url,
@@ -202,6 +257,9 @@ async def serve(
202257
ignore_robots_txt: Whether to ignore robots.txt restrictions
203258
proxy_url: Optional proxy URL to use for requests
204259
"""
260+
if proxy_url:
261+
proxy_url = normalize_proxy_url(proxy_url)
262+
normalize_proxy_env()
205263
server = Server("mcp-fetch")
206264
user_agent_autonomous = custom_user_agent or DEFAULT_USER_AGENT_AUTONOMOUS
207265
user_agent_manual = custom_user_agent or DEFAULT_USER_AGENT_MANUAL

‎tests/test_protocol.py‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -672,12 +672,23 @@ async def test_no_proxy_by_default(web: FakeWeb) -> None:
672672
assert web.client_kwargs == [{"proxy": None}]
673673

674674

675-
# KNOWN BUG #767, #1401: pins current (wrong) behavior; the fix changes this assertion.
675+
async def test_socks_alias_proxy_url_is_passed_as_socks5(web: FakeWeb) -> None:
676+
# #767: httpx rejects the common socks:// alias, so it reaches the client
677+
# as socks5:// (supported through httpx[socks], #1401).
678+
web.add(ROBOTS, plain("", status=404))
679+
web.add(PAGE, plain("ok"))
680+
async with connect(proxy_url="socks://127.0.0.1:2080") as (session, _):
681+
result = await call(session, {"url": PAGE})
682+
assert wire(result)["isError"] is False
683+
socks5 = "socks5://127.0.0.1:2080"
684+
assert web.client_kwargs == [{"proxy": socks5}, {"proxy": socks5}]
685+
686+
676687
@pytest.mark.parametrize("ignore_robots_txt", [False, True])
677-
async def test_bad_proxy_config_surfaces_as_bare_text(ignore_robots_txt: bool) -> None:
678-
# Characterizes #767 and #1401: a bad proxy URL makes httpx.AsyncClient()
679-
# raise at construction, outside the `except HTTPError`, so the client gets
680-
# httpx's bare exception text with no mention of fetch or the proxy flag.
688+
async def test_bad_proxy_config_is_a_tool_error(ignore_robots_txt: bool) -> None:
689+
# #767 and #1401: a proxy httpx cannot use makes httpx.AsyncClient() raise
690+
# at construction, before any request. The client gets an isError result
691+
# that names the proxy settings, not httpx's bare exception text.
681692
# No `web` fixture: the real AsyncClient fails before any request is made.
682693
async with connect(
683694
proxy_url="ftp://proxy.example.com", ignore_robots_txt=ignore_robots_txt
@@ -687,7 +698,10 @@ async def test_bad_proxy_config_surfaces_as_bare_text(ignore_robots_txt: bool) -
687698
"content": [
688699
{
689700
"type": "text",
690-
"text": "Unknown scheme for proxy URL URL('ftp://proxy.example.com')",
701+
"text": "Failed to set up the HTTP client, check the proxy "
702+
"configuration (--proxy-url or the HTTP_PROXY, HTTPS_PROXY and "
703+
"ALL_PROXY environment variables): "
704+
"Unknown scheme for proxy URL URL('ftp://proxy.example.com')",
691705
}
692706
],
693707
"isError": True,
@@ -801,7 +815,7 @@ async def test_get_prompt_does_not_validate_the_url(
801815
# a "Failed to fetch" prompt message rather than a validation error.
802816
# No `web` fixture: the real transport rejects the URL before any I/O.
803817
# Proxy variables are cleared so the real client cannot pick up one from
804-
# the environment (a socks:// proxy would fail on the missing socksio).
818+
# the environment and send the request through it.
805819
for name in list(os.environ):
806820
if name.lower() in ("http_proxy", "https_proxy", "all_proxy"):
807821
monkeypatch.delenv(name)

‎tests/test_server.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,17 @@
44
test_protocol.py.
55
"""
66

7+
import os
8+
9+
import httpx
710
import pytest
811

912
from mcp_server_fetch.server import (
13+
PROXY_ENV_VARS,
1014
extract_content_from_html,
1115
get_robots_txt_url,
16+
normalize_proxy_env,
17+
normalize_proxy_url,
1218
)
1319

1420

@@ -86,3 +92,42 @@ def test_empty_content_returns_error(self):
8692
html = ""
8793
result = extract_content_from_html(html)
8894
assert "<error>" in result
95+
96+
97+
class TestNormalizeProxyUrl:
98+
"""Tests for normalize_proxy_url and normalize_proxy_env (#767)."""
99+
100+
@pytest.mark.parametrize(
101+
("given", "expected"),
102+
[
103+
("socks://127.0.0.1:2080/", "socks5://127.0.0.1:2080/"),
104+
("SOCKS://127.0.0.1:2080", "socks5://127.0.0.1:2080"),
105+
("socks5://127.0.0.1:2080", "socks5://127.0.0.1:2080"),
106+
("http://proxy.example.com:8080", "http://proxy.example.com:8080"),
107+
("ftp://proxy.example.com", "ftp://proxy.example.com"),
108+
],
109+
)
110+
def test_only_the_socks_alias_is_rewritten(self, given, expected):
111+
assert normalize_proxy_url(given) == expected
112+
113+
def test_environment_socks_alias_is_rewritten(self, monkeypatch):
114+
for name in PROXY_ENV_VARS:
115+
monkeypatch.delenv(name, raising=False)
116+
monkeypatch.setenv("ALL_PROXY", "socks://127.0.0.1:2080/")
117+
monkeypatch.setenv("https_proxy", "http://proxy.example.com:8080")
118+
normalize_proxy_env()
119+
assert os.environ["ALL_PROXY"] == "socks5://127.0.0.1:2080/"
120+
assert os.environ["https_proxy"] == "http://proxy.example.com:8080"
121+
assert "HTTP_PROXY" not in os.environ
122+
123+
def test_normalized_socks_environment_builds_a_client(self, monkeypatch):
124+
# #767 and #1401 together: httpx rejects socks:// outright, and a
125+
# socks5:// proxy needs socksio, which httpx[socks] now installs.
126+
for name in PROXY_ENV_VARS:
127+
monkeypatch.delenv(name, raising=False)
128+
monkeypatch.setenv("ALL_PROXY", "socks://127.0.0.1:2080/")
129+
with pytest.raises(ValueError, match="Unknown scheme for proxy URL"):
130+
httpx.AsyncClient()
131+
normalize_proxy_env()
132+
httpx.AsyncClient()
133+
httpx.AsyncClient(proxy="socks5://127.0.0.1:2080/")

‎uv.lock‎

Lines changed: 16 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)