|
1 | | -from typing import Annotated, Tuple |
| 1 | +import os |
| 2 | +from typing import TYPE_CHECKING, Annotated, Tuple |
2 | 3 | from urllib.parse import urlparse, urlunparse |
3 | 4 |
|
4 | 5 | import markdownify |
|
20 | 21 | from protego import Protego |
21 | 22 | from pydantic import BaseModel, Field, AnyUrl |
22 | 23 |
|
| 24 | +if TYPE_CHECKING: |
| 25 | + from httpx import AsyncClient |
| 26 | + |
23 | 27 | DEFAULT_USER_AGENT_AUTONOMOUS = "ModelContextProtocol/1.0 (Autonomous; +https://github.com/modelcontextprotocol/servers)" |
24 | 28 | DEFAULT_USER_AGENT_MANUAL = "ModelContextProtocol/1.0 (User-Specified; +https://github.com/modelcontextprotocol/servers)" |
25 | 29 |
|
@@ -63,18 +67,69 @@ def get_robots_txt_url(url: str) -> str: |
63 | 67 | return robots_url |
64 | 68 |
|
65 | 69 |
|
| 70 | +PROXY_ENV_VARS = ( |
| 71 | + "HTTP_PROXY", |
| 72 | + "http_proxy", |
| 73 | + "HTTPS_PROXY", |
| 74 | + "https_proxy", |
| 75 | + "ALL_PROXY", |
| 76 | + "all_proxy", |
| 77 | +) |
| 78 | + |
| 79 | + |
| 80 | +def normalize_proxy_url(proxy_url: str) -> str: |
| 81 | + """Rewrite the ``socks://`` alias, which httpx rejects, to ``socks5://``. |
| 82 | +
|
| 83 | + Desktop proxy settings often export ``socks://host:port``; httpx only |
| 84 | + accepts ``socks5://``. Every other value is returned unchanged. |
| 85 | + """ |
| 86 | + if proxy_url.lower().startswith("socks://"): |
| 87 | + return "socks5://" + proxy_url[len("socks://") :] |
| 88 | + return proxy_url |
| 89 | + |
| 90 | + |
| 91 | +def normalize_proxy_env() -> None: |
| 92 | + """Apply ``normalize_proxy_url`` to the proxy variables httpx reads.""" |
| 93 | + for name in PROXY_ENV_VARS: |
| 94 | + value = os.environ.get(name) |
| 95 | + if value: |
| 96 | + os.environ[name] = normalize_proxy_url(value) |
| 97 | + |
| 98 | + |
| 99 | +def make_client(proxy_url: str | None) -> "AsyncClient": |
| 100 | + """Build the httpx client, turning a bad proxy setting into a tool error. |
| 101 | +
|
| 102 | + httpx validates the proxy (``--proxy-url`` or the proxy environment |
| 103 | + variables) when the client is constructed, before any request, so this |
| 104 | + failure is not an ``HTTPError``. |
| 105 | + """ |
| 106 | + from httpx import AsyncClient |
| 107 | + |
| 108 | + try: |
| 109 | + return AsyncClient(proxy=proxy_url) |
| 110 | + except (ValueError, ImportError) as e: |
| 111 | + raise McpError( |
| 112 | + ErrorData( |
| 113 | + code=INTERNAL_ERROR, |
| 114 | + message=f"Failed to set up the HTTP client, check the proxy " |
| 115 | + f"configuration (--proxy-url or the HTTP_PROXY, HTTPS_PROXY and " |
| 116 | + f"ALL_PROXY environment variables): {e}", |
| 117 | + ) |
| 118 | + ) |
| 119 | + |
| 120 | + |
66 | 121 | async def check_may_autonomously_fetch_url( |
67 | 122 | url: str, user_agent: str, proxy_url: str | None = None |
68 | 123 | ) -> None: |
69 | 124 | """ |
70 | 125 | Check if the URL can be fetched by the user agent according to the robots.txt file. |
71 | 126 | Raises a McpError if not. |
72 | 127 | """ |
73 | | - from httpx import AsyncClient, HTTPError |
| 128 | + from httpx import HTTPError |
74 | 129 |
|
75 | 130 | robot_txt_url = get_robots_txt_url(url) |
76 | 131 |
|
77 | | - async with AsyncClient(proxy=proxy_url) as client: |
| 132 | + async with make_client(proxy_url) as client: |
78 | 133 | try: |
79 | 134 | response = await client.get( |
80 | 135 | robot_txt_url, |
@@ -122,9 +177,9 @@ async def fetch_url( |
122 | 177 | """ |
123 | 178 | Fetch the URL and return the content in a form ready for the LLM, as well as a prefix string with status information. |
124 | 179 | """ |
125 | | - from httpx import AsyncClient, HTTPError |
| 180 | + from httpx import HTTPError |
126 | 181 |
|
127 | | - async with AsyncClient(proxy=proxy_url) as client: |
| 182 | + async with make_client(proxy_url) as client: |
128 | 183 | try: |
129 | 184 | response = await client.get( |
130 | 185 | url, |
@@ -202,6 +257,9 @@ async def serve( |
202 | 257 | ignore_robots_txt: Whether to ignore robots.txt restrictions |
203 | 258 | proxy_url: Optional proxy URL to use for requests |
204 | 259 | """ |
| 260 | + if proxy_url: |
| 261 | + proxy_url = normalize_proxy_url(proxy_url) |
| 262 | + normalize_proxy_env() |
205 | 263 | server = Server("mcp-fetch") |
206 | 264 | user_agent_autonomous = custom_user_agent or DEFAULT_USER_AGENT_AUTONOMOUS |
207 | 265 | user_agent_manual = custom_user_agent or DEFAULT_USER_AGENT_MANUAL |
|
0 commit comments