|
| 1 | +import os |
| 2 | +import ssl |
1 | 3 | from typing import Annotated, Tuple |
2 | 4 | from urllib.parse import urlparse, urlunparse |
3 | 5 |
|
|
20 | 22 | from protego import Protego |
21 | 23 | from pydantic import BaseModel, Field, AnyUrl |
22 | 24 |
|
| 25 | +# ============================================================================= |
| 26 | +# SSL CONFIGURATION |
| 27 | +# ============================================================================= |
| 28 | +# Set MCP_FETCH_SSL_VERIFY=false to disable SSL certificate verification. |
| 29 | +# This is useful for internal servers with self-signed certificates. |
| 30 | +# WARNING: Disabling SSL verification reduces security. Only use in trusted environments. |
| 31 | +SSL_VERIFY = os.getenv("MCP_FETCH_SSL_VERIFY", "true").lower() == "true" |
| 32 | + |
23 | 33 | DEFAULT_USER_AGENT_AUTONOMOUS = "ModelContextProtocol/1.0 (Autonomous; +https://github.com/modelcontextprotocol/servers)" |
24 | 34 | DEFAULT_USER_AGENT_MANUAL = "ModelContextProtocol/1.0 (User-Specified; +https://github.com/modelcontextprotocol/servers)" |
25 | 35 |
|
@@ -68,18 +78,41 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: |
68 | 78 | Check if the URL can be fetched by the user agent according to the robots.txt file. |
69 | 79 | Raises a McpError if not. |
70 | 80 | """ |
71 | | - from httpx import AsyncClient, HTTPError |
| 81 | + import httpx |
72 | 82 |
|
73 | 83 | robot_txt_url = get_robots_txt_url(url) |
74 | 84 |
|
75 | | - async with AsyncClient(proxies=proxy_url) as client: |
| 85 | + async with httpx.AsyncClient(proxies=proxy_url, verify=SSL_VERIFY) as client: |
76 | 86 | try: |
77 | 87 | response = await client.get( |
78 | 88 | robot_txt_url, |
79 | 89 | follow_redirects=True, |
80 | 90 | headers={"User-Agent": user_agent}, |
| 91 | + timeout=30, |
81 | 92 | ) |
82 | | - except HTTPError: |
| 93 | + except ssl.SSLError as e: |
| 94 | + raise McpError(ErrorData( |
| 95 | + code=INTERNAL_ERROR, |
| 96 | + message=f"SSL Certificate verification failed for {robot_txt_url}. " |
| 97 | + f"If this is an internal server with a self-signed certificate, " |
| 98 | + f"set MCP_FETCH_SSL_VERIFY=false in your environment. " |
| 99 | + f"Error details: {str(e)}", |
| 100 | + )) |
| 101 | + except httpx.ConnectError as e: |
| 102 | + error_str = str(e).lower() |
| 103 | + if "ssl" in error_str or "certificate" in error_str or "verify" in error_str: |
| 104 | + raise McpError(ErrorData( |
| 105 | + code=INTERNAL_ERROR, |
| 106 | + message=f"SSL Certificate verification failed for {robot_txt_url}. " |
| 107 | + f"If this is an internal server with a self-signed certificate, " |
| 108 | + f"set MCP_FETCH_SSL_VERIFY=false in your environment. " |
| 109 | + f"Error details: {str(e)}", |
| 110 | + )) |
| 111 | + raise McpError(ErrorData( |
| 112 | + code=INTERNAL_ERROR, |
| 113 | + message=f"Failed to connect to {robot_txt_url}: {str(e)}", |
| 114 | + )) |
| 115 | + except httpx.HTTPError: |
83 | 116 | raise McpError(ErrorData( |
84 | 117 | code=INTERNAL_ERROR, |
85 | 118 | message=f"Failed to fetch robots.txt {robot_txt_url} due to a connection issue", |
@@ -114,17 +147,39 @@ async def fetch_url( |
114 | 147 | """ |
115 | 148 | Fetch the URL and return the content in a form ready for the LLM, as well as a prefix string with status information. |
116 | 149 | """ |
117 | | - from httpx import AsyncClient, HTTPError |
| 150 | + import httpx |
118 | 151 |
|
119 | | - async with AsyncClient(proxies=proxy_url) as client: |
| 152 | + async with httpx.AsyncClient(proxies=proxy_url, verify=SSL_VERIFY) as client: |
120 | 153 | try: |
121 | 154 | response = await client.get( |
122 | 155 | url, |
123 | 156 | follow_redirects=True, |
124 | 157 | headers={"User-Agent": user_agent}, |
125 | 158 | timeout=30, |
126 | 159 | ) |
127 | | - except HTTPError as e: |
| 160 | + except ssl.SSLError as e: |
| 161 | + raise McpError(ErrorData( |
| 162 | + code=INTERNAL_ERROR, |
| 163 | + message=f"SSL Certificate verification failed for {url}. " |
| 164 | + f"If this is an internal server with a self-signed certificate, " |
| 165 | + f"set MCP_FETCH_SSL_VERIFY=false in your environment. " |
| 166 | + f"Error details: {str(e)}", |
| 167 | + )) |
| 168 | + except httpx.ConnectError as e: |
| 169 | + error_str = str(e).lower() |
| 170 | + if "ssl" in error_str or "certificate" in error_str or "verify" in error_str: |
| 171 | + raise McpError(ErrorData( |
| 172 | + code=INTERNAL_ERROR, |
| 173 | + message=f"SSL Certificate verification failed for {url}. " |
| 174 | + f"If this is an internal server with a self-signed certificate, " |
| 175 | + f"set MCP_FETCH_SSL_VERIFY=false in your environment. " |
| 176 | + f"Error details: {str(e)}", |
| 177 | + )) |
| 178 | + raise McpError(ErrorData( |
| 179 | + code=INTERNAL_ERROR, |
| 180 | + message=f"Failed to connect to {url}: {str(e)}", |
| 181 | + )) |
| 182 | + except httpx.HTTPError as e: |
128 | 183 | raise McpError(ErrorData(code=INTERNAL_ERROR, message=f"Failed to fetch {url}: {e!r}")) |
129 | 184 | if response.status_code >= 400: |
130 | 185 | raise McpError(ErrorData( |
|
0 commit comments