Skip to content

Commit 1fe5467

Browse files
authored
Python: Make MCP connection-loss recovery ownership-aware (#9583)
* Python: Make MCP recovery ownership-aware * Refactor MCP connection recovery * Address MCP recovery review feedback
1 parent 0200599 commit 1fe5467

3 files changed

Lines changed: 302 additions & 84 deletions

File tree

‎python/packages/core/agent_framework/_mcp.py‎

Lines changed: 111 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -2279,6 +2279,44 @@ def _set_server_capabilities(self, capabilities: types.ServerCapabilities | None
22792279
self._supports_prompts = getattr(capabilities, "prompts", None) is not None
22802280
self._supports_logging = getattr(capabilities, "logging", None) is not None
22812281

2282+
async def _recover_connection_after_loss(
2283+
self,
2284+
*,
2285+
operation: str,
2286+
inner_exception: Exception,
2287+
load_configured: bool,
2288+
) -> None:
2289+
"""Recreate a framework-owned connection or fail clearly for a caller-owned connection."""
2290+
connection = self._connection
2291+
if connection is not None and not connection.is_framework_owned:
2292+
connection_type = "Client" if connection.client is not None else "ClientSession"
2293+
raise _mcp_tool_execution_exception(
2294+
f"MCP connection lost during {operation}. The active {connection_type} is caller-owned and cannot "
2295+
"be recreated by Agent Framework. Create a replacement connection and MCP tool instance.",
2296+
inner_exception=inner_exception,
2297+
) from inner_exception
2298+
2299+
logger.info("MCP connection was lost during %s. Reconnecting...", operation)
2300+
try:
2301+
if load_configured:
2302+
await self.connect(reset=True)
2303+
else:
2304+
await self._reconnect_without_loading()
2305+
except Exception as reconnect_ex:
2306+
raise _mcp_tool_execution_exception(
2307+
f"Failed to reconnect to MCP server after {operation}.",
2308+
inner_exception=reconnect_ex,
2309+
) from reconnect_ex
2310+
2311+
def _is_connection_loss(self, exception: Exception) -> bool:
2312+
"""Return whether an exception proves that the active MCP connection ended."""
2313+
from anyio import ClosedResourceError
2314+
from mcp import MCPError
2315+
2316+
return isinstance(exception, ClosedResourceError) or (
2317+
isinstance(exception, MCPError) and "session terminated" in exception.error.message.lower()
2318+
)
2319+
22822320
async def _reconnect_without_loading(self) -> None:
22832321
if self._is_lifecycle_owner_task():
22842322
await self._connect_on_owner(reset=True, load_configured=False)
@@ -2903,14 +2941,11 @@ async def _load_prompts_locked(self) -> None:
29032941
break
29042942
except ClosedResourceError as cl_ex:
29052943
if attempt == 0:
2906-
logger.info("MCP connection closed unexpectedly while loading prompts. Reconnecting...")
2907-
try:
2908-
await self._reconnect_without_loading()
2909-
except Exception as reconn_ex:
2910-
raise _mcp_tool_execution_exception(
2911-
"Failed to reconnect to MCP server.",
2912-
inner_exception=reconn_ex,
2913-
) from reconn_ex
2944+
await self._recover_connection_after_loss(
2945+
operation="prompt discovery",
2946+
inner_exception=cl_ex,
2947+
load_configured=False,
2948+
)
29142949
continue
29152950
_log_mcp_exception(
29162951
logging.ERROR,
@@ -3049,14 +3084,11 @@ async def _load_tools_locked(self) -> None:
30493084
break
30503085
except ClosedResourceError as cl_ex:
30513086
if attempt == 0:
3052-
logger.info("MCP connection closed unexpectedly while loading tools. Reconnecting...")
3053-
try:
3054-
await self._reconnect_without_loading()
3055-
except Exception as reconn_ex:
3056-
raise _mcp_tool_execution_exception(
3057-
"Failed to reconnect to MCP server.",
3058-
inner_exception=reconn_ex,
3059-
) from reconn_ex
3087+
await self._recover_connection_after_loss(
3088+
operation="tool discovery",
3089+
inner_exception=cl_ex,
3090+
load_configured=False,
3091+
)
30603092
continue
30613093
_log_mcp_exception(
30623094
logging.ERROR,
@@ -3234,6 +3266,7 @@ async def _ensure_connected(self) -> None:
32343266
Raises:
32353267
ToolExecutionException: If reconnection fails.
32363268
"""
3269+
from anyio import ClosedResourceError
32373270
from mcp import MCPError
32383271

32393272
if not self._ping_available:
@@ -3246,23 +3279,36 @@ async def _ensure_connected(self) -> None:
32463279
self._ping_available = False
32473280
logger.debug("Skipping future MCP pings because the server does not support ping.")
32483281
return
3249-
logger.info("MCP connection invalid or closed. Reconnecting...")
3250-
try:
3251-
await self._reconnect_without_loading()
3252-
except Exception as ex:
3253-
raise _mcp_tool_execution_exception(
3254-
"Failed to establish MCP connection.",
3255-
inner_exception=ex,
3256-
) from ex
3257-
except Exception:
3258-
logger.info("MCP connection invalid or closed. Reconnecting...")
3259-
try:
3260-
await self._reconnect_without_loading()
3261-
except Exception as ex:
3262-
raise _mcp_tool_execution_exception(
3263-
"Failed to establish MCP connection.",
3264-
inner_exception=ex,
3265-
) from ex
3282+
if self._is_connection_loss(mcp_exc) or self._connection is None or self._connection.is_framework_owned:
3283+
await self._recover_connection_after_loss(
3284+
operation="connection liveness check",
3285+
inner_exception=mcp_exc,
3286+
load_configured=False,
3287+
)
3288+
return
3289+
raise _mcp_tool_execution_exception(
3290+
"MCP connection liveness check failed.",
3291+
inner_exception=mcp_exc,
3292+
) from mcp_exc
3293+
except ClosedResourceError as connection_ex:
3294+
await self._recover_connection_after_loss(
3295+
operation="connection liveness check",
3296+
inner_exception=connection_ex,
3297+
load_configured=False,
3298+
)
3299+
except Exception as connection_ex:
3300+
connection = self._connection
3301+
if connection is None or connection.is_framework_owned:
3302+
await self._recover_connection_after_loss(
3303+
operation="connection liveness check",
3304+
inner_exception=connection_ex,
3305+
load_configured=False,
3306+
)
3307+
return
3308+
raise _mcp_tool_execution_exception(
3309+
"MCP connection liveness check failed.",
3310+
inner_exception=connection_ex,
3311+
) from connection_ex
32663312

32673313
async def call_tool(self, tool_name: str, **kwargs: Any) -> str | list[Content]:
32683314
"""Call a tool with the given arguments.
@@ -3365,29 +3411,24 @@ async def _call_tool_with_retries(
33653411
set_mcp_span_error(span, type(tool_ex).__name__, str(tool_ex))
33663412
raise
33673413
except (ClosedResourceError, MCPError) as call_ex:
3368-
is_session_terminated = (
3369-
isinstance(call_ex, MCPError) and "session terminated" in call_ex.error.message.lower()
3370-
)
3371-
is_connection_lost = isinstance(call_ex, ClosedResourceError) or is_session_terminated
3372-
if not is_connection_lost:
3414+
if not self._is_connection_loss(call_ex):
33733415
error_message = call_ex.error.message if isinstance(call_ex, MCPError) else str(call_ex)
33743416
if span.is_recording():
33753417
set_mcp_span_error(span, type(call_ex).__name__, error_message)
33763418
raise _mcp_tool_execution_exception(error_message, inner_exception=call_ex) from call_ex
33773419

33783420
if attempt == 0:
3379-
# First attempt failed, try reconnecting.
3380-
logger.info("MCP connection closed or terminated unexpectedly. Reconnecting...")
33813421
try:
3382-
await self.connect(reset=True)
3383-
continue
3384-
except Exception as reconn_ex:
3422+
await self._recover_connection_after_loss(
3423+
operation="tool call",
3424+
inner_exception=call_ex,
3425+
load_configured=True,
3426+
)
3427+
except ToolExecutionException as recovery_error:
33853428
if span.is_recording():
3386-
set_mcp_span_error(span, type(reconn_ex).__name__, str(reconn_ex))
3387-
raise _mcp_tool_execution_exception(
3388-
"Failed to reconnect to MCP server.",
3389-
inner_exception=reconn_ex,
3390-
) from reconn_ex
3429+
set_mcp_span_error(span, type(recovery_error).__name__, str(recovery_error))
3430+
raise
3431+
continue
33913432

33923433
# Second attempt also failed, give up.
33933434
_log_mcp_exception(
@@ -3508,31 +3549,29 @@ async def get_prompt(self, prompt_name: str, **kwargs: Any) -> str:
35083549
return parser(prompt_result)
35093550
except ClosedResourceError as cl_ex:
35103551
if attempt == 0:
3511-
# First attempt failed, try reconnecting
3512-
logger.info("MCP connection closed unexpectedly. Reconnecting...")
35133552
try:
3514-
await self.connect(reset=True)
3515-
continue # Retry the operation
3516-
except Exception as reconn_ex:
3517-
set_mcp_span_error(span, type(reconn_ex).__name__, str(reconn_ex))
3518-
raise _mcp_tool_execution_exception(
3519-
"Failed to reconnect to MCP server.",
3520-
inner_exception=reconn_ex,
3521-
) from reconn_ex
3522-
else:
3523-
# Second attempt also failed, give up
3524-
_log_mcp_exception(
3525-
logging.ERROR,
3526-
cl_ex,
3527-
"MCP connection closed unexpectedly after reconnection: %s",
3528-
cl_ex,
3529-
redacted_message="MCP prompt call connection closed after reconnection",
3530-
)
3531-
set_mcp_span_error(span, type(cl_ex).__name__, str(cl_ex))
3532-
raise _mcp_tool_execution_exception(
3533-
f"Failed to call prompt '{prompt_name}' - connection lost.",
3534-
inner_exception=cl_ex,
3535-
) from cl_ex
3553+
await self._recover_connection_after_loss(
3554+
operation="prompt call",
3555+
inner_exception=cl_ex,
3556+
load_configured=True,
3557+
)
3558+
except ToolExecutionException as recovery_error:
3559+
set_mcp_span_error(span, type(recovery_error).__name__, str(recovery_error))
3560+
raise
3561+
continue
3562+
# Second attempt also failed, give up
3563+
_log_mcp_exception(
3564+
logging.ERROR,
3565+
cl_ex,
3566+
"MCP connection closed unexpectedly after reconnection: %s",
3567+
cl_ex,
3568+
redacted_message="MCP prompt call connection closed after reconnection",
3569+
)
3570+
set_mcp_span_error(span, type(cl_ex).__name__, str(cl_ex))
3571+
raise _mcp_tool_execution_exception(
3572+
f"Failed to call prompt '{prompt_name}' - connection lost.",
3573+
inner_exception=cl_ex,
3574+
) from cl_ex
35363575
except MCPError as mcp_exc:
35373576
error_message = mcp_exc.error.message
35383577
set_mcp_span_error(span, type(mcp_exc).__name__, error_message)

0 commit comments

Comments
 (0)