Skip to content

Commit 7495eb7

Browse files
CopilotCopilot
andcommitted
fix(python): prevent MCP message_handler deadlock on notification reload
When an MCP server sends a notifications/tools/list_changed or notifications/prompts/list_changed notification, the message_handler previously awaited load_tools()/load_prompts() directly. Since the handler runs on the MCP SDK's single-threaded receive loop, this caused a deadlock: load_tools() sends a list_tools request and waits for its response, but the receive loop cannot deliver that response while blocked in the handler. This manifested as a timeout in call_tool(), which then surfaced as "Error: Function failed." to the model instead of the real tool output. The MATLAB MCP server reliably triggers this because it sends a tools/list_changed notification during tool execution. Fix: schedule reloads as background asyncio.Tasks via a new _schedule_reload() helper, freeing the receive loop immediately. Fixes microsoft#4828 Co-authored-by: Copilot <[email protected]>
1 parent 7e6d87e commit 7495eb7

2 files changed

Lines changed: 88 additions & 3 deletions

File tree

‎python/packages/core/agent_framework/_mcp.py‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,7 @@ def __init__(
491491
self.is_connected: bool = False
492492
self._tools_loaded: bool = False
493493
self._prompts_loaded: bool = False
494+
self._pending_reload_tasks: set[asyncio.Task[None]] = set()
494495

495496
def __str__(self) -> str:
496497
return f"MCPTool(name={self.name}, description={self.description})"
@@ -804,12 +805,37 @@ async def message_handler(
804805
if isinstance(message, types.ServerNotification):
805806
match message.root.method:
806807
case "notifications/tools/list_changed":
807-
await self.load_tools()
808+
self._schedule_reload(self.load_tools())
808809
case "notifications/prompts/list_changed":
809-
await self.load_prompts()
810+
self._schedule_reload(self.load_prompts())
810811
case _:
811812
logger.debug("Unhandled notification: %s", message.root.method)
812813

814+
def _schedule_reload(self, coro: Any) -> None:
815+
"""Schedule a reload coroutine as a background task.
816+
817+
Reloads (load_tools / load_prompts) triggered by MCP server
818+
notifications must NOT be awaited inside the message handler because
819+
the handler runs on the MCP SDK's single-threaded receive loop.
820+
Awaiting a session request (e.g. ``list_tools``) from within that loop
821+
deadlocks: the receive loop cannot read the response while it is
822+
blocked waiting for the handler to return.
823+
824+
Instead we fire the reload as an independent ``asyncio.Task`` and keep
825+
a strong reference in ``_pending_reload_tasks`` so it is not garbage-
826+
collected before completion.
827+
"""
828+
829+
async def _safe_reload() -> None:
830+
try:
831+
await coro
832+
except Exception as exc:
833+
logger.warning("Background MCP reload failed: %s", exc, exc_info=exc)
834+
835+
task = asyncio.create_task(_safe_reload(), name=f"mcp-reload:{self.name}")
836+
self._pending_reload_tasks.add(task)
837+
task.add_done_callback(self._pending_reload_tasks.discard)
838+
813839
def _determine_approval_mode(
814840
self,
815841
*candidate_names: str,

‎python/packages/core/tests/core/test_mcp.py‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Copyright (c) Microsoft. All rights reserved.
22
# type: ignore[reportPrivateUsage]
3+
import asyncio
34
import logging
45
import os
56
from contextlib import _AsyncGeneratorContextManager # type: ignore
@@ -1478,7 +1479,7 @@ async def call_tool_with_error(*args, **kwargs):
14781479

14791480
async def test_mcp_tool_message_handler_notification():
14801481
"""Test that message_handler correctly processes tools/list_changed and prompts/list_changed
1481-
notifications."""
1482+
notifications by scheduling reloads as background tasks."""
14821483
tool = MCPStdioTool(name="test_tool", command="python")
14831484

14841485
# Mock the load_tools and load_prompts methods
@@ -1492,6 +1493,8 @@ async def test_mcp_tool_message_handler_notification():
14921493

14931494
result = await tool.message_handler(tools_notification)
14941495
assert result is None
1496+
# The reload is scheduled as a background task; let it run.
1497+
await asyncio.sleep(0)
14951498
tool.load_tools.assert_called_once()
14961499

14971500
# Reset mock
@@ -1504,6 +1507,7 @@ async def test_mcp_tool_message_handler_notification():
15041507

15051508
result = await tool.message_handler(prompts_notification)
15061509
assert result is None
1510+
await asyncio.sleep(0)
15071511
tool.load_prompts.assert_called_once()
15081512

15091513
# Test unhandled notification
@@ -1527,6 +1531,61 @@ async def test_mcp_tool_message_handler_error():
15271531
assert result is None
15281532

15291533

1534+
async def test_mcp_tool_message_handler_does_not_block_receive_loop():
1535+
"""Test that message_handler does not deadlock the MCP receive loop.
1536+
1537+
Regression test for https://github.com/microsoft/agent-framework/issues/4828.
1538+
When the MCP server sends a ``notifications/tools/list_changed``
1539+
notification, the handler must NOT await ``load_tools()`` synchronously
1540+
because that would block the single-threaded MCP receive loop, preventing
1541+
it from delivering the ``list_tools`` response — a classic deadlock.
1542+
"""
1543+
tool = MCPStdioTool(name="test_tool", command="python")
1544+
1545+
# Use an event to make load_tools block until we release it.
1546+
# This simulates load_tools waiting for a session response that the
1547+
# receive loop would need to deliver.
1548+
release = asyncio.Event()
1549+
1550+
async def slow_load_tools():
1551+
await release.wait()
1552+
1553+
tool.load_tools = slow_load_tools # type: ignore[assignment]
1554+
1555+
tools_notification = Mock(spec=types.ServerNotification)
1556+
tools_notification.root = Mock()
1557+
tools_notification.root.method = "notifications/tools/list_changed"
1558+
1559+
# message_handler must return immediately even though load_tools blocks.
1560+
await tool.message_handler(tools_notification)
1561+
1562+
# If the handler had awaited load_tools synchronously, we would never
1563+
# reach this line (deadlock). Verify the reload task is pending.
1564+
assert len(tool._pending_reload_tasks) == 1
1565+
1566+
# Unblock the reload so the background task finishes cleanly.
1567+
release.set()
1568+
await asyncio.sleep(0)
1569+
# Give the done-callback a chance to fire.
1570+
await asyncio.sleep(0)
1571+
assert len(tool._pending_reload_tasks) == 0
1572+
1573+
1574+
async def test_mcp_tool_message_handler_reload_failure_is_logged():
1575+
"""Background reload errors are logged, not raised into the receive loop."""
1576+
tool = MCPStdioTool(name="test_tool", command="python")
1577+
tool.load_tools = AsyncMock(side_effect=RuntimeError("connection lost"))
1578+
1579+
tools_notification = Mock(spec=types.ServerNotification)
1580+
tools_notification.root = Mock()
1581+
tools_notification.root.method = "notifications/tools/list_changed"
1582+
1583+
await tool.message_handler(tools_notification)
1584+
# Let the background task run — it should not propagate the exception.
1585+
await asyncio.sleep(0)
1586+
tool.load_tools.assert_called_once()
1587+
1588+
15301589
async def test_mcp_tool_sampling_callback_no_client():
15311590
"""Test sampling callback error path when no chat client is available."""
15321591
tool = MCPStdioTool(name="test_tool", command="python")

0 commit comments

Comments
 (0)