Skip to content

Commit 7d23582

Browse files
giles17CopilotCopilot
authored
Python: fix: prevent MCP message_handler deadlock on notification reload (#4866)
* fix(python): prevent MCP message_handler deadlock on notification reload When an MCP server sends a notifications/tools/list_changed or notifications/prompts/list_changed notification, the message_handler previously awaited load_tools()/load_prompts() directly. Since the handler runs on the MCP SDK's single-threaded receive loop, this caused a deadlock: load_tools() sends a list_tools request and waits for its response, but the receive loop cannot deliver that response while blocked in the handler. This manifested as a timeout in call_tool(), which then surfaced as "Error: Function failed." to the model instead of the real tool output. The MATLAB MCP server reliably triggers this because it sends a tools/list_changed notification during tool execution. Fix: schedule reloads as background asyncio.Tasks via a new _schedule_reload() helper, freeing the receive loop immediately. Fixes #4828 Co-authored-by: Copilot <[email protected]> * Address PR review feedback: fix exc_info, coalesce reloads, shutdown cleanup, tests - Fix exc_info=exc -> exc_info=True in _schedule_reload and message_handler - Tighten _schedule_reload param type from Any to Coroutine[Any, Any, None] - Coalesce reloads: cancel-and-replace per reload kind to prevent unbounded growth - Cancel pending reload tasks in _close_on_owner before tearing down session - Re-raise CancelledError in _safe_reload to respect task cancellation - Replace flaky asyncio.sleep(0) with asyncio.wait_for/gather in tests - Add caplog assertions to verify reload failure is actually logged - Assert _pending_reload_tasks cleanup on error path Co-authored-by: Copilot <[email protected]> * fix: address review comments on MCP reload handling - Fix exc_info=True -> exc_info=message in message_handler error logging, since the handler is not called from an except block - Await cancelled reload tasks in _close_on_owner before tearing down the session to avoid 'Task was destroyed but pending' warnings - Add cancel-and-replace test verifying duplicate notifications cancel the first reload task and only keep one in flight Co-authored-by: Copilot <[email protected]> * fix: remove Task.cancelling() call for Python 3.10 compat Task.cancelling() was added in Python 3.11. Replace with awaiting the task and checking cancelled() instead. Co-authored-by: Copilot <[email protected]> * Add debug log when cancelling superseded reload task Log at DEBUG level when a new notification cancels an in-flight reload task, improving observability of the cancel-and-replace behavior. Co-authored-by: Copilot <[email protected]> --------- Co-authored-by: Copilot <[email protected]> Co-authored-by: Copilot <[email protected]>
1 parent 5746316 commit 7d23582

2 files changed

Lines changed: 158 additions & 4 deletions

File tree

‎python/packages/core/agent_framework/_mcp.py‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import re
1111
import sys
1212
from abc import abstractmethod
13-
from collections.abc import Callable, Collection, Sequence
13+
from collections.abc import Callable, Collection, Coroutine, Sequence
1414
from contextlib import AsyncExitStack, _AsyncGeneratorContextManager # type: ignore
1515
from datetime import timedelta
1616
from functools import partial
@@ -264,6 +264,7 @@ def __init__(
264264
self.is_connected: bool = False
265265
self._tools_loaded: bool = False
266266
self._prompts_loaded: bool = False
267+
self._pending_reload_tasks: set[asyncio.Task[None]] = set()
267268

268269
def __str__(self) -> str:
269270
return f"MCPTool(name={self.name}, description={self.description})"
@@ -905,12 +906,47 @@ async def message_handler(
905906
if isinstance(message, types.ServerNotification):
906907
match message.root.method:
907908
case "notifications/tools/list_changed":
908-
await self.load_tools()
909+
self._schedule_reload(self.load_tools())
909910
case "notifications/prompts/list_changed":
910-
await self.load_prompts()
911+
self._schedule_reload(self.load_prompts())
911912
case _:
912913
logger.debug("Unhandled notification: %s", message.root.method)
913914

915+
def _schedule_reload(self, coro: Coroutine[Any, Any, None]) -> None:
916+
"""Schedule a reload coroutine as a background task.
917+
918+
Reloads (load_tools / load_prompts) triggered by MCP server
919+
notifications must NOT be awaited inside the message handler because
920+
the handler runs on the MCP SDK's single-threaded receive loop.
921+
Awaiting a session request (e.g. ``list_tools``) from within that loop
922+
deadlocks: the receive loop cannot read the response while it is
923+
blocked waiting for the handler to return.
924+
925+
Instead we fire the reload as an independent ``asyncio.Task`` and keep
926+
a strong reference in ``_pending_reload_tasks`` so it is not garbage-
927+
collected before completion. Only one reload per kind (tools / prompts)
928+
is kept in flight; a new notification cancels the previous pending task
929+
for the same coroutine name to avoid unbounded growth.
930+
"""
931+
# Cancel-and-replace: only one reload per kind should be in flight.
932+
reload_name = f"mcp-reload:{self.name}:{coro.__qualname__}"
933+
for existing in list(self._pending_reload_tasks):
934+
if existing.get_name() == reload_name and not existing.done():
935+
logger.debug("Cancelling in-flight reload %s; superseded by new notification", reload_name)
936+
existing.cancel()
937+
938+
async def _safe_reload() -> None:
939+
try:
940+
await coro
941+
except asyncio.CancelledError:
942+
raise
943+
except Exception:
944+
logger.warning("Background MCP reload failed", exc_info=True)
945+
946+
task = asyncio.create_task(_safe_reload(), name=reload_name)
947+
self._pending_reload_tasks.add(task)
948+
task.add_done_callback(self._pending_reload_tasks.discard)
949+
914950
def _determine_approval_mode(
915951
self,
916952
*candidate_names: str,
@@ -1047,6 +1083,14 @@ async def _call_tool_with_runtime_kwargs(
10471083
params = types.PaginatedRequestParams(cursor=tool_list.nextCursor)
10481084

10491085
async def _close_on_owner(self) -> None:
1086+
# Cancel any pending reload tasks before tearing down the session.
1087+
tasks = list(self._pending_reload_tasks)
1088+
for task in tasks:
1089+
task.cancel()
1090+
self._pending_reload_tasks.clear()
1091+
if tasks:
1092+
await asyncio.gather(*tasks, return_exceptions=True)
1093+
10501094
await self._safe_close_exit_stack()
10511095
self._exit_stack = AsyncExitStack()
10521096
self.session = None

‎python/packages/core/tests/core/test_mcp.py‎

Lines changed: 111 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Copyright (c) Microsoft. All rights reserved.
22
# type: ignore[reportPrivateUsage]
33
import asyncio
4+
import contextlib
45
import json
56
import logging
67
import os
@@ -1615,7 +1616,7 @@ async def call_tool_with_error(*args, **kwargs):
16151616

16161617
async def test_mcp_tool_message_handler_notification():
16171618
"""Test that message_handler correctly processes tools/list_changed and prompts/list_changed
1618-
notifications."""
1619+
notifications by scheduling reloads as background tasks."""
16191620
tool = MCPStdioTool(name="test_tool", command="python")
16201621

16211622
# Mock the load_tools and load_prompts methods
@@ -1629,6 +1630,8 @@ async def test_mcp_tool_message_handler_notification():
16291630

16301631
result = await tool.message_handler(tools_notification)
16311632
assert result is None
1633+
# The reload is scheduled as a background task; let it run.
1634+
await asyncio.sleep(0)
16321635
tool.load_tools.assert_called_once()
16331636

16341637
# Reset mock
@@ -1641,6 +1644,7 @@ async def test_mcp_tool_message_handler_notification():
16411644

16421645
result = await tool.message_handler(prompts_notification)
16431646
assert result is None
1647+
await asyncio.sleep(0)
16441648
tool.load_prompts.assert_called_once()
16451649

16461650
# Test unhandled notification
@@ -1664,6 +1668,112 @@ async def test_mcp_tool_message_handler_error():
16641668
assert result is None
16651669

16661670

1671+
async def test_mcp_tool_message_handler_does_not_block_receive_loop():
1672+
"""Test that message_handler does not deadlock the MCP receive loop.
1673+
1674+
Regression test for https://github.com/microsoft/agent-framework/issues/4828.
1675+
When the MCP server sends a ``notifications/tools/list_changed``
1676+
notification, the handler must NOT await ``load_tools()`` synchronously
1677+
because that would block the single-threaded MCP receive loop, preventing
1678+
it from delivering the ``list_tools`` response — a classic deadlock.
1679+
"""
1680+
tool = MCPStdioTool(name="test_tool", command="python")
1681+
1682+
# Use an event to make load_tools block until we release it.
1683+
# This simulates load_tools waiting for a session response that the
1684+
# receive loop would need to deliver.
1685+
release = asyncio.Event()
1686+
1687+
async def slow_load_tools():
1688+
await release.wait()
1689+
1690+
tool.load_tools = slow_load_tools # type: ignore[assignment]
1691+
1692+
tools_notification = Mock(spec=types.ServerNotification)
1693+
tools_notification.root = Mock()
1694+
tools_notification.root.method = "notifications/tools/list_changed"
1695+
1696+
# message_handler must return immediately even though load_tools blocks.
1697+
await tool.message_handler(tools_notification)
1698+
1699+
# If the handler had awaited load_tools synchronously, we would never
1700+
# reach this line (deadlock). Verify the reload task is pending.
1701+
assert len(tool._pending_reload_tasks) == 1
1702+
1703+
# Unblock the reload so the background task finishes cleanly.
1704+
release.set()
1705+
# Wait for the pending reload task(s) to complete so their done-callbacks
1706+
# have a chance to remove them from _pending_reload_tasks.
1707+
await asyncio.wait_for(asyncio.gather(*tool._pending_reload_tasks), timeout=1)
1708+
assert len(tool._pending_reload_tasks) == 0
1709+
1710+
1711+
async def test_mcp_tool_message_handler_reload_failure_is_logged(caplog: pytest.LogCaptureFixture):
1712+
"""Background reload errors are logged, not raised into the receive loop."""
1713+
tool = MCPStdioTool(name="test_tool", command="python")
1714+
tool.load_tools = AsyncMock(side_effect=RuntimeError("connection lost"))
1715+
1716+
tools_notification = Mock(spec=types.ServerNotification)
1717+
tools_notification.root = Mock()
1718+
tools_notification.root.method = "notifications/tools/list_changed"
1719+
1720+
await tool.message_handler(tools_notification)
1721+
# Let the background task run — it should not propagate the exception.
1722+
# Snapshot tasks and await them to ensure done-callbacks fire.
1723+
pending = list(tool._pending_reload_tasks)
1724+
if pending:
1725+
await asyncio.wait_for(asyncio.gather(*pending, return_exceptions=True), timeout=1)
1726+
tool.load_tools.assert_called_once()
1727+
assert len(tool._pending_reload_tasks) == 0
1728+
1729+
# Verify the warning was actually logged with exception info.
1730+
reload_warnings = [r for r in caplog.records if "Background MCP reload failed" in r.message]
1731+
assert len(reload_warnings) == 1
1732+
assert reload_warnings[0].levelname == "WARNING"
1733+
assert reload_warnings[0].exc_info is not None
1734+
1735+
1736+
async def test_mcp_tool_message_handler_cancel_and_replace():
1737+
"""Sending two notifications in quick succession cancels the first reload task."""
1738+
tool = MCPStdioTool(name="test_tool", command="python")
1739+
1740+
release = asyncio.Event()
1741+
call_count = 0
1742+
1743+
async def blocking_load_tools():
1744+
nonlocal call_count
1745+
call_count += 1
1746+
await release.wait()
1747+
1748+
tool.load_tools = blocking_load_tools # type: ignore[assignment]
1749+
1750+
notification = Mock(spec=types.ServerNotification)
1751+
notification.root = Mock()
1752+
notification.root.method = "notifications/tools/list_changed"
1753+
1754+
# First notification — starts a blocking reload task.
1755+
await tool.message_handler(notification)
1756+
assert len(tool._pending_reload_tasks) == 1
1757+
first_task = next(iter(tool._pending_reload_tasks))
1758+
1759+
# Second notification — should cancel the first and replace it.
1760+
await tool.message_handler(notification)
1761+
# Yield to the event loop so the cancellation is processed.
1762+
with contextlib.suppress(asyncio.CancelledError):
1763+
await first_task
1764+
1765+
assert first_task.cancelled()
1766+
1767+
assert len(tool._pending_reload_tasks) == 1
1768+
second_task = next(iter(tool._pending_reload_tasks))
1769+
assert second_task is not first_task
1770+
1771+
# Unblock and let the second task finish.
1772+
release.set()
1773+
await asyncio.wait_for(asyncio.gather(*tool._pending_reload_tasks), timeout=1)
1774+
assert len(tool._pending_reload_tasks) == 0
1775+
1776+
16671777
async def test_mcp_tool_sampling_callback_no_client():
16681778
"""Test sampling callback error path when no chat client is available."""
16691779
tool = MCPStdioTool(name="test_tool", command="python")

0 commit comments

Comments
 (0)