Skip to content

Commit c4c04b6

Browse files
authored
fix(mcp): stop swallowing worker task cancellation (#5055)
_ServerWorker._run wrapped command execution in except BaseException, so an external task.cancel() was recorded on the command future and the worker went back to the queue loop while staying in the cancelling state. is_done never flipped and anything awaiting the task, including loop shutdown, hung The loop now compares the worker task cancellation count before and after each command and re-raises CancelledError only when that count grew while the command ran, which keeps a server raising CancelledError on its own recoverable through the future while an external cancel ends the worker. The count is carried by the worker itself and every cancel() on the task is counted through a shim installed at creation time, so the distinction works on Python 3.10 too, where Task.cancelling() does not exist, and the worker task is still created through the event loop's task factory so applications supervising tasks through set_task_factory keep seeing it When the external cancel lands while a command runs, the worker runs the server cleanup in the same task before re-raising, so a partially acquired connection from an interrupted connect is released instead of leaked. That emergency cleanup is bounded by the manager cleanup timeout instead of the interrupted connect command's own timeout, since connect and cleanup have separate allowances and an unbounded cleanup could still hang loop shutdown A failed or timed-out emergency cleanup used to be swallowed silently, so the worker still looked cleanly stopped: the done callback discarded it, and a later reconnect started a second connection over the transport that was never cleaned up. The failure is now recorded on the cleanup result future, which keeps the worker registered, keeps the failure observable through the manager errors, and keeps reconnect from retrying the uncleaned server The worker loop also settles the futures of commands still queued when it exits for any reason, so callers waiting on a queued command get CancelledError instead of hanging forever
1 parent 74461d0 commit c4c04b6

2 files changed

Lines changed: 395 additions & 22 deletions

File tree

‎src/agents/mcp/manager.py‎

Lines changed: 133 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from collections.abc import Awaitable, Callable, Iterable
66
from contextlib import AbstractAsyncContextManager
77
from dataclasses import dataclass
8-
from typing import Any
8+
from typing import Any, cast
99

1010
from ..logger import log_tool_action_debug, log_tool_action_error, logger
1111
from ._logging import get_mcp_server_log_message
@@ -35,12 +35,77 @@ class _ServerCommand:
3535

3636

3737
class _ServerWorker:
38-
def __init__(self, server: MCPServer) -> None:
38+
def __init__(
39+
self,
40+
server: MCPServer,
41+
cleanup_timeout_seconds_fn: Callable[[], float | None],
42+
) -> None:
3943
self._server = server
44+
# The emergency cleanup below is a cleanup, not a connect, so it must be
45+
# bounded by the manager cleanup timeout rather than by the interrupted
46+
# command's own timeout. The callable is read at cancel time so runtime
47+
# reassignment of the manager property stays honored.
48+
self._cleanup_timeout_seconds_fn = cleanup_timeout_seconds_fn
4049
self._queue: asyncio.Queue[_ServerCommand] = asyncio.Queue()
41-
self._task = asyncio.create_task(self._run())
50+
# The worker needs to tell an external cancel from a CancelledError a
51+
# server raised on its own. Task.cancelling() covers that only on
52+
# Python 3.11+, and this package still supports 3.10, so every cancel()
53+
# call on the worker task is counted here.
54+
self._cancel_requests = 0
55+
self._task: asyncio.Task[None] = self._create_task()
4256
self._cleanup_future: asyncio.Future[None] | None = None
4357

58+
def _create_task(self) -> asyncio.Task[None]:
59+
# Route worker creation through the event loop's task factory so
60+
# applications supervising or instrumenting tasks through
61+
# set_task_factory still see this worker. The temporary factory keeps
62+
# the app factory in charge of the task type and only adds the cancel
63+
# counting on top, because loop shutdown and supervisors cancel the
64+
# task object directly and would otherwise bypass the count.
65+
loop = asyncio.get_running_loop()
66+
app_factory = loop.get_task_factory()
67+
68+
def factory(
69+
loop: asyncio.AbstractEventLoop, coro: Any, **kwargs: Any
70+
) -> asyncio.Task[None]:
71+
task: asyncio.Task[None] = (
72+
cast(asyncio.Task[None], app_factory(loop, coro, **kwargs))
73+
if app_factory
74+
else cast(asyncio.Task[None], asyncio.Task(coro, loop=loop, **kwargs))
75+
)
76+
original_cancel = task.cancel
77+
78+
def counting_cancel(msg: Any = None) -> bool:
79+
self._cancel_requests += 1
80+
return bool(original_cancel(msg))
81+
82+
task.cancel = counting_cancel # type: ignore[method-assign]
83+
return task
84+
85+
# The typeshed _TaskFactory protocol is a private generic stub; this
86+
# factory matches its runtime call shape exactly.
87+
loop.set_task_factory(factory) # type: ignore[arg-type]
88+
try:
89+
return asyncio.create_task(self._run())
90+
finally:
91+
loop.set_task_factory(app_factory)
92+
93+
def _record_cleanup_failure(self, exc: BaseException) -> None:
94+
"""Record an emergency cleanup failure on the cleanup result future.
95+
96+
The done callback reads the cleanup_error property through that
97+
future, so a recorded failure keeps the worker registered instead of
98+
letting the manager treat the server as cleanly stopped. A later
99+
cleanup() call awaits the same settled future and surfaces the failure
100+
to the manager, which keeps reconnect from starting a second
101+
connection over a transport that was never successfully cleaned up.
102+
"""
103+
if self._cleanup_future is None:
104+
self._cleanup_future = asyncio.get_running_loop().create_future()
105+
if self._cleanup_future.done():
106+
return
107+
self._cleanup_future.set_exception(exc)
108+
44109
@property
45110
def is_done(self) -> bool:
46111
return self._task.done()
@@ -91,23 +156,67 @@ async def _submit(self, action: str, timeout_seconds: float | None) -> None:
91156
await future
92157

93158
async def _run(self) -> None:
94-
while True:
95-
command = await self._queue.get()
96-
should_exit = command.action == "cleanup"
97-
try:
98-
if command.action == "connect":
99-
await _run_with_timeout_in_task(self._server.connect, command.timeout_seconds)
100-
elif command.action == "cleanup":
101-
await _run_with_timeout_in_task(self._server.cleanup, command.timeout_seconds)
102-
else:
103-
raise ValueError(f"Unknown command: {command.action}")
104-
if not command.future.cancelled():
105-
command.future.set_result(None)
106-
except BaseException as exc:
107-
if not command.future.cancelled():
108-
command.future.set_exception(exc)
109-
if should_exit:
110-
return
159+
try:
160+
while True:
161+
command = await self._queue.get()
162+
should_exit = command.action == "cleanup"
163+
cancellation_baseline = self._cancel_requests
164+
try:
165+
if command.action == "connect":
166+
await _run_with_timeout_in_task(
167+
self._server.connect, command.timeout_seconds
168+
)
169+
elif command.action == "cleanup":
170+
await _run_with_timeout_in_task(
171+
self._server.cleanup, command.timeout_seconds
172+
)
173+
else:
174+
raise ValueError(f"Unknown command: {command.action}")
175+
if not command.future.cancelled():
176+
command.future.set_result(None)
177+
except BaseException as exc:
178+
external_cancel = (
179+
isinstance(exc, asyncio.CancelledError)
180+
and self._cancel_requests > cancellation_baseline
181+
)
182+
if external_cancel and command.action != "cleanup":
183+
# The worker task itself was cancelled while the command
184+
# was running. A partially acquired connection from the
185+
# interrupted command still needs cleanup, and this task
186+
# owns it (some transports require cleanup in the same
187+
# task), so clean up here before the caller is released.
188+
# The manager cleanup timeout bounds the emergency
189+
# cleanup, not the interrupted command's own timeout.
190+
try:
191+
await _run_with_timeout_in_task(
192+
self._server.cleanup, self._cleanup_timeout_seconds_fn()
193+
)
194+
except BaseException as cleanup_exc:
195+
# A failed or timed-out emergency cleanup must not be
196+
# swallowed silently. Recording it on the cleanup
197+
# result future keeps the worker registered, so a
198+
# later reconnect surfaces the failure and never
199+
# starts a second connection over the transport
200+
# that was never cleaned up.
201+
self._record_cleanup_failure(cleanup_exc)
202+
if not command.future.cancelled():
203+
command.future.set_exception(exc)
204+
if external_cancel:
205+
# The future carries the error to the caller; re-raise
206+
# here to end the task and honor the cancellation.
207+
raise
208+
if should_exit:
209+
return
210+
finally:
211+
# The worker loop has ended, so commands still queued will never run.
212+
# Settle their futures so waiting callers do not hang forever.
213+
while True:
214+
try:
215+
pending = self._queue.get_nowait()
216+
except asyncio.QueueEmpty:
217+
break
218+
if not pending.future.done():
219+
pending.future.cancel()
111220

112221

113222
async def _run_with_timeout_in_task(
@@ -535,7 +644,10 @@ async def _get_worker(self, server: MCPServer) -> _ServerWorker:
535644
self._discard_worker(server, worker)
536645
worker = self._workers.get(server)
537646
if worker is None:
538-
worker = _ServerWorker(server=server)
647+
worker = _ServerWorker(
648+
server=server,
649+
cleanup_timeout_seconds_fn=lambda: self.cleanup_timeout_seconds,
650+
)
539651
self._workers[server] = worker
540652
worker.add_done_callback(lambda _task: self._handle_worker_done(server, worker))
541653
return worker

0 commit comments

Comments
 (0)