|
5 | 5 | from collections.abc import Awaitable, Callable, Iterable |
6 | 6 | from contextlib import AbstractAsyncContextManager |
7 | 7 | from dataclasses import dataclass |
8 | | -from typing import Any |
| 8 | +from typing import Any, cast |
9 | 9 |
|
10 | 10 | from ..logger import log_tool_action_debug, log_tool_action_error, logger |
11 | 11 | from ._logging import get_mcp_server_log_message |
@@ -35,12 +35,77 @@ class _ServerCommand: |
35 | 35 |
|
36 | 36 |
|
37 | 37 | class _ServerWorker: |
38 | | - def __init__(self, server: MCPServer) -> None: |
| 38 | + def __init__( |
| 39 | + self, |
| 40 | + server: MCPServer, |
| 41 | + cleanup_timeout_seconds_fn: Callable[[], float | None], |
| 42 | + ) -> None: |
39 | 43 | self._server = server |
| 44 | + # The emergency cleanup below is a cleanup, not a connect, so it must be |
| 45 | + # bounded by the manager cleanup timeout rather than by the interrupted |
| 46 | + # command's own timeout. The callable is read at cancel time so runtime |
| 47 | + # reassignment of the manager property stays honored. |
| 48 | + self._cleanup_timeout_seconds_fn = cleanup_timeout_seconds_fn |
40 | 49 | self._queue: asyncio.Queue[_ServerCommand] = asyncio.Queue() |
41 | | - self._task = asyncio.create_task(self._run()) |
| 50 | + # The worker needs to tell an external cancel from a CancelledError a |
| 51 | + # server raised on its own. Task.cancelling() covers that only on |
| 52 | + # Python 3.11+, and this package still supports 3.10, so every cancel() |
| 53 | + # call on the worker task is counted here. |
| 54 | + self._cancel_requests = 0 |
| 55 | + self._task: asyncio.Task[None] = self._create_task() |
42 | 56 | self._cleanup_future: asyncio.Future[None] | None = None |
43 | 57 |
|
| 58 | + def _create_task(self) -> asyncio.Task[None]: |
| 59 | + # Route worker creation through the event loop's task factory so |
| 60 | + # applications supervising or instrumenting tasks through |
| 61 | + # set_task_factory still see this worker. The temporary factory keeps |
| 62 | + # the app factory in charge of the task type and only adds the cancel |
| 63 | + # counting on top, because loop shutdown and supervisors cancel the |
| 64 | + # task object directly and would otherwise bypass the count. |
| 65 | + loop = asyncio.get_running_loop() |
| 66 | + app_factory = loop.get_task_factory() |
| 67 | + |
| 68 | + def factory( |
| 69 | + loop: asyncio.AbstractEventLoop, coro: Any, **kwargs: Any |
| 70 | + ) -> asyncio.Task[None]: |
| 71 | + task: asyncio.Task[None] = ( |
| 72 | + cast(asyncio.Task[None], app_factory(loop, coro, **kwargs)) |
| 73 | + if app_factory |
| 74 | + else cast(asyncio.Task[None], asyncio.Task(coro, loop=loop, **kwargs)) |
| 75 | + ) |
| 76 | + original_cancel = task.cancel |
| 77 | + |
| 78 | + def counting_cancel(msg: Any = None) -> bool: |
| 79 | + self._cancel_requests += 1 |
| 80 | + return bool(original_cancel(msg)) |
| 81 | + |
| 82 | + task.cancel = counting_cancel # type: ignore[method-assign] |
| 83 | + return task |
| 84 | + |
| 85 | + # The typeshed _TaskFactory protocol is a private generic stub; this |
| 86 | + # factory matches its runtime call shape exactly. |
| 87 | + loop.set_task_factory(factory) # type: ignore[arg-type] |
| 88 | + try: |
| 89 | + return asyncio.create_task(self._run()) |
| 90 | + finally: |
| 91 | + loop.set_task_factory(app_factory) |
| 92 | + |
| 93 | + def _record_cleanup_failure(self, exc: BaseException) -> None: |
| 94 | + """Record an emergency cleanup failure on the cleanup result future. |
| 95 | +
|
| 96 | + The done callback reads the cleanup_error property through that |
| 97 | + future, so a recorded failure keeps the worker registered instead of |
| 98 | + letting the manager treat the server as cleanly stopped. A later |
| 99 | + cleanup() call awaits the same settled future and surfaces the failure |
| 100 | + to the manager, which keeps reconnect from starting a second |
| 101 | + connection over a transport that was never successfully cleaned up. |
| 102 | + """ |
| 103 | + if self._cleanup_future is None: |
| 104 | + self._cleanup_future = asyncio.get_running_loop().create_future() |
| 105 | + if self._cleanup_future.done(): |
| 106 | + return |
| 107 | + self._cleanup_future.set_exception(exc) |
| 108 | + |
44 | 109 | @property |
45 | 110 | def is_done(self) -> bool: |
46 | 111 | return self._task.done() |
@@ -91,23 +156,67 @@ async def _submit(self, action: str, timeout_seconds: float | None) -> None: |
91 | 156 | await future |
92 | 157 |
|
93 | 158 | async def _run(self) -> None: |
94 | | - while True: |
95 | | - command = await self._queue.get() |
96 | | - should_exit = command.action == "cleanup" |
97 | | - try: |
98 | | - if command.action == "connect": |
99 | | - await _run_with_timeout_in_task(self._server.connect, command.timeout_seconds) |
100 | | - elif command.action == "cleanup": |
101 | | - await _run_with_timeout_in_task(self._server.cleanup, command.timeout_seconds) |
102 | | - else: |
103 | | - raise ValueError(f"Unknown command: {command.action}") |
104 | | - if not command.future.cancelled(): |
105 | | - command.future.set_result(None) |
106 | | - except BaseException as exc: |
107 | | - if not command.future.cancelled(): |
108 | | - command.future.set_exception(exc) |
109 | | - if should_exit: |
110 | | - return |
| 159 | + try: |
| 160 | + while True: |
| 161 | + command = await self._queue.get() |
| 162 | + should_exit = command.action == "cleanup" |
| 163 | + cancellation_baseline = self._cancel_requests |
| 164 | + try: |
| 165 | + if command.action == "connect": |
| 166 | + await _run_with_timeout_in_task( |
| 167 | + self._server.connect, command.timeout_seconds |
| 168 | + ) |
| 169 | + elif command.action == "cleanup": |
| 170 | + await _run_with_timeout_in_task( |
| 171 | + self._server.cleanup, command.timeout_seconds |
| 172 | + ) |
| 173 | + else: |
| 174 | + raise ValueError(f"Unknown command: {command.action}") |
| 175 | + if not command.future.cancelled(): |
| 176 | + command.future.set_result(None) |
| 177 | + except BaseException as exc: |
| 178 | + external_cancel = ( |
| 179 | + isinstance(exc, asyncio.CancelledError) |
| 180 | + and self._cancel_requests > cancellation_baseline |
| 181 | + ) |
| 182 | + if external_cancel and command.action != "cleanup": |
| 183 | + # The worker task itself was cancelled while the command |
| 184 | + # was running. A partially acquired connection from the |
| 185 | + # interrupted command still needs cleanup, and this task |
| 186 | + # owns it (some transports require cleanup in the same |
| 187 | + # task), so clean up here before the caller is released. |
| 188 | + # The manager cleanup timeout bounds the emergency |
| 189 | + # cleanup, not the interrupted command's own timeout. |
| 190 | + try: |
| 191 | + await _run_with_timeout_in_task( |
| 192 | + self._server.cleanup, self._cleanup_timeout_seconds_fn() |
| 193 | + ) |
| 194 | + except BaseException as cleanup_exc: |
| 195 | + # A failed or timed-out emergency cleanup must not be |
| 196 | + # swallowed silently. Recording it on the cleanup |
| 197 | + # result future keeps the worker registered, so a |
| 198 | + # later reconnect surfaces the failure and never |
| 199 | + # starts a second connection over the transport |
| 200 | + # that was never cleaned up. |
| 201 | + self._record_cleanup_failure(cleanup_exc) |
| 202 | + if not command.future.cancelled(): |
| 203 | + command.future.set_exception(exc) |
| 204 | + if external_cancel: |
| 205 | + # The future carries the error to the caller; re-raise |
| 206 | + # here to end the task and honor the cancellation. |
| 207 | + raise |
| 208 | + if should_exit: |
| 209 | + return |
| 210 | + finally: |
| 211 | + # The worker loop has ended, so commands still queued will never run. |
| 212 | + # Settle their futures so waiting callers do not hang forever. |
| 213 | + while True: |
| 214 | + try: |
| 215 | + pending = self._queue.get_nowait() |
| 216 | + except asyncio.QueueEmpty: |
| 217 | + break |
| 218 | + if not pending.future.done(): |
| 219 | + pending.future.cancel() |
111 | 220 |
|
112 | 221 |
|
113 | 222 | async def _run_with_timeout_in_task( |
@@ -535,7 +644,10 @@ async def _get_worker(self, server: MCPServer) -> _ServerWorker: |
535 | 644 | self._discard_worker(server, worker) |
536 | 645 | worker = self._workers.get(server) |
537 | 646 | if worker is None: |
538 | | - worker = _ServerWorker(server=server) |
| 647 | + worker = _ServerWorker( |
| 648 | + server=server, |
| 649 | + cleanup_timeout_seconds_fn=lambda: self.cleanup_timeout_seconds, |
| 650 | + ) |
539 | 651 | self._workers[server] = worker |
540 | 652 | worker.add_done_callback(lambda _task: self._handle_worker_done(server, worker)) |
541 | 653 | return worker |
|
0 commit comments