Skip to content

Commit 540df96

Browse files
cdeustclaude
andauthored
feat(admission): count workers still running after their caller was cancelled (#728, slice 2) (#903)
A cancelled tool call leaves its asyncio.to_thread worker running; slice 1 (#885) kept its admission slot and logged its outcome, but nothing said how many such workers were still running or how the ended ones did. handlers/abandoned_work counts, per tool, the abandoned workers in flight and the outcome of each one that ended (completed / raised / cancelled). run_admitted records the abandonment, and _log_abandoned records the outcome and names in its log line how many abandoned workers of the tool remain. Refs #728 Co-authored-by: Claude <[email protected]>
1 parent d3da98f commit 540df96

4 files changed

Lines changed: 219 additions & 5 deletions

File tree

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
"""Status of tool workers that keep running after their caller was cancelled.
2+
3+
``asyncio.to_thread`` cannot stop a running thread, so a cancelled tool call
4+
leaves its worker to finish on its own (issue #728). A mutating tool's write
5+
then lands, or fails, after the client stopped listening. This registry makes
6+
that state readable: how many abandoned workers are still running per tool,
7+
and how each one that ended did (completed, raised, cancelled).
8+
9+
The counts are process-local and start at zero with each server process. They
10+
say nothing about which rows a worker wrote; a tool's ``readOnlyHint`` says
11+
whether its worker can write at all.
12+
13+
source: issue #728 (acceptance: "report in-flight mutation status and work
14+
completing after client abandonment")"""
15+
16+
from __future__ import annotations
17+
18+
import threading
19+
from collections import defaultdict
20+
from typing import Literal
21+
22+
Outcome = Literal["completed", "raised", "cancelled"]
23+
24+
_OUTCOMES: tuple[Outcome, ...] = ("completed", "raised", "cancelled")
25+
26+
_LOCK = threading.Lock()
27+
_in_flight: dict[str, int] = defaultdict(int)
28+
_ended: dict[str, dict[Outcome, int]] = {}
29+
30+
31+
def record_abandoned(tool_name: str) -> None:
32+
"""Count a worker whose caller was cancelled while it still runs."""
33+
with _LOCK:
34+
_in_flight[tool_name] += 1
35+
36+
37+
def record_ended(tool_name: str, outcome: Outcome) -> int:
38+
"""Move one abandoned worker of ``tool_name`` to its outcome.
39+
40+
Returns how many abandoned workers of that tool are still running."""
41+
with _LOCK:
42+
_in_flight[tool_name] -= 1
43+
ended = _ended.setdefault(tool_name, dict.fromkeys(_OUTCOMES, 0))
44+
ended[outcome] += 1
45+
return _in_flight[tool_name]
46+
47+
48+
def snapshot() -> dict[str, dict]:
49+
"""Return ``{"in_flight": {tool: n}, "ended": {tool: {outcome: n}}}``.
50+
51+
Tools with no running abandoned worker are left out of ``in_flight``."""
52+
with _LOCK:
53+
return {
54+
"in_flight": {t: n for t, n in sorted(_in_flight.items()) if n},
55+
"ended": {t: dict(c) for t, c in sorted(_ended.items())},
56+
}
57+
58+
59+
def reset() -> None:
60+
"""Drop all counts. For tests only."""
61+
with _LOCK:
62+
_in_flight.clear()
63+
_ended.clear()

‎mcp_server/handlers/admission.py‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from contextlib import asynccontextmanager
1818
from typing import Any, AsyncIterator, Callable, TypeVar
1919

20+
from mcp_server.handlers import abandoned_work
2021
from mcp_server.handlers.latency_class import DEFAULT_SEMAPHORE, classify
2122

2223
logger = logging.getLogger(__name__)
@@ -82,8 +83,9 @@ async def run_admitted(tool_name: str, func: Callable[..., T], *args: Any) -> T:
8283
waiting: ``asyncio.to_thread`` cannot stop a running thread, so a cancelled
8384
caller that released its slot let a new request start a worker beyond the
8485
declared budget (issue #728: budget 4, four cancelled callers, a fifth
85-
request made five active workers). A worker whose caller was cancelled has
86-
its outcome logged when it ends.
86+
request made five active workers). A worker whose caller was cancelled is
87+
counted in ``abandoned_work`` while it runs, and its outcome is logged and
88+
counted when it ends.
8789
8890
source: issue #728"""
8991
sem = _get_semaphore(tool_name)
@@ -93,6 +95,7 @@ async def run_admitted(tool_name: str, func: Callable[..., T], *args: Any) -> T:
9395
try:
9496
return await asyncio.shield(worker)
9597
except asyncio.CancelledError:
98+
abandoned_work.record_abandoned(tool_name)
9699
worker.add_done_callback(functools.partial(_log_abandoned, tool_name))
97100
raise
98101

@@ -108,12 +111,21 @@ def _log_abandoned(tool_name: str, worker: asyncio.Future) -> None:
108111
109112
source: issue #728"""
110113
if worker.cancelled():
111-
outcome = "was cancelled"
114+
outcome: abandoned_work.Outcome = "cancelled"
115+
text = "was cancelled"
112116
elif worker.exception() is not None:
113-
outcome = f"raised {type(worker.exception()).__name__}"
117+
outcome = "raised"
118+
text = f"raised {type(worker.exception()).__name__}"
114119
else:
115120
outcome = "completed"
116-
logger.warning("%s worker %s after its caller was cancelled", tool_name, outcome)
121+
text = "completed"
122+
still_running = abandoned_work.record_ended(tool_name, outcome)
123+
logger.warning(
124+
"%s worker %s after its caller was cancelled (%d abandoned still running)",
125+
tool_name,
126+
text,
127+
still_running,
128+
)
117129

118130

119131
def current_budget(tool_name: str) -> int:
Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
"""Issue #728, slice 2: the status of workers that outlive their caller.
2+
3+
A cancelled tool call leaves its worker running. While it runs it is counted
4+
in ``abandoned_work`` under ``in_flight``; when it ends it moves to ``ended``
5+
under its outcome, and the log line names how many are still running.
6+
7+
Orderings are driven by ``threading.Event``; ``_GUARD`` only bounds a wait a
8+
broken implementation would never satisfy. No verdict depends on elapsed time.
9+
"""
10+
11+
from __future__ import annotations
12+
13+
import asyncio
14+
import logging
15+
import threading
16+
17+
import pytest
18+
19+
from mcp_server.handlers import abandoned_work
20+
from mcp_server.handlers.admission import reset_semaphores, run_admitted
21+
22+
# source: mcp_server/handlers/latency_class.py — remember is interactive,
23+
# budget 4 (admission override), so two held slots leave room for a third call.
24+
_TOOL = "remember"
25+
# source: deadlock guard, not a measurement.
26+
_GUARD = 5.0
27+
_LOGGER = "mcp_server.handlers.admission"
28+
29+
30+
@pytest.fixture(autouse=True)
31+
def _clean():
32+
reset_semaphores()
33+
abandoned_work.reset()
34+
yield
35+
reset_semaphores()
36+
abandoned_work.reset()
37+
38+
39+
async def _abandon(func) -> asyncio.Future:
40+
"""Start ``func`` admitted, wait until it runs, cancel its caller.
41+
42+
Returns the worker future, which outlives the caller."""
43+
started = threading.Event()
44+
45+
def body():
46+
started.set()
47+
return func()
48+
49+
before = asyncio.all_tasks()
50+
caller = asyncio.ensure_future(run_admitted(_TOOL, body))
51+
assert await asyncio.to_thread(started.wait, _GUARD)
52+
(worker,) = asyncio.all_tasks() - before - {caller}
53+
caller.cancel()
54+
with pytest.raises(asyncio.CancelledError):
55+
await caller
56+
return worker
57+
58+
59+
async def _settle(*workers: asyncio.Future) -> None:
60+
"""Wait for the workers to end.
61+
62+
``asyncio.wait`` registers its done callback after admission's, so by the
63+
time it returns the outcome has been recorded."""
64+
_, pending = await asyncio.wait(workers, timeout=_GUARD)
65+
assert not pending
66+
67+
68+
def _late_failure():
69+
raise RuntimeError("late failure")
70+
71+
72+
@pytest.mark.asyncio
73+
async def test_running_abandoned_worker_is_in_flight_until_it_ends():
74+
release = threading.Event()
75+
worker = await _abandon(lambda: release.wait(_GUARD))
76+
assert abandoned_work.snapshot() == {"in_flight": {_TOOL: 1}, "ended": {}}
77+
78+
release.set()
79+
await _settle(worker)
80+
assert abandoned_work.snapshot() == {
81+
"in_flight": {},
82+
"ended": {_TOOL: {"completed": 1, "raised": 0, "cancelled": 0}},
83+
}
84+
85+
86+
@pytest.mark.asyncio
87+
async def test_each_outcome_is_counted_apart():
88+
release = threading.Event()
89+
90+
def fails_after_release():
91+
release.wait(_GUARD)
92+
_late_failure()
93+
94+
first = await _abandon(lambda: release.wait(_GUARD))
95+
second = await _abandon(fails_after_release)
96+
assert abandoned_work.snapshot()["in_flight"] == {_TOOL: 2}
97+
98+
release.set()
99+
await _settle(first, second)
100+
assert abandoned_work.snapshot() == {
101+
"in_flight": {},
102+
"ended": {_TOOL: {"completed": 1, "raised": 1, "cancelled": 0}},
103+
}
104+
105+
106+
@pytest.mark.asyncio
107+
async def test_a_caller_that_waits_is_never_counted():
108+
assert await asyncio.wait_for(run_admitted(_TOOL, lambda: 7), _GUARD) == 7
109+
with pytest.raises(RuntimeError):
110+
await asyncio.wait_for(run_admitted(_TOOL, _late_failure), _GUARD)
111+
assert abandoned_work.snapshot() == {"in_flight": {}, "ended": {}}
112+
113+
114+
@pytest.mark.asyncio
115+
async def test_same_outcome_adds_up_and_the_log_counts_down(caplog):
116+
caplog.set_level(logging.WARNING, logger=_LOGGER)
117+
first_go, second_go = threading.Event(), threading.Event()
118+
first = await _abandon(lambda: first_go.wait(_GUARD))
119+
second = await _abandon(lambda: second_go.wait(_GUARD))
120+
121+
first_go.set()
122+
await _settle(first)
123+
assert abandoned_work.snapshot()["in_flight"] == {_TOOL: 1}
124+
second_go.set()
125+
await _settle(second)
126+
127+
assert abandoned_work.snapshot() == {
128+
"in_flight": {},
129+
"ended": {_TOOL: {"completed": 2, "raised": 0, "cancelled": 0}},
130+
}
131+
assert [r.getMessage() for r in caplog.records if r.name == _LOGGER] == [
132+
f"{_TOOL} worker completed after its caller was cancelled"
133+
" (1 abandoned still running)",
134+
f"{_TOOL} worker completed after its caller was cancelled"
135+
" (0 abandoned still running)",
136+
]

‎tests_py/handlers/test_admission_cancellation.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ async def test_abandoned_worker_success_is_logged(workers, warnings_):
127127
await _admitted(lambda: None)
128128
assert warnings_() == [
129129
"consolidate worker completed after its caller was cancelled"
130+
" (0 abandoned still running)"
130131
]
131132

132133

@@ -137,6 +138,7 @@ async def test_abandoned_worker_error_is_logged_and_retrieved(workers, warnings_
137138
await _admitted(lambda: None)
138139
assert warnings_() == [
139140
"consolidate worker raised RuntimeError after its caller was cancelled"
141+
" (0 abandoned still running)"
140142
]
141143

142144

@@ -156,5 +158,6 @@ async def test_abandoned_worker_cancelled_by_loop_teardown_is_logged(
156158
await worker
157159
assert warnings_() == [
158160
"consolidate worker was cancelled after its caller was cancelled"
161+
" (0 abandoned still running)"
159162
]
160163
workers.release.set()

0 commit comments

Comments
 (0)