|
| 1 | +"""Issue #728, slice 2: the status of workers that outlive their caller. |
| 2 | +
|
| 3 | +A cancelled tool call leaves its worker running. While it runs it is counted |
| 4 | +in ``abandoned_work`` under ``in_flight``; when it ends it moves to ``ended`` |
| 5 | +under its outcome, and the log line names how many are still running. |
| 6 | +
|
| 7 | +Orderings are driven by ``threading.Event``; ``_GUARD`` only bounds a wait a |
| 8 | +broken implementation would never satisfy. No verdict depends on elapsed time. |
| 9 | +""" |
| 10 | + |
| 11 | +from __future__ import annotations |
| 12 | + |
| 13 | +import asyncio |
| 14 | +import logging |
| 15 | +import threading |
| 16 | + |
| 17 | +import pytest |
| 18 | + |
| 19 | +from mcp_server.handlers import abandoned_work |
| 20 | +from mcp_server.handlers.admission import reset_semaphores, run_admitted |
| 21 | + |
| 22 | +# source: mcp_server/handlers/latency_class.py — remember is interactive, |
| 23 | +# budget 4 (admission override), so two held slots leave room for a third call. |
| 24 | +_TOOL = "remember" |
| 25 | +# source: deadlock guard, not a measurement. |
| 26 | +_GUARD = 5.0 |
| 27 | +_LOGGER = "mcp_server.handlers.admission" |
| 28 | + |
| 29 | + |
| 30 | +@pytest.fixture(autouse=True) |
| 31 | +def _clean(): |
| 32 | + reset_semaphores() |
| 33 | + abandoned_work.reset() |
| 34 | + yield |
| 35 | + reset_semaphores() |
| 36 | + abandoned_work.reset() |
| 37 | + |
| 38 | + |
| 39 | +async def _abandon(func) -> asyncio.Future: |
| 40 | + """Start ``func`` admitted, wait until it runs, cancel its caller. |
| 41 | +
|
| 42 | + Returns the worker future, which outlives the caller.""" |
| 43 | + started = threading.Event() |
| 44 | + |
| 45 | + def body(): |
| 46 | + started.set() |
| 47 | + return func() |
| 48 | + |
| 49 | + before = asyncio.all_tasks() |
| 50 | + caller = asyncio.ensure_future(run_admitted(_TOOL, body)) |
| 51 | + assert await asyncio.to_thread(started.wait, _GUARD) |
| 52 | + (worker,) = asyncio.all_tasks() - before - {caller} |
| 53 | + caller.cancel() |
| 54 | + with pytest.raises(asyncio.CancelledError): |
| 55 | + await caller |
| 56 | + return worker |
| 57 | + |
| 58 | + |
| 59 | +async def _settle(*workers: asyncio.Future) -> None: |
| 60 | + """Wait for the workers to end. |
| 61 | +
|
| 62 | + ``asyncio.wait`` registers its done callback after admission's, so by the |
| 63 | + time it returns the outcome has been recorded.""" |
| 64 | + _, pending = await asyncio.wait(workers, timeout=_GUARD) |
| 65 | + assert not pending |
| 66 | + |
| 67 | + |
| 68 | +def _late_failure(): |
| 69 | + raise RuntimeError("late failure") |
| 70 | + |
| 71 | + |
| 72 | +@pytest.mark.asyncio |
| 73 | +async def test_running_abandoned_worker_is_in_flight_until_it_ends(): |
| 74 | + release = threading.Event() |
| 75 | + worker = await _abandon(lambda: release.wait(_GUARD)) |
| 76 | + assert abandoned_work.snapshot() == {"in_flight": {_TOOL: 1}, "ended": {}} |
| 77 | + |
| 78 | + release.set() |
| 79 | + await _settle(worker) |
| 80 | + assert abandoned_work.snapshot() == { |
| 81 | + "in_flight": {}, |
| 82 | + "ended": {_TOOL: {"completed": 1, "raised": 0, "cancelled": 0}}, |
| 83 | + } |
| 84 | + |
| 85 | + |
| 86 | +@pytest.mark.asyncio |
| 87 | +async def test_each_outcome_is_counted_apart(): |
| 88 | + release = threading.Event() |
| 89 | + |
| 90 | + def fails_after_release(): |
| 91 | + release.wait(_GUARD) |
| 92 | + _late_failure() |
| 93 | + |
| 94 | + first = await _abandon(lambda: release.wait(_GUARD)) |
| 95 | + second = await _abandon(fails_after_release) |
| 96 | + assert abandoned_work.snapshot()["in_flight"] == {_TOOL: 2} |
| 97 | + |
| 98 | + release.set() |
| 99 | + await _settle(first, second) |
| 100 | + assert abandoned_work.snapshot() == { |
| 101 | + "in_flight": {}, |
| 102 | + "ended": {_TOOL: {"completed": 1, "raised": 1, "cancelled": 0}}, |
| 103 | + } |
| 104 | + |
| 105 | + |
| 106 | +@pytest.mark.asyncio |
| 107 | +async def test_a_caller_that_waits_is_never_counted(): |
| 108 | + assert await asyncio.wait_for(run_admitted(_TOOL, lambda: 7), _GUARD) == 7 |
| 109 | + with pytest.raises(RuntimeError): |
| 110 | + await asyncio.wait_for(run_admitted(_TOOL, _late_failure), _GUARD) |
| 111 | + assert abandoned_work.snapshot() == {"in_flight": {}, "ended": {}} |
| 112 | + |
| 113 | + |
| 114 | +@pytest.mark.asyncio |
| 115 | +async def test_same_outcome_adds_up_and_the_log_counts_down(caplog): |
| 116 | + caplog.set_level(logging.WARNING, logger=_LOGGER) |
| 117 | + first_go, second_go = threading.Event(), threading.Event() |
| 118 | + first = await _abandon(lambda: first_go.wait(_GUARD)) |
| 119 | + second = await _abandon(lambda: second_go.wait(_GUARD)) |
| 120 | + |
| 121 | + first_go.set() |
| 122 | + await _settle(first) |
| 123 | + assert abandoned_work.snapshot()["in_flight"] == {_TOOL: 1} |
| 124 | + second_go.set() |
| 125 | + await _settle(second) |
| 126 | + |
| 127 | + assert abandoned_work.snapshot() == { |
| 128 | + "in_flight": {}, |
| 129 | + "ended": {_TOOL: {"completed": 2, "raised": 0, "cancelled": 0}}, |
| 130 | + } |
| 131 | + assert [r.getMessage() for r in caplog.records if r.name == _LOGGER] == [ |
| 132 | + f"{_TOOL} worker completed after its caller was cancelled" |
| 133 | + " (1 abandoned still running)", |
| 134 | + f"{_TOOL} worker completed after its caller was cancelled" |
| 135 | + " (0 abandoned still running)", |
| 136 | + ] |
0 commit comments