Skip to content

Commit f0dd560

Browse files
AtulJoshi1206copybara-github
authored andcommitted
fix: cancel the in-flight tool call when the caller is cancelled
Merge #7199 PiperOrigin-RevId: 991901077
1 parent 41bebdc commit f0dd560

2 files changed

Lines changed: 259 additions & 9 deletions

File tree

‎src/google/adk/tools/mcp_tool/session_context.py‎

Lines changed: 48 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,41 @@ def _warn_probe_unavailable() -> None:
7676
)
7777

7878

79+
_CANCEL_DRAIN_TIMEOUT: float = 5.0
80+
81+
82+
async def _cancel_and_drain(
83+
task: asyncio.Future[Any],
84+
*,
85+
timeout: float | None = None,
86+
) -> None:
87+
"""Cancels an in-flight call and waits for it to actually stop.
88+
89+
Returning before the call has unwound would let it keep reading the
90+
transport after the caller believes it is finished. A timeout bound
91+
prevents a call that does not unwind promptly from turning a
92+
cancellation into an indefinite hang.
93+
"""
94+
task.cancel()
95+
if timeout is None:
96+
timeout = _CANCEL_DRAIN_TIMEOUT
97+
try:
98+
done, _ = await asyncio.wait([task], timeout=timeout)
99+
if task in done:
100+
if not task.cancelled():
101+
task.exception()
102+
else:
103+
task.add_done_callback(lambda t: None if t.cancelled() else t.exception())
104+
logger.warning(
105+
'Timed out after %ss waiting for cancelled task to unwind',
106+
timeout,
107+
)
108+
except BaseException:
109+
if not task.done():
110+
task.add_done_callback(lambda t: None if t.cancelled() else t.exception())
111+
raise
112+
113+
79114
def _read_timeout(seconds: Optional[float]) -> Optional[float | timedelta]:
80115
"""Converts a timeout in seconds to the type ``ClientSession`` expects.
81116
@@ -311,10 +346,18 @@ async def _run_guarded(self, coro: Coroutine[Any, Any, _T]) -> _T:
311346

312347
coro_task = asyncio.ensure_future(coro)
313348

314-
done, _ = await asyncio.wait(
315-
[coro_task, self._task],
316-
return_when=asyncio.FIRST_COMPLETED,
317-
)
349+
try:
350+
done, _ = await asyncio.wait(
351+
[coro_task, self._task],
352+
return_when=asyncio.FIRST_COMPLETED,
353+
)
354+
except BaseException:
355+
# asyncio.wait does not own what it waits on, so it leaves the call
356+
# running when this frame is cancelled. The caller's `finally` then
357+
# releases the session back to the pool while that call is still
358+
# reading the transport, and the pool is free to evict it underneath.
359+
await _cancel_and_drain(coro_task)
360+
raise
318361

319362
if coro_task in done:
320363
# If the coroutine itself raised, the exception propagates as-is
@@ -325,11 +368,7 @@ async def _run_guarded(self, coro: Coroutine[Any, Any, _T]) -> _T:
325368

326369
# The background task finished first, indicating a transport crash.
327370
# Cancel the in-flight tool call and surface the original error.
328-
coro_task.cancel()
329-
try:
330-
await coro_task
331-
except BaseException:
332-
pass
371+
await _cancel_and_drain(coro_task)
333372

334373
exc = self._task.exception() if not self._task.cancelled() else None
335374
raise ConnectionError(

‎tests/unittests/tools/mcp_tool/test_session_context.py‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
from google.adk.features import FeatureName
2727
from google.adk.features._feature_registry import temporary_feature_override
28+
from google.adk.tools.mcp_tool.session_context import _cancel_and_drain
2829
from google.adk.tools.mcp_tool.session_context import _connect
2930
from google.adk.tools.mcp_tool.session_context import _format_exception
3031
from google.adk.tools.mcp_tool.session_context import _read_timeout
@@ -999,6 +1000,216 @@ async def kill_background_task():
9991000
finally:
10001001
await killer
10011002

1003+
@pytest.mark.asyncio
1004+
async def test_run_guarded_cancels_coro_when_caller_is_cancelled(self):
1005+
"""Cancelling the caller also cancels the in-flight tool call.
1006+
1007+
asyncio.wait does not own the futures it waits on, so a cancelled
1008+
caller would leave the call running against a session that
1009+
McpTool._run_async_impl has already released back to the pool.
1010+
"""
1011+
mock_client = MockClient()
1012+
session_context = SessionContext(
1013+
mock_client, timeout=5.0, sse_read_timeout=None
1014+
)
1015+
1016+
with patch(
1017+
'google.adk.tools.mcp_tool.session_context.ClientSession'
1018+
) as mock_session_class:
1019+
mock_session_class.return_value = MockClientSession()
1020+
await session_context.start()
1021+
1022+
coro_started = asyncio.Event()
1023+
coro_was_cancelled = False
1024+
1025+
async def slow_coro():
1026+
nonlocal coro_was_cancelled
1027+
coro_started.set()
1028+
try:
1029+
await asyncio.sleep(300)
1030+
return 'should never reach here'
1031+
except asyncio.CancelledError:
1032+
coro_was_cancelled = True
1033+
raise
1034+
1035+
caller = asyncio.create_task(session_context._run_guarded(slow_coro()))
1036+
await coro_started.wait()
1037+
caller.cancel()
1038+
1039+
with pytest.raises(asyncio.CancelledError):
1040+
await caller
1041+
1042+
assert coro_was_cancelled is True
1043+
1044+
@pytest.mark.asyncio
1045+
async def test_cancel_and_drain_bounds_wait_when_task_does_not_unwind(
1046+
self, caplog: pytest.LogCaptureFixture
1047+
):
1048+
"""If a task does not unwind promptly, drain gives up after the timeout bound."""
1049+
coro_started = asyncio.Event()
1050+
1051+
async def stubborn_coro():
1052+
coro_started.set()
1053+
try:
1054+
await asyncio.sleep(300)
1055+
except asyncio.CancelledError:
1056+
# Deliberately ignore prompt unwinding
1057+
await asyncio.sleep(300)
1058+
1059+
task = asyncio.create_task(stubborn_coro())
1060+
await coro_started.wait()
1061+
1062+
start = time.monotonic()
1063+
with caplog.at_level(logging.WARNING, logger='google_adk'):
1064+
await _cancel_and_drain(task, timeout=0.05)
1065+
elapsed = time.monotonic() - start
1066+
1067+
assert elapsed < 1.0
1068+
assert not task.done()
1069+
assert (
1070+
'Timed out after 0.05s waiting for cancelled task to unwind'
1071+
in caplog.text
1072+
)
1073+
1074+
# Clean up stubborn task
1075+
task.cancel()
1076+
try:
1077+
await task
1078+
except asyncio.CancelledError:
1079+
pass
1080+
1081+
@pytest.mark.asyncio
1082+
async def test_cancel_and_drain_attaches_done_callback_on_timeout(self):
1083+
"""When drain times out, a done callback is attached to retrieve later exceptions."""
1084+
coro_started = asyncio.Event()
1085+
task_done = asyncio.Event()
1086+
1087+
async def stubborn_failing_coro():
1088+
coro_started.set()
1089+
try:
1090+
await asyncio.sleep(300)
1091+
except asyncio.CancelledError:
1092+
try:
1093+
await asyncio.sleep(0.02)
1094+
raise RuntimeError('belated failure')
1095+
finally:
1096+
task_done.set()
1097+
1098+
task = asyncio.create_task(stubborn_failing_coro())
1099+
await coro_started.wait()
1100+
1101+
with patch.object(
1102+
task, 'add_done_callback', wraps=task.add_done_callback
1103+
) as mock_add_callback:
1104+
await _cancel_and_drain(task, timeout=0.005)
1105+
callbacks = [call.args[0] for call in mock_add_callback.call_args_list]
1106+
assert any(cb.__name__ == '<lambda>' for cb in callbacks)
1107+
1108+
await task_done.wait()
1109+
assert task.done()
1110+
assert isinstance(task.exception(), RuntimeError)
1111+
1112+
@pytest.mark.asyncio
1113+
async def test_cancel_and_drain_bounds_wait_when_timeout_is_zero(self):
1114+
"""If timeout is zero, drain polls completion immediately without awaiting unbounded."""
1115+
coro_started = asyncio.Event()
1116+
1117+
async def stubborn_coro():
1118+
coro_started.set()
1119+
try:
1120+
await asyncio.sleep(300)
1121+
except asyncio.CancelledError:
1122+
await asyncio.sleep(300)
1123+
1124+
task = asyncio.create_task(stubborn_coro())
1125+
await coro_started.wait()
1126+
1127+
drain_task = asyncio.create_task(_cancel_and_drain(task, timeout=0.0))
1128+
await asyncio.sleep(0.05)
1129+
try:
1130+
assert drain_task.done(), (
1131+
'drain with timeout=0.0 hung on stubborn task instead of returning'
1132+
' immediately'
1133+
)
1134+
finally:
1135+
drain_task.cancel()
1136+
task.cancel()
1137+
await asyncio.gather(drain_task, task, return_exceptions=True)
1138+
1139+
@pytest.mark.asyncio
1140+
async def test_run_guarded_unwinds_when_caller_cancelled_even_if_coro_hangs(
1141+
self,
1142+
):
1143+
"""Caller cancellation unwinds promptly even if the coro does not unwind."""
1144+
mock_client = MockClient()
1145+
session_context = SessionContext(
1146+
mock_client, timeout=5.0, sse_read_timeout=None
1147+
)
1148+
1149+
with (
1150+
patch(
1151+
'google.adk.tools.mcp_tool.session_context.ClientSession'
1152+
) as mock_session_class,
1153+
patch(
1154+
'google.adk.tools.mcp_tool.session_context._CANCEL_DRAIN_TIMEOUT',
1155+
0.05,
1156+
),
1157+
):
1158+
mock_session_class.return_value = MockClientSession()
1159+
await session_context.start()
1160+
1161+
coro_started = asyncio.Event()
1162+
1163+
async def stubborn_coro():
1164+
coro_started.set()
1165+
try:
1166+
await asyncio.sleep(300)
1167+
except asyncio.CancelledError:
1168+
await asyncio.sleep(300)
1169+
1170+
caller = asyncio.create_task(
1171+
session_context._run_guarded(stubborn_coro())
1172+
)
1173+
await coro_started.wait()
1174+
1175+
start = time.monotonic()
1176+
caller.cancel()
1177+
1178+
with pytest.raises(asyncio.CancelledError):
1179+
await caller
1180+
elapsed = time.monotonic() - start
1181+
1182+
assert elapsed < 1.0
1183+
1184+
@pytest.mark.asyncio
1185+
async def test_cancel_and_drain_propagates_cancellation_to_caller(self):
1186+
"""Cancelling the task executing _cancel_and_drain must raise CancelledError."""
1187+
coro_started = asyncio.Event()
1188+
1189+
async def stubborn_coro():
1190+
coro_started.set()
1191+
try:
1192+
await asyncio.sleep(300)
1193+
except asyncio.CancelledError:
1194+
await asyncio.sleep(300)
1195+
1196+
task = asyncio.create_task(stubborn_coro())
1197+
await coro_started.wait()
1198+
1199+
drain_caller = asyncio.create_task(_cancel_and_drain(task, timeout=5.0))
1200+
await asyncio.sleep(0.01)
1201+
drain_caller.cancel()
1202+
1203+
try:
1204+
with pytest.raises(asyncio.CancelledError):
1205+
await drain_caller
1206+
finally:
1207+
task.cancel()
1208+
try:
1209+
await task
1210+
except asyncio.CancelledError:
1211+
pass
1212+
10021213

10031214
class TestSessionContextFlagOffPreservesPreFixBehavior:
10041215
"""Pin down that flag=OFF reproduces pre-fix behavior exactly.

0 commit comments

Comments
 (0)