Skip to content

Commit c08335d

Browse files
committed
fix(ipc): unblock thread executor aclose() when initialize is cancelled
A cancelled initialize() never resolves the initialize future (its CancelledError isn't caught), so the main task never started the monitor task, the sole reader of the IPC channel. aclose() then waited forever for a ShuttingDown that nobody read, hanging the worker shutdown (ThreadJobExecutor is the default executor on Windows and forced by the devmode CLI). Fail the future from aclose() when it is still unresolved, mirroring the SupervisedProc guard, so the main task starts the monitor task and the runner acks the queued shutdown request.
1 parent ceb3d59 commit c08335d

2 files changed

Lines changed: 44 additions & 1 deletion

File tree

‎livekit-agents/livekit/agents/ipc/job_thread_executor.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,15 @@ async def aclose(self) -> None:
213213
if not self.started:
214214
return
215215

216+
if not self._initialize_fut.done():
217+
# never initialized (a cancelled initialize() also leaves this future
218+
# unresolved, its CancelledError isn't caught): unblock the main task
219+
# waiting on this future so it starts the monitor task, the sole reader
220+
# of the channel, and the runner can ack the queued shutdown request
221+
self._initialize_fut.set_exception(
222+
RuntimeError("runner closed before initialization completed")
223+
)
224+
216225
self._closing = True
217226
with contextlib.suppress(utils.aio.duplex_unix.DuplexClosed):
218227
await channel.asend_message(self._pch, proto.ShutdownRequest())

‎tests/test_ipc.py‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import logging
77
import multiprocessing as mp
88
import socket
9+
import threading
910
import time
1011
import uuid
1112
from collections.abc import Callable
@@ -614,13 +615,14 @@ def _create_thread_proc(
614615
*,
615616
close_timeout: float,
616617
job_entrypoint_fnc: Callable[[JobContext], object] = _job_entrypoint,
618+
initialize_process_fnc: Callable[[JobProcess], None] = _initialize_proc,
617619
) -> tuple[ipc.job_thread_executor.ThreadJobExecutor, _StartArgs]:
618620
# the runner is a thread of the test process, so the mp primitives of _StartArgs are plain
619621
# in-process synchronization here
620622
start_args = _new_start_args(mp.get_context("spawn"))
621623
loop = asyncio.get_running_loop()
622624
proc = ipc.job_thread_executor.ThreadJobExecutor(
623-
initialize_process_fnc=_initialize_proc,
625+
initialize_process_fnc=initialize_process_fnc,
624626
job_entrypoint_fnc=job_entrypoint_fnc,
625627
session_end_fnc=None,
626628
simulation_end_fnc=None,
@@ -668,6 +670,38 @@ async def test_thread_shutdown_no_job_join_before_ack_is_read():
668670
assert start_args.shutdown_counter.value == 0
669671

670672

673+
async def test_thread_aclose_after_initialize_cancelled():
674+
"""Regression test: a cancelled initialize() never resolves the initialize future,
675+
so the main task never started the monitor task (the sole channel reader) and
676+
aclose() waited forever for a ShuttingDown message that nobody read, hanging the
677+
worker shutdown."""
678+
init_started = threading.Event()
679+
release_init = threading.Event()
680+
681+
def _initialize_proc_gated(proc: JobProcess) -> None:
682+
# hold the runner's warm-up until initialize() has been cancelled, so the
683+
# cancel can't race with a warm-up that already finished
684+
init_started.set()
685+
release_init.wait(timeout=10.0)
686+
687+
proc, _ = _create_thread_proc(close_timeout=0.5, initialize_process_fnc=_initialize_proc_gated)
688+
await proc.start()
689+
690+
init_task = asyncio.create_task(proc.initialize())
691+
try:
692+
await _poll_until(init_started.is_set)
693+
694+
init_task.cancel()
695+
with pytest.raises(asyncio.CancelledError):
696+
await init_task
697+
698+
release_init.set() # the runner acks the queued shutdown once its warm-up ends
699+
await asyncio.wait_for(proc.aclose(), timeout=5.0)
700+
assert proc.status == ipc.job_executor.JobStatus.SUCCESS
701+
finally:
702+
release_init.set() # never leave the runner blocked in its warm-up
703+
704+
671705
async def test_thread_job_graceful_shutdown():
672706
proc, start_args = _create_thread_proc(close_timeout=10.0)
673707
start_args.shutdown_simulate_work_time = 0.3

0 commit comments

Comments
 (0)