Skip to content

Commit 6b21603

Browse files
committed
fix(ipc): coordinate close with in-flight initialize and accept shutdown-before-init
Two gaps found by an automated review of the previous commit: aclose() racing a still-running (not cancelled) initialize() resolved the initialize future under the pending response read, starting the monitor task as a second reader of the duplex (a concurrent-read RuntimeError that also failed aclose()), and initialize() then crashed double-setting the already-failed future. aclose() now stops the tracked in-flight read first, and initialize() guards its future writes. aclose() without initialize() made the ShutdownRequest the runner's first message, which died on its "first message must be InitializeRequest" assert (an uncaught exception in the runner thread) while the executor reported SUCCESS. The proc client now accepts an initial ShutdownRequest and the runners exit cleanly.
1 parent c08335d commit 6b21603

5 files changed

Lines changed: 142 additions & 20 deletions

File tree

‎livekit-agents/livekit/agents/ipc/inference_proc_lazy_main.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,13 @@ def proc_main(args: ProcStartArgs) -> None:
4343

4444
client = _ProcClient(args.mp_cch, args.log_cch, inf_proc.initialize, inf_proc.entrypoint)
4545
try:
46-
client.initialize()
46+
initialized = client.initialize()
4747
except Exception:
4848
return # initialization failed, exit (initialize will send an error to the worker)
4949

50+
if not initialized:
51+
return # shutdown requested before initialization, exit
52+
5053
client.run()
5154

5255

‎livekit-agents/livekit/agents/ipc/job_proc_lazy_main.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,11 +95,15 @@ def proc_main(args: ProcStartArgs) -> None:
9595

9696
client = _ProcClient(args.mp_cch, args.log_cch, job_proc.initialize, job_proc.entrypoint)
9797
try:
98-
client.initialize()
98+
initialized = client.initialize()
9999
except Exception:
100100
log_handler.close()
101101
return # initialization failed, exit (initialize will send an error to the worker)
102102

103+
if not initialized:
104+
log_handler.close()
105+
return # shutdown requested before initialization, exit
106+
103107
client.run()
104108

105109
import sys
@@ -618,7 +622,10 @@ def thread_main(
618622
)
619623

620624
client = _ProcClient(args.mp_cch, None, job_proc.initialize, job_proc.entrypoint)
621-
client.initialize()
625+
if not client.initialize():
626+
# the worker closed the runner before it was initialized: nothing
627+
# ran, exit cleanly without starting the job loop
628+
return
622629
client.run()
623630
finally:
624631
args.join_fnc()

‎livekit-agents/livekit/agents/ipc/job_thread_executor.py‎

Lines changed: 39 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ def __init__(
6969

7070
self._main_atask: asyncio.Task[None] | None = None
7171
self._initialize_fut = asyncio.Future[None]()
72+
self._init_read_task: asyncio.Task[channel.Message] | None = None
7273
self._closing = False
7374
self._lock = asyncio.Lock()
7475
self._shutdown_ack_fut = asyncio.Future[None]()
@@ -180,10 +181,18 @@ async def initialize(self) -> None:
180181
try:
181182
logger.info("initializing job runner", extra=self.logging_extra())
182183
start_time = time.perf_counter()
183-
init_res = await asyncio.wait_for(
184-
channel.arecv_message(self._pch, proto.IPC_MESSAGES),
185-
timeout=self._opts.initialize_timeout,
186-
)
184+
# the response read is tracked so a concurrent aclose() can stop it
185+
# before the monitor task (started once the initialize future is
186+
# resolved) becomes a second reader of the duplex
187+
read_task = asyncio.create_task(channel.arecv_message(self._pch, proto.IPC_MESSAGES))
188+
self._init_read_task = read_task
189+
try:
190+
init_res = await asyncio.wait_for(read_task, timeout=self._opts.initialize_timeout)
191+
finally:
192+
# the read must never outlive initialize()
193+
if not read_task.done():
194+
await utils.aio.cancel_and_wait(read_task)
195+
self._init_read_task = None
187196
assert isinstance(init_res, proto.InitializeResponse), (
188197
"first message must be InitializeResponse"
189198
)
@@ -195,15 +204,21 @@ async def initialize(self) -> None:
195204
},
196205
)
197206
except asyncio.TimeoutError:
198-
self._initialize_fut.set_exception(
199-
asyncio.TimeoutError("runner initialization timed out")
200-
)
207+
if not self._initialize_fut.done():
208+
self._initialize_fut.set_exception(
209+
asyncio.TimeoutError("runner initialization timed out")
210+
)
201211
raise
202212
except Exception as e: # should be channel.ChannelClosed most of the time
203-
self._initialize_fut.set_exception(e)
213+
if not self._initialize_fut.done():
214+
self._initialize_fut.set_exception(e)
204215
raise
205216
else:
206-
self._initialize_fut.set_result(None)
217+
if not self._initialize_fut.done():
218+
# a concurrent aclose() may have failed the future while this read
219+
# was completing; the main task treats both as "initialization
220+
# did not complete"
221+
self._initialize_fut.set_result(None)
207222

208223
async def aclose(self) -> None:
209224
"""
@@ -214,13 +229,21 @@ async def aclose(self) -> None:
214229
return
215230

216231
if not self._initialize_fut.done():
217-
# never initialized (a cancelled initialize() also leaves this future
218-
# unresolved, its CancelledError isn't caught): unblock the main task
219-
# waiting on this future so it starts the monitor task, the sole reader
220-
# of the channel, and the runner can ack the queued shutdown request
221-
self._initialize_fut.set_exception(
222-
RuntimeError("runner closed before initialization completed")
223-
)
232+
# a still-running initialize() may be reading the channel; stop its
233+
# read first, otherwise the monitor task started once the initialize
234+
# future is resolved below becomes a second reader of the duplex
235+
init_read_task = self._init_read_task
236+
if init_read_task is not None and not init_read_task.done():
237+
await utils.aio.cancel_and_wait(init_read_task)
238+
239+
if not self._initialize_fut.done():
240+
# never initialized (a cancelled initialize() also leaves this future
241+
# unresolved, its CancelledError isn't caught): unblock the main task
242+
# waiting on this future so it starts the monitor task, the sole reader
243+
# of the channel, and the runner can ack the queued shutdown request
244+
self._initialize_fut.set_exception(
245+
RuntimeError("runner closed before initialization completed")
246+
)
224247

225248
self._closing = True
226249
with contextlib.suppress(utils.aio.duplex_unix.DuplexClosed):

‎livekit-agents/livekit/agents/ipc/proc_client.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
InitializeResponse,
1818
PingRequest,
1919
PongResponse,
20+
ShutdownRequest,
2021
)
2122

2223

@@ -33,11 +34,21 @@ def __init__(
3334
self._main_task_fnc = main_task_fnc
3435
self._initialized = False
3536

36-
def initialize(self) -> None:
37+
def initialize(self) -> bool:
38+
"""Read the InitializeRequest handshake. Returns False when the first
39+
message is a ShutdownRequest instead: the worker closed the runner before
40+
it was initialized (e.g. a thread executor closed without initialize()),
41+
no job ever ran, so the caller must exit without starting the job loop."""
3742
try:
3843
cch = aio.duplex_unix._Duplex.open(self._mp_cch)
3944
first_req = recv_message(cch, IPC_MESSAGES)
4045

46+
if isinstance(first_req, ShutdownRequest):
47+
logger.info("shutting down before initialization")
48+
with contextlib.suppress(aio.duplex_unix.DuplexClosed):
49+
cch.close()
50+
return False
51+
4152
assert isinstance(first_req, InitializeRequest), (
4253
"first message must be proto.InitializeRequest"
4354
)
@@ -52,6 +63,7 @@ def initialize(self) -> None:
5263

5364
self._initialized = True
5465
cch.detach()
66+
return True
5567
except aio.duplex_unix.DuplexClosed as e:
5668
raise RuntimeError("failed to initialize proc_client") from e
5769

‎tests/test_ipc.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -702,6 +702,83 @@ def _initialize_proc_gated(proc: JobProcess) -> None:
702702
release_init.set() # never leave the runner blocked in its warm-up
703703

704704

705+
async def test_thread_aclose_during_pending_initialize():
706+
"""Regression test: aclose() racing a still-running (not cancelled) initialize().
707+
The close used to resolve the initialize future while initialize()'s response read
708+
was still pending, starting the monitor task as a second reader of the duplex
709+
(a concurrent-read error that also failed aclose()), and initialize() then
710+
crashed double-setting the already-failed future. aclose() must first stop the
711+
in-flight read, and both calls must end cleanly."""
712+
init_started = threading.Event()
713+
release_init = threading.Event()
714+
715+
def _initialize_proc_gated(proc: JobProcess) -> None:
716+
# hold the runner's warm-up so initialize()'s response read stays pending
717+
init_started.set()
718+
release_init.wait(timeout=10.0)
719+
720+
proc, _ = _create_thread_proc(close_timeout=2.0, initialize_process_fnc=_initialize_proc_gated)
721+
await proc.start()
722+
723+
init_task = asyncio.create_task(proc.initialize())
724+
await _poll_until(init_started.is_set)
725+
assert not proc._initialize_fut.done(), "initialize() should still be reading"
726+
727+
close_task = asyncio.create_task(proc.aclose())
728+
try:
729+
# the deterministic window: aclose() resolves the initialize future while
730+
# the runner is still warming up (the response read still pending), and
731+
# only then is the warm-up allowed to finish
732+
await _poll_until(proc._initialize_fut.done, timeout=5.0)
733+
release_init.set()
734+
735+
await asyncio.wait_for(close_task, timeout=5.0)
736+
assert proc.status == ipc.job_executor.JobStatus.SUCCESS
737+
738+
# initialize() must fail cleanly (the close aborts its read), not crash
739+
# double-setting the resolved future
740+
with pytest.raises(asyncio.CancelledError):
741+
await init_task
742+
finally:
743+
release_init.set() # never leave the runner blocked in its warm-up
744+
745+
746+
async def test_thread_aclose_without_initialize():
747+
"""Regression test: start() then aclose() with no initialize(). The
748+
ShutdownRequest is the runner's very first message; it used to die on its
749+
"first message must be InitializeRequest" assert (an uncaught exception in
750+
the runner thread) while the executor reported SUCCESS. The runner must exit
751+
cleanly and the executor status must stay accurate."""
752+
from livekit.agents.ipc import job_proc_lazy_main
753+
754+
thread_crashed = threading.Event()
755+
original_thread_main = job_proc_lazy_main.thread_main
756+
757+
def _spy_thread_main(args) -> None:
758+
try:
759+
original_thread_main(args)
760+
except BaseException:
761+
thread_crashed.set()
762+
raise
763+
764+
job_proc_lazy_main.thread_main = _spy_thread_main
765+
try:
766+
proc, start_args = _create_thread_proc(close_timeout=2.0)
767+
await proc.start()
768+
769+
await asyncio.wait_for(proc.aclose(), timeout=5.0)
770+
finally:
771+
job_proc_lazy_main.thread_main = original_thread_main
772+
773+
# the executor can observe the join before the thread has fully unwound:
774+
# wait for the thread itself before judging whether it crashed
775+
await _poll_until(lambda: not proc._thread.is_alive())
776+
assert not thread_crashed.is_set(), "runner crashed on the initial ShutdownRequest"
777+
assert proc.status == ipc.job_executor.JobStatus.SUCCESS
778+
assert start_args.initialize_counter.value == 0
779+
assert start_args.shutdown_counter.value == 0, "shutdown_cb isn't called when there is no job"
780+
781+
705782
async def test_thread_job_graceful_shutdown():
706783
proc, start_args = _create_thread_proc(close_timeout=10.0)
707784
start_args.shutdown_simulate_work_time = 0.3

0 commit comments

Comments
 (0)