Skip to content

Commit 45e9056

Browse files
committed
fix(ipc): finish the in-flight initialize frame before aclose() takes the channel
aclose() stopped initialize()'s response read with a plain cancel, which could land between the frame header and its payload (the async duplex reads them with two separate readexactly calls). The header was lost while the payload stayed on the stream, so the monitor task handed the channel read the pending payload as a new frame header: decoding failed with a struct.error that escaped aclose() itself, and the ShutdownRequestAck/ShuttingDown messages were lost even though the runner exited cleanly. Let the read finish the frame it has started first (bounded at 1s, like the monitor handover in _main_task: a runner that dies closes its end of the channel and ends the read on its own). The cancel now only ever lands on a read parked on the next frame header, where nothing was consumed yet and nothing is lost.
1 parent 6b21603 commit 45e9056

2 files changed

Lines changed: 87 additions & 1 deletion

File tree

‎livekit-agents/livekit/agents/ipc/job_thread_executor.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,18 @@ async def aclose(self) -> None:
234234
# future is resolved below becomes a second reader of the duplex
235235
init_read_task = self._init_read_task
236236
if init_read_task is not None and not init_read_task.done():
237-
await utils.aio.cancel_and_wait(init_read_task)
237+
# the read must first finish the frame it has started: cancelling
238+
# between the frame header and its payload would desynchronize the
239+
# channel and the monitor task would read the pending payload as a
240+
# new frame header, failing to decode it and losing the messages
241+
# that follow (e.g. the shutdown ack). when the runner is
242+
# unresponsive instead, the read is parked on the next frame header
243+
# with nothing consumed and the cancel below loses nothing. the
244+
# bound only covers a runner that stops writing a frame halfway;
245+
# once its end of the channel closes, the read ends on its own
246+
await asyncio.wait([init_read_task], timeout=1.0)
247+
if not init_read_task.done():
248+
await utils.aio.cancel_and_wait(init_read_task)
238249

239250
if not self._initialize_fut.done():
240251
# never initialized (a cancelled initialize() also leaves this future

‎tests/test_ipc.py‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -743,6 +743,81 @@ def _initialize_proc_gated(proc: JobProcess) -> None:
743743
release_init.set() # never leave the runner blocked in its warm-up
744744

745745

746+
async def test_thread_aclose_during_inflight_response_frame():
747+
"""Regression test: aclose() used to cancel initialize()'s response read even
748+
when it had already consumed the frame header of the InitializeResponse.
749+
The monitor task then read the pending payload as a new frame header: the
750+
stream was desynchronized, decoding failed with a struct.error that escaped
751+
aclose() itself, and the ShutdownRequestAck/ShuttingDown messages were lost.
752+
The close must let the in-flight frame finish before handing the channel
753+
over to the monitor task."""
754+
proc, _ = _create_thread_proc(close_timeout=5.0)
755+
await proc.start()
756+
757+
# instrument the parent duplex (per instance, safe to run concurrently): hold
758+
# the first payload read so the close commits its strategy while the
759+
# response frame is in flight, and record every frame the duplex delivers
760+
# as an aligned header+payload pair
761+
reader = proc._pch._reader
762+
orig_readexactly = reader.readexactly
763+
header_consumed = asyncio.Event()
764+
payload_released = asyncio.Event()
765+
read_state = {"expect_header": True, "header": b"", "gated": False}
766+
frames: list[bytes] = []
767+
768+
async def _readexactly_spy(n: int) -> bytes:
769+
if read_state["expect_header"]:
770+
data = await orig_readexactly(n)
771+
read_state["header"] = data
772+
read_state["expect_header"] = False
773+
header_consumed.set()
774+
return data
775+
776+
if not read_state["gated"]:
777+
read_state["gated"] = True
778+
# the frame header is consumed but not its payload yet
779+
await payload_released.wait()
780+
data = await orig_readexactly(n)
781+
frames.append(read_state["header"] + data)
782+
read_state["expect_header"] = True
783+
return data
784+
785+
reader.readexactly = _readexactly_spy
786+
787+
init_task = asyncio.create_task(proc.initialize())
788+
await _poll_until(header_consumed.is_set)
789+
init_read_task = proc._init_read_task
790+
assert not init_read_task.done(), "the response read should be parked mid-frame"
791+
792+
close_task = asyncio.create_task(proc.aclose())
793+
# the close either cancels the mid-frame read right away (the old behavior,
794+
# which desynchronizes the stream) or waits for it to finish the frame;
795+
# once it has committed either way, let the response payload through
796+
for _ in range(20):
797+
if init_read_task.done():
798+
break
799+
await asyncio.sleep(0.01)
800+
payload_released.set()
801+
802+
try:
803+
await init_task
804+
except asyncio.CancelledError:
805+
pass # the close aborted the in-flight initialize()
806+
807+
await asyncio.wait_for(close_task, timeout=10.0)
808+
809+
await _poll_until(lambda: not proc._thread.is_alive())
810+
811+
assert proc.status == ipc.job_executor.JobStatus.SUCCESS
812+
813+
def _frame(msg: object) -> bytes:
814+
payload = ipc.channel._write_message(msg)
815+
return len(payload).to_bytes(4, "big") + payload
816+
817+
assert _frame(ipc.proto.ShutdownRequestAck()) in frames, "the monitor lost the shutdown ack"
818+
assert _frame(ipc.proto.ShuttingDown()) in frames, "the monitor lost the ShuttingDown message"
819+
820+
746821
async def test_thread_aclose_without_initialize():
747822
"""Regression test: start() then aclose() with no initialize(). The
748823
ShutdownRequest is the runner's very first message; it used to die on its

0 commit comments

Comments
 (0)