Skip to content

Commit ab16cf4

Browse files
dyurk-lilaclaude
andcommitted
[skyrl-train][bugfix] Wake the KV cache before resuming after an interrupted sync
On the fully-async offload-KV path, a broadcast that failed or was cancelled skipped wake_for_weight_sync(kv_cache), and the resume cleanup then scheduled the frozen requests onto a sleeping KV cache. Bracket the KV wake with cleanup_preserving_primary inside the resume bracket, as the synchronous-trainer path already does for its wake_up(kv_cache). Also wait on the transfer with asyncio.wait rather than shield: a transfer that failed after a cancellation arrived escaped the wait loop and replaced the cancellation. The cancellation now propagates, chained to the transfer's error. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Dominic Yurk <[email protected]>
1 parent eb3d520 commit ab16cf4

2 files changed

Lines changed: 59 additions & 9 deletions

File tree

‎skyrl/backends/skyrl_train/workers/worker_dispatch.py‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -809,8 +809,9 @@ async def _broadcast_and_finish() -> None:
809809
The transfer blocks on ``ray.get`` for the whole broadcast, so it runs in a
810810
worker thread: on the event loop it would stall every coroutine sharing the
811811
loop (the fully-async generators) and expire their timeouts. A cancellation
812-
waits for the transfer to finish before propagating, so the enclosing resume
813-
bracket never resumes generation onto partially transferred weights.
812+
waits for the transfer to finish before propagating (chained to the transfer's
813+
error, if any), so the enclosing resume bracket never resumes generation onto
814+
partially transferred weights.
814815
815816
``last_weight_sync_seconds`` is surfaced as
816817
``timing/sync_weights_only_transfer`` via :meth:`get_timing_metrics`,
@@ -823,13 +824,11 @@ async def _broadcast_and_finish() -> None:
823824
cancelled = False
824825
while not transfer.done():
825826
try:
826-
await asyncio.shield(transfer)
827+
await asyncio.wait({transfer})
827828
except asyncio.CancelledError:
828829
cancelled = True
829830
if cancelled:
830-
if transfer.exception() is not None:
831-
logger.warning(f"Weight transfer failed while cancelled: {transfer.exception()!r}")
832-
raise asyncio.CancelledError
831+
raise asyncio.CancelledError() from transfer.exception()
833832
transfer.result()
834833
self.last_weight_sync_seconds = time.perf_counter() - start
835834

@@ -865,9 +864,14 @@ async def _broadcast_and_finish() -> None:
865864
self._inference_engine_client.resume_generation, "resume_generation"
866865
):
867866
await self._inference_engine_client.sleep_for_weight_sync(offload_kv=offload_kv)
868-
await self._inference_engine_client.wake_for_weight_sync(tags=["weights"])
869-
await _broadcast_and_finish()
870-
await self._inference_engine_client.wake_for_weight_sync(tags=["kv_cache"])
867+
# Wake the KV cache even if the sync fails, so the resume above never
868+
# schedules the frozen requests onto a sleeping KV cache.
869+
async with cleanup_preserving_primary(
870+
lambda: self._inference_engine_client.wake_for_weight_sync(tags=["kv_cache"]),
871+
"wake_for_weight_sync(kv_cache)",
872+
):
873+
await self._inference_engine_client.wake_for_weight_sync(tags=["weights"])
874+
await _broadcast_and_finish()
871875
else:
872876
# Synchronous trainer: generation is complete at sync time, so there
873877
# are no in-flight requests to preserve. A plain sleep (discards KV,

‎tests/backends/skyrl_train/distributed/test_worker_dispatch.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,52 @@ def broadcast(*args, **kwargs):
310310

311311
assert events == ["broadcast_done", "resumed"]
312312

313+
@pytest.mark.asyncio
314+
async def test_cancellation_survives_a_failed_transfer(self):
315+
"""A transfer that fails after the cancellation arrived must not replace the cancellation."""
316+
317+
def broadcast(*args, **kwargs):
318+
time.sleep(0.3)
319+
raise RuntimeError("broadcast failed")
320+
321+
dispatch = self._nccl_dispatch(broadcast)
322+
323+
sync = asyncio.create_task(dispatch.save_weights_for_sampler())
324+
await asyncio.sleep(0.05)
325+
sync.cancel()
326+
with pytest.raises(asyncio.CancelledError) as raised:
327+
await sync
328+
329+
assert isinstance(raised.value.__cause__, RuntimeError)
330+
331+
@pytest.mark.asyncio
332+
@pytest.mark.parametrize("cancel, expected", [(False, RuntimeError), (True, asyncio.CancelledError)])
333+
async def test_offload_kv_wakes_kv_cache_before_resuming(self, cancel, expected):
334+
"""However the sync is interrupted, generation must not resume onto a sleeping KV cache."""
335+
events = []
336+
337+
def broadcast(*args, **kwargs):
338+
time.sleep(0.2)
339+
events.append("broadcast_done")
340+
if not cancel:
341+
raise RuntimeError("broadcast failed")
342+
343+
dispatch = self._nccl_dispatch(broadcast)
344+
dispatch.cfg.generator.inference_engine.offload_kv_for_weight_sync = True
345+
dispatch.cfg.trainer.fully_async = SimpleNamespace(enabled=True, clear_kv_cache_on_weight_sync=False)
346+
client = dispatch._inference_engine_client
347+
client.wake_for_weight_sync.side_effect = lambda tags: events.append(f"wake_{tags[0]}")
348+
client.resume_generation.side_effect = lambda: events.append("resumed")
349+
350+
sync = asyncio.create_task(dispatch.save_weights_for_sampler())
351+
await asyncio.sleep(0.05)
352+
if cancel:
353+
sync.cancel()
354+
with pytest.raises(expected):
355+
await sync
356+
357+
assert events == ["wake_weights", "broadcast_done", "wake_kv_cache", "resumed"]
358+
313359

314360
def _adapter_sync_dispatch(*, model_on_gpu: bool, optimizer_on_gpu: bool = False):
315361
"""Dispatch wired for the colocated adapter-only sync path

0 commit comments

Comments
 (0)