Skip to content

Commit eb3d520

Browse files
dyurk-lilaclaude
andcommitted
[skyrl-train][bugfix] Run the sampler weight broadcast off the event loop
save_weights_for_sampler ran the broadcast's blocking ray.get inline in an async def, freezing the trainer's event loop for the whole transfer. Under the fully-async trainer the generator coroutines share that loop, so environment and tool work stalled for the transfer and any timeout armed just before it fired on the first tick after. Run the transfer in a worker thread. The thread is shielded: a cancellation arriving mid-transfer waits for it to finish before propagating, so the resume bracket never resumes generation onto partially transferred weights (the blocking version could not be cancelled mid-transfer either). Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Dominic Yurk <[email protected]>
1 parent b304652 commit eb3d520

2 files changed

Lines changed: 82 additions & 9 deletions

File tree

‎skyrl/backends/skyrl_train/workers/worker_dispatch.py‎

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
The trainer interacts with the worker dispatch if all models are always on GPU.
99
"""
1010

11+
import asyncio
1112
import time
1213
from dataclasses import dataclass
1314
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
@@ -798,17 +799,38 @@ async def save_weights_for_sampler(self, model_id: Optional[str] = None) -> None
798799

799800
adapter_only_sync = self.colocate_all and self._is_lora_no_merge()
800801

801-
def _broadcast_and_finish() -> None:
802+
def _transfer() -> None:
803+
self._broadcast_to_inference_engines(self._inference_engine_client, model_id=model_id)
804+
self._finish_weight_sync(adapter_only_sync=adapter_only_sync)
805+
806+
async def _broadcast_and_finish() -> None:
802807
"""The weight transfer proper, timed on its own.
803808
809+
The transfer blocks on ``ray.get`` for the whole broadcast, so it runs in a
810+
worker thread: on the event loop it would stall every coroutine sharing the
811+
loop (the fully-async generators) and expire their timeouts. A cancellation
812+
waits for the transfer to finish before propagating, so the enclosing resume
813+
bracket never resumes generation onto partially transferred weights.
814+
804815
``last_weight_sync_seconds`` is surfaced as
805816
``timing/sync_weights_only_transfer`` via :meth:`get_timing_metrics`,
806817
alongside the trainer's own ``sync_weights`` timer, which wraps the
807818
enclosing pause/resume bracket too.
808819
"""
809820
start = time.perf_counter()
810-
self._broadcast_to_inference_engines(self._inference_engine_client, model_id=model_id)
811-
self._finish_weight_sync(adapter_only_sync=adapter_only_sync)
821+
with deadline.operation("broadcast_to_inference_engines"):
822+
transfer = asyncio.ensure_future(asyncio.to_thread(_transfer))
823+
cancelled = False
824+
while not transfer.done():
825+
try:
826+
await asyncio.shield(transfer)
827+
except asyncio.CancelledError:
828+
cancelled = True
829+
if cancelled:
830+
if transfer.exception() is not None:
831+
logger.warning(f"Weight transfer failed while cancelled: {transfer.exception()!r}")
832+
raise asyncio.CancelledError
833+
transfer.result()
812834
self.last_weight_sync_seconds = time.perf_counter() - start
813835

814836
# Sync weights to inference engine
@@ -818,7 +840,7 @@ def _broadcast_and_finish() -> None:
818840
self.ensure_active_adapter("policy", model_id, require_model_resident=not adapter_only_sync)
819841
if self.colocate_all:
820842
await self._inference_engine_client.wake_up(tags=["weights"])
821-
_broadcast_and_finish()
843+
await _broadcast_and_finish()
822844
await self._inference_engine_client.wake_up(tags=["kv_cache"])
823845
else:
824846
strategy = self.cfg.trainer.strategy
@@ -827,7 +849,7 @@ def _broadcast_and_finish() -> None:
827849
strategy == "megatron" and self.cfg.trainer.policy.megatron_config.lora_config.merge_lora
828850
):
829851
# in-place lora case (mostly for multi-tenant training) - no need to pause - can just rely on load_lora_adapter to swap adapter in place
830-
_broadcast_and_finish()
852+
await _broadcast_and_finish()
831853
elif self.cfg.generator.inference_engine.offload_kv_for_weight_sync:
832854
# Sleep the engine to free GPU memory during the sync (wake weights,
833855
# broadcast, wake KV cache) so gpu_memory_utilization can run higher.
@@ -844,7 +866,7 @@ def _broadcast_and_finish() -> None:
844866
):
845867
await self._inference_engine_client.sleep_for_weight_sync(offload_kv=offload_kv)
846868
await self._inference_engine_client.wake_for_weight_sync(tags=["weights"])
847-
_broadcast_and_finish()
869+
await _broadcast_and_finish()
848870
await self._inference_engine_client.wake_for_weight_sync(tags=["kv_cache"])
849871
else:
850872
# Synchronous trainer: generation is complete at sync time, so there
@@ -855,20 +877,20 @@ def _broadcast_and_finish() -> None:
855877
lambda: self._inference_engine_client.wake_up(tags=["kv_cache"]), "wake_up(kv_cache)"
856878
):
857879
await self._inference_engine_client.wake_up(tags=["weights"])
858-
_broadcast_and_finish()
880+
await _broadcast_and_finish()
859881
else:
860882
# Non-colocated single tenant: pause generation to prevent in-flight requests from
861883
# reading partially-updated weights during the NCCL broadcast.
862884
if self.cfg.generator.inference_engine.weight_sync_backend == "delta":
863885
# Delta disk sync performs publish/fetch before pausing and
864886
# pauses internally only around the final reload.
865-
_broadcast_and_finish()
887+
await _broadcast_and_finish()
866888
else:
867889
await self._inference_engine_client.pause_generation()
868890
async with cleanup_preserving_primary(
869891
self._inference_engine_client.resume_generation, "resume_generation"
870892
):
871-
_broadcast_and_finish()
893+
await _broadcast_and_finish()
872894

873895
# Advance the policy version so prefix-cache salting isolates blocks from the previous weights
874896
# (see `GeneratorConfig.use_cache_salt`).

‎tests/backends/skyrl_train/distributed/test_worker_dispatch.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
dispatch is built via ``__new__`` with mocked actor groups and client.
77
"""
88

9+
import asyncio
910
import time
1011
from types import SimpleNamespace
1112
from unittest.mock import AsyncMock, MagicMock
@@ -259,6 +260,56 @@ async def test_non_colocated_megatron_merge_lora_still_pauses(self):
259260
dispatch._inference_engine_client.pause_generation.assert_awaited_once()
260261
dispatch._inference_engine_client.resume_generation.assert_awaited_once()
261262

263+
@staticmethod
264+
def _nccl_dispatch(broadcast):
265+
from skyrl.backends.skyrl_train.workers.worker_dispatch import WorkerDispatch
266+
267+
dispatch = WorkerDispatch.__new__(WorkerDispatch)
268+
dispatch.colocate_all = False
269+
dispatch.cfg = _fft_dispatch_cfg()
270+
dispatch._inference_engine_client = AsyncMock()
271+
dispatch._inference_engine_client.increment_weight_version = MagicMock()
272+
dispatch._broadcast_to_inference_engines = MagicMock(side_effect=broadcast)
273+
dispatch._prepare_for_weight_sync = AsyncMock()
274+
dispatch._finish_weight_sync = MagicMock()
275+
dispatch.ensure_active_adapter = MagicMock()
276+
return dispatch
277+
278+
@pytest.mark.asyncio
279+
async def test_broadcast_does_not_block_event_loop(self):
280+
"""The broadcast blocks on ``ray.get``. A timeout armed by a coroutine sharing the
281+
loop (a fully-async generator's HTTP request) must not expire because of it."""
282+
dispatch = self._nccl_dispatch(lambda *args, **kwargs: time.sleep(0.5))
283+
284+
sync = asyncio.create_task(dispatch.save_weights_for_sampler())
285+
async with asyncio.timeout(0.2):
286+
await asyncio.sleep(0.05)
287+
assert not sync.done()
288+
await sync
289+
290+
dispatch._broadcast_to_inference_engines.assert_called_once()
291+
assert dispatch.last_weight_sync_seconds >= 0.5
292+
293+
@pytest.mark.asyncio
294+
async def test_cancellation_waits_for_the_transfer_before_resuming(self):
295+
"""A cancellation mid-broadcast must not resume generation onto partially transferred weights."""
296+
events = []
297+
298+
def broadcast(*args, **kwargs):
299+
time.sleep(0.3)
300+
events.append("broadcast_done")
301+
302+
dispatch = self._nccl_dispatch(broadcast)
303+
dispatch._inference_engine_client.resume_generation.side_effect = lambda: events.append("resumed")
304+
305+
sync = asyncio.create_task(dispatch.save_weights_for_sampler())
306+
await asyncio.sleep(0.05)
307+
sync.cancel()
308+
with pytest.raises(asyncio.CancelledError):
309+
await sync
310+
311+
assert events == ["broadcast_done", "resumed"]
312+
262313

263314
def _adapter_sync_dispatch(*, model_on_gpu: bool, optimizer_on_gpu: bool = False):
264315
"""Dispatch wired for the colocated adapter-only sync path

0 commit comments

Comments
 (0)