88The trainer interacts with the worker dispatch if all models are always on GPU.
99"""
1010
11+ import asyncio
1112import time
1213from dataclasses import dataclass
1314from typing import TYPE_CHECKING , Any , Dict , List , Optional , Tuple
@@ -798,17 +799,38 @@ async def save_weights_for_sampler(self, model_id: Optional[str] = None) -> None
798799
799800 adapter_only_sync = self .colocate_all and self ._is_lora_no_merge ()
800801
801- def _broadcast_and_finish () -> None :
802+ def _transfer () -> None :
803+ self ._broadcast_to_inference_engines (self ._inference_engine_client , model_id = model_id )
804+ self ._finish_weight_sync (adapter_only_sync = adapter_only_sync )
805+
806+ async def _broadcast_and_finish () -> None :
802807 """The weight transfer proper, timed on its own.
803808
809+ The transfer blocks on ``ray.get`` for the whole broadcast, so it runs in a
810+ worker thread: on the event loop it would stall every coroutine sharing the
811+ loop (the fully-async generators) and expire their timeouts. A cancellation
812+ waits for the transfer to finish before propagating, so the enclosing resume
813+ bracket never resumes generation onto partially transferred weights.
814+
804815 ``last_weight_sync_seconds`` is surfaced as
805816 ``timing/sync_weights_only_transfer`` via :meth:`get_timing_metrics`,
806817 alongside the trainer's own ``sync_weights`` timer, which wraps the
807818 enclosing pause/resume bracket too.
808819 """
809820 start = time .perf_counter ()
810- self ._broadcast_to_inference_engines (self ._inference_engine_client , model_id = model_id )
811- self ._finish_weight_sync (adapter_only_sync = adapter_only_sync )
821+ with deadline .operation ("broadcast_to_inference_engines" ):
822+ transfer = asyncio .ensure_future (asyncio .to_thread (_transfer ))
823+ cancelled = False
824+ while not transfer .done ():
825+ try :
826+ await asyncio .shield (transfer )
827+ except asyncio .CancelledError :
828+ cancelled = True
829+ if cancelled :
830+ if transfer .exception () is not None :
831+ logger .warning (f"Weight transfer failed while cancelled: { transfer .exception ()!r} " )
832+ raise asyncio .CancelledError
833+ transfer .result ()
812834 self .last_weight_sync_seconds = time .perf_counter () - start
813835
814836 # Sync weights to inference engine
@@ -818,7 +840,7 @@ def _broadcast_and_finish() -> None:
818840 self .ensure_active_adapter ("policy" , model_id , require_model_resident = not adapter_only_sync )
819841 if self .colocate_all :
820842 await self ._inference_engine_client .wake_up (tags = ["weights" ])
821- _broadcast_and_finish ()
843+ await _broadcast_and_finish ()
822844 await self ._inference_engine_client .wake_up (tags = ["kv_cache" ])
823845 else :
824846 strategy = self .cfg .trainer .strategy
@@ -827,7 +849,7 @@ def _broadcast_and_finish() -> None:
827849 strategy == "megatron" and self .cfg .trainer .policy .megatron_config .lora_config .merge_lora
828850 ):
829851 # in-place lora case (mostly for multi-tenant training) - no need to pause - can just rely on load_lora_adapter to swap adapter in place
830- _broadcast_and_finish ()
852+ await _broadcast_and_finish ()
831853 elif self .cfg .generator .inference_engine .offload_kv_for_weight_sync :
832854 # Sleep the engine to free GPU memory during the sync (wake weights,
833855 # broadcast, wake KV cache) so gpu_memory_utilization can run higher.
@@ -844,7 +866,7 @@ def _broadcast_and_finish() -> None:
844866 ):
845867 await self ._inference_engine_client .sleep_for_weight_sync (offload_kv = offload_kv )
846868 await self ._inference_engine_client .wake_for_weight_sync (tags = ["weights" ])
847- _broadcast_and_finish ()
869+ await _broadcast_and_finish ()
848870 await self ._inference_engine_client .wake_for_weight_sync (tags = ["kv_cache" ])
849871 else :
850872 # Synchronous trainer: generation is complete at sync time, so there
@@ -855,20 +877,20 @@ def _broadcast_and_finish() -> None:
855877 lambda : self ._inference_engine_client .wake_up (tags = ["kv_cache" ]), "wake_up(kv_cache)"
856878 ):
857879 await self ._inference_engine_client .wake_up (tags = ["weights" ])
858- _broadcast_and_finish ()
880+ await _broadcast_and_finish ()
859881 else :
860882 # Non-colocated single tenant: pause generation to prevent in-flight requests from
861883 # reading partially-updated weights during the NCCL broadcast.
862884 if self .cfg .generator .inference_engine .weight_sync_backend == "delta" :
863885 # Delta disk sync performs publish/fetch before pausing and
864886 # pauses internally only around the final reload.
865- _broadcast_and_finish ()
887+ await _broadcast_and_finish ()
866888 else :
867889 await self ._inference_engine_client .pause_generation ()
868890 async with cleanup_preserving_primary (
869891 self ._inference_engine_client .resume_generation , "resume_generation"
870892 ):
871- _broadcast_and_finish ()
893+ await _broadcast_and_finish ()
872894
873895 # Advance the policy version so prefix-cache salting isolates blocks from the previous weights
874896 # (see `GeneratorConfig.use_cache_salt`).
0 commit comments