|
20 | 20 | from ._fsdp_param import FSDPParam, ShardedState |
21 | 21 |
|
22 | 22 |
|
23 | | -def _label_with_suffix(label: str, suffix: str) -> str: |
24 | | - if suffix: |
25 | | - return f"{label} {suffix}" |
26 | | - return label |
27 | | - |
28 | | - |
29 | 23 | class AllGatherResult(NamedTuple): |
30 | 24 | all_gather_output: torch.Tensor |
31 | 25 | all_gather_event: torch.Event | None |
@@ -336,49 +330,43 @@ def foreach_all_gather( |
336 | 330 | all_gather_stream: torch.Stream, |
337 | 331 | device: torch.device, |
338 | 332 | all_gather_comm: AllGather, |
339 | | - label_suffix: str = "", |
340 | 333 | ) -> AllGatherResult | None: |
341 | 334 | world_size, rank = group.size(), group.rank() |
342 | 335 | device_handle = _get_device_handle(device.type) |
343 | | - |
344 | 336 | with device_handle.stream(all_gather_copy_in_stream): |
345 | | - with torch.profiler.record_function( |
346 | | - _label_with_suffix("FSDP::all_gather_copy_in", label_suffix) |
347 | | - ): |
348 | | - param_all_gather_inputs = _get_param_all_gather_inputs(fsdp_params) |
349 | | - ( |
350 | | - param_all_gather_input_dtypes, |
351 | | - param_all_gather_input_numels, |
352 | | - dtype, |
353 | | - ) = _get_all_gather_input_metadatas(param_all_gather_inputs) |
354 | | - if dtype == torch.uint8: |
355 | | - all_gather_inputs = [ |
356 | | - t.view(torch.uint8) for ts in param_all_gather_inputs for t in ts |
357 | | - ] |
358 | | - else: |
359 | | - all_gather_inputs = [*chain.from_iterable(param_all_gather_inputs)] |
360 | | - inp_split_sizes = [t.numel() for t in all_gather_inputs] |
361 | | - all_gather_input_numel = sum(inp_split_sizes) |
362 | | - all_gather_output = all_gather_comm.allocate( |
363 | | - (all_gather_input_numel * world_size,), dtype=dtype, device=device |
364 | | - ) |
365 | | - all_gather_input, all_gather_output = torch.ops.fsdp.all_gather_copy_in( |
366 | | - all_gather_inputs, |
367 | | - all_gather_output, |
368 | | - inp_split_sizes, |
369 | | - all_gather_input_numel, |
370 | | - rank, |
371 | | - ) |
372 | | - del param_all_gather_inputs |
| 337 | + param_all_gather_inputs = _get_param_all_gather_inputs(fsdp_params) |
| 338 | + ( |
| 339 | + param_all_gather_input_dtypes, |
| 340 | + param_all_gather_input_numels, |
| 341 | + dtype, |
| 342 | + ) = _get_all_gather_input_metadatas(param_all_gather_inputs) |
| 343 | + if dtype == torch.uint8: |
| 344 | + all_gather_inputs = [ |
| 345 | + t.view(torch.uint8) for ts in param_all_gather_inputs for t in ts |
| 346 | + ] |
| 347 | + else: |
| 348 | + all_gather_inputs = [*chain.from_iterable(param_all_gather_inputs)] |
| 349 | + inp_split_sizes = [t.numel() for t in all_gather_inputs] |
| 350 | + all_gather_input_numel = sum(inp_split_sizes) |
| 351 | + all_gather_output = all_gather_comm.allocate( |
| 352 | + (all_gather_input_numel * world_size,), dtype=dtype, device=device |
| 353 | + ) |
| 354 | + all_gather_input, all_gather_output = torch.ops.fsdp.all_gather_copy_in( |
| 355 | + all_gather_inputs, |
| 356 | + all_gather_output, |
| 357 | + inp_split_sizes, |
| 358 | + all_gather_input_numel, |
| 359 | + rank, |
| 360 | + ) |
| 361 | + del param_all_gather_inputs |
373 | 362 | all_gather_stream.wait_stream(all_gather_copy_in_stream) |
374 | 363 | with device_handle.stream(all_gather_stream): |
375 | | - with dist.record_comm(_label_with_suffix("FSDP::all_gather", label_suffix)): |
376 | | - all_gather_work = all_gather_comm( |
377 | | - output_tensor=all_gather_output, |
378 | | - input_tensor=all_gather_input, |
379 | | - group=group, |
380 | | - async_op=async_op, |
381 | | - ) |
| 364 | + all_gather_work = all_gather_comm( |
| 365 | + output_tensor=all_gather_output, |
| 366 | + input_tensor=all_gather_input, |
| 367 | + group=group, |
| 368 | + async_op=async_op, |
| 369 | + ) |
382 | 370 | all_gather_event = all_gather_stream.record_event() |
383 | 371 | return AllGatherResult( |
384 | 372 | all_gather_output, |
@@ -549,7 +537,6 @@ def foreach_reduce( |
549 | 537 | partial_reduce_output: torch.Tensor | None, # only used for HSDP |
550 | 538 | all_reduce_hook: Callable[[torch.Tensor], None] | None, |
551 | 539 | force_sum_reduction_for_comms: bool = False, |
552 | | - label_suffix: str = "", |
553 | 540 | ) -> tuple[ |
554 | 541 | torch.Tensor, |
555 | 542 | torch.Event, |
@@ -630,15 +617,12 @@ def foreach_reduce( |
630 | 617 | ) |
631 | 618 | _div_if_needed(reduce_scatter_input, predivide_factor) |
632 | 619 | if world_size > 1: |
633 | | - with dist.record_comm( |
634 | | - _label_with_suffix("FSDP::reduce_scatter", label_suffix) |
635 | | - ): |
636 | | - reduce_scatter_comm( |
637 | | - output_tensor=reduce_output, |
638 | | - input_tensor=reduce_scatter_input, |
639 | | - group=reduce_scatter_group, |
640 | | - op=reduce_scatter_op, |
641 | | - ) |
| 620 | + reduce_scatter_comm( |
| 621 | + output_tensor=reduce_output, |
| 622 | + input_tensor=reduce_scatter_input, |
| 623 | + group=reduce_scatter_group, |
| 624 | + op=reduce_scatter_op, |
| 625 | + ) |
642 | 626 | else: |
643 | 627 | # For single GPU, just copy the input to output (no actual reduce-scatter needed), and |
644 | 628 | # account for a possible gradient_divide_factor. |
@@ -671,14 +655,19 @@ def foreach_reduce( |
671 | 655 | else: |
672 | 656 | all_reduce_stream.wait_stream(current_stream) |
673 | 657 | with device_handle.stream(all_reduce_stream): |
674 | | - with dist.record_comm( |
675 | | - _label_with_suffix("FSDP::all_reduce", label_suffix) |
676 | | - ): |
677 | | - dist.all_reduce( |
678 | | - reduce_output, |
679 | | - group=all_reduce_group, |
680 | | - op=all_reduce_op, |
681 | | - ) |
| 658 | + dist.all_reduce( |
| 659 | + reduce_output, |
| 660 | + group=all_reduce_group, |
| 661 | + op=all_reduce_op, |
| 662 | + ) |
| 663 | + # Keep refs to the reduce-dtype AR buffer + completion |
| 664 | + # event so FSDPParamGroup._all_reduce_state can hold them |
| 665 | + # across layers. This keeps the buffer off the caching |
| 666 | + # allocator's free list; otherwise the next layer's |
| 667 | + # reduce-scatter can reuse the same physical block while |
| 668 | + # this layer's AR is still in flight, causing cross-layer |
| 669 | + # gradient aliasing under slow AR. See PR #140044, |
| 670 | + # regression test PR #180900. |
682 | 671 | all_reduce_input = reduce_output |
683 | 672 | all_reduce_event = all_reduce_stream.record_event() |
684 | 673 | # -- END: ops in reduce_scatter stream |
|
0 commit comments