Skip to content

Commit 06f10d0

Browse files
authored
Revert "[FSDP2] add fqn to communication ops" (#182100) (#182157)
Reverts #173838. It messed up async_op=True profiler traces This removes the FSDP2 communication-op FQN annotations added via `dist.record_comm()` and stops threading the collective label suffix through the foreach collective helpers. I The revert preserves later HSDP all-reduce buffer lifetime handling from #180900. Pull Request resolved: #182100 Approved by: https://github.com/anshul-si ghstack dependencies: #181792 (cherry picked from commit 25bc8e0)
1 parent 449e339 commit 06f10d0

3 files changed

Lines changed: 54 additions & 77 deletions

File tree

‎test/distributed/_composable/fsdp/test_fully_shard_training.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2011,7 +2011,6 @@ def foreach_all_gather_with_assert(
20112011
all_gather_stream: torch.Stream,
20122012
device: torch.device,
20132013
all_gather_comm: AllGather,
2014-
module_fqn: str | None = None,
20152014
):
20162015
nonlocal all_gather_streams
20172016
all_gather_streams.add(all_gather_stream)
@@ -2023,7 +2022,6 @@ def foreach_all_gather_with_assert(
20232022
all_gather_stream,
20242023
device,
20252024
all_gather_comm,
2026-
module_fqn,
20272025
)
20282026

20292027
orig_foreach_reduce = foreach_reduce
@@ -2045,7 +2043,6 @@ def foreach_reduce_with_assert(
20452043
partial_reduce_output: torch.Tensor | None, # only used for HSDP
20462044
all_reduce_hook: Callable[[torch.Tensor], None] | None,
20472045
force_sum_reduction_for_comms: bool = False,
2048-
module_fqn: str | None = None,
20492046
):
20502047
nonlocal reduce_scatter_streams
20512048
reduce_scatter_streams.add(reduce_scatter_stream)
@@ -2065,7 +2062,6 @@ def foreach_reduce_with_assert(
20652062
partial_reduce_output,
20662063
all_reduce_hook,
20672064
force_sum_reduction_for_comms,
2068-
module_fqn,
20692065
)
20702066

20712067
with (

‎torch/distributed/fsdp/_fully_shard/_fsdp_collectives.py‎

Lines changed: 50 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,6 @@
2020
from ._fsdp_param import FSDPParam, ShardedState
2121

2222

23-
def _label_with_suffix(label: str, suffix: str) -> str:
24-
if suffix:
25-
return f"{label} {suffix}"
26-
return label
27-
28-
2923
class AllGatherResult(NamedTuple):
3024
all_gather_output: torch.Tensor
3125
all_gather_event: torch.Event | None
@@ -336,49 +330,43 @@ def foreach_all_gather(
336330
all_gather_stream: torch.Stream,
337331
device: torch.device,
338332
all_gather_comm: AllGather,
339-
label_suffix: str = "",
340333
) -> AllGatherResult | None:
341334
world_size, rank = group.size(), group.rank()
342335
device_handle = _get_device_handle(device.type)
343-
344336
with device_handle.stream(all_gather_copy_in_stream):
345-
with torch.profiler.record_function(
346-
_label_with_suffix("FSDP::all_gather_copy_in", label_suffix)
347-
):
348-
param_all_gather_inputs = _get_param_all_gather_inputs(fsdp_params)
349-
(
350-
param_all_gather_input_dtypes,
351-
param_all_gather_input_numels,
352-
dtype,
353-
) = _get_all_gather_input_metadatas(param_all_gather_inputs)
354-
if dtype == torch.uint8:
355-
all_gather_inputs = [
356-
t.view(torch.uint8) for ts in param_all_gather_inputs for t in ts
357-
]
358-
else:
359-
all_gather_inputs = [*chain.from_iterable(param_all_gather_inputs)]
360-
inp_split_sizes = [t.numel() for t in all_gather_inputs]
361-
all_gather_input_numel = sum(inp_split_sizes)
362-
all_gather_output = all_gather_comm.allocate(
363-
(all_gather_input_numel * world_size,), dtype=dtype, device=device
364-
)
365-
all_gather_input, all_gather_output = torch.ops.fsdp.all_gather_copy_in(
366-
all_gather_inputs,
367-
all_gather_output,
368-
inp_split_sizes,
369-
all_gather_input_numel,
370-
rank,
371-
)
372-
del param_all_gather_inputs
337+
param_all_gather_inputs = _get_param_all_gather_inputs(fsdp_params)
338+
(
339+
param_all_gather_input_dtypes,
340+
param_all_gather_input_numels,
341+
dtype,
342+
) = _get_all_gather_input_metadatas(param_all_gather_inputs)
343+
if dtype == torch.uint8:
344+
all_gather_inputs = [
345+
t.view(torch.uint8) for ts in param_all_gather_inputs for t in ts
346+
]
347+
else:
348+
all_gather_inputs = [*chain.from_iterable(param_all_gather_inputs)]
349+
inp_split_sizes = [t.numel() for t in all_gather_inputs]
350+
all_gather_input_numel = sum(inp_split_sizes)
351+
all_gather_output = all_gather_comm.allocate(
352+
(all_gather_input_numel * world_size,), dtype=dtype, device=device
353+
)
354+
all_gather_input, all_gather_output = torch.ops.fsdp.all_gather_copy_in(
355+
all_gather_inputs,
356+
all_gather_output,
357+
inp_split_sizes,
358+
all_gather_input_numel,
359+
rank,
360+
)
361+
del param_all_gather_inputs
373362
all_gather_stream.wait_stream(all_gather_copy_in_stream)
374363
with device_handle.stream(all_gather_stream):
375-
with dist.record_comm(_label_with_suffix("FSDP::all_gather", label_suffix)):
376-
all_gather_work = all_gather_comm(
377-
output_tensor=all_gather_output,
378-
input_tensor=all_gather_input,
379-
group=group,
380-
async_op=async_op,
381-
)
364+
all_gather_work = all_gather_comm(
365+
output_tensor=all_gather_output,
366+
input_tensor=all_gather_input,
367+
group=group,
368+
async_op=async_op,
369+
)
382370
all_gather_event = all_gather_stream.record_event()
383371
return AllGatherResult(
384372
all_gather_output,
@@ -549,7 +537,6 @@ def foreach_reduce(
549537
partial_reduce_output: torch.Tensor | None, # only used for HSDP
550538
all_reduce_hook: Callable[[torch.Tensor], None] | None,
551539
force_sum_reduction_for_comms: bool = False,
552-
label_suffix: str = "",
553540
) -> tuple[
554541
torch.Tensor,
555542
torch.Event,
@@ -630,15 +617,12 @@ def foreach_reduce(
630617
)
631618
_div_if_needed(reduce_scatter_input, predivide_factor)
632619
if world_size > 1:
633-
with dist.record_comm(
634-
_label_with_suffix("FSDP::reduce_scatter", label_suffix)
635-
):
636-
reduce_scatter_comm(
637-
output_tensor=reduce_output,
638-
input_tensor=reduce_scatter_input,
639-
group=reduce_scatter_group,
640-
op=reduce_scatter_op,
641-
)
620+
reduce_scatter_comm(
621+
output_tensor=reduce_output,
622+
input_tensor=reduce_scatter_input,
623+
group=reduce_scatter_group,
624+
op=reduce_scatter_op,
625+
)
642626
else:
643627
# For single GPU, just copy the input to output (no actual reduce-scatter needed), and
644628
# account for a possible gradient_divide_factor.
@@ -671,14 +655,19 @@ def foreach_reduce(
671655
else:
672656
all_reduce_stream.wait_stream(current_stream)
673657
with device_handle.stream(all_reduce_stream):
674-
with dist.record_comm(
675-
_label_with_suffix("FSDP::all_reduce", label_suffix)
676-
):
677-
dist.all_reduce(
678-
reduce_output,
679-
group=all_reduce_group,
680-
op=all_reduce_op,
681-
)
658+
dist.all_reduce(
659+
reduce_output,
660+
group=all_reduce_group,
661+
op=all_reduce_op,
662+
)
663+
# Keep refs to the reduce-dtype AR buffer + completion
664+
# event so FSDPParamGroup._all_reduce_state can hold them
665+
# across layers. This keeps the buffer off the caching
666+
# allocator's free list; otherwise the next layer's
667+
# reduce-scatter can reuse the same physical block while
668+
# this layer's AR is still in flight, causing cross-layer
669+
# gradient aliasing under slow AR. See PR #140044,
670+
# regression test PR #180900.
682671
all_reduce_input = reduce_output
683672
all_reduce_event = all_reduce_stream.record_event()
684673
# -- END: ops in reduce_scatter stream

‎torch/distributed/fsdp/_fully_shard/_fsdp_param_group.py‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -376,7 +376,6 @@ def unshard(self, async_op: bool = False):
376376
*self.comm_ctx.get_all_gather_streams(async_op, self._training_state),
377377
self.device,
378378
self._all_gather_comm,
379-
self._label_suffix,
380379
)
381380

382381
def wait_for_unshard(self):
@@ -613,7 +612,6 @@ def post_backward(self, *unused: Any):
613612
self._partial_reduce_output,
614613
self._all_reduce_hook,
615614
self.force_sum_reduction_for_comms,
616-
self._label_suffix,
617615
)
618616
self.comm_ctx.reduce_scatter_states.append(
619617
ReduceScatterState(reduce_scatter_input, reduce_scatter_event)
@@ -839,17 +837,11 @@ def _all_reduce_process_group(self) -> dist.ProcessGroup:
839837
)
840838
return self.mesh_info.replicate_process_group
841839

842-
@property
843-
def _label_suffix(self) -> str:
844-
suffix = f"({self._module_fqn})" if self._module_fqn else ""
845-
if self._num_param_groups > 1 and isinstance(self.mesh_info, FSDPMeshInfo):
846-
suffix = f"{suffix} [pg={self.mesh_info.shard_mesh_size}]".lstrip()
847-
return suffix
848-
849840
def _with_fqn(self, label: str) -> str:
850-
suffix = self._label_suffix
851-
if suffix:
852-
return f"{label} {suffix}"
841+
if self._module_fqn:
842+
label = f"{label} ({self._module_fqn})"
843+
if self._num_param_groups > 1 and isinstance(self.mesh_info, FSDPMeshInfo):
844+
label = f"{label} [pg={self.mesh_info.shard_mesh_size}]"
853845
return label
854846

855847
def __repr__(self):

0 commit comments

Comments
 (0)