Skip to content

Commit aaac2bf

Browse files
[release/2.13] Revert "[Reland] Port D104346887/PR 182675 for index_add fast path (#184824)" (#188196)
Revert "[Reland] Port D104346887/PR 182675 for index_add fast path (#184824) (#184824)" This reverts commit 9e051b3. Reverted #184824 on behalf of https://github.com/meta-codesync due to Diff reverted internally ([comment](#184824 (comment))) (cherry picked from commit eaa0ca8) Co-authored-by: PyTorch MergeBot <[email protected]>
1 parent 9330813 commit aaac2bf

2 files changed

Lines changed: 0 additions & 168 deletions

File tree

‎aten/src/ATen/native/cuda/Indexing.cu‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1179,35 +1179,6 @@ void index_add_cuda_impl(const Tensor& self, int64_t dim, const Tensor& index, c
11791179

11801180
const int mpc = at::cuda::getCurrentDeviceProperties()->multiProcessorCount;
11811181

1182-
#if !defined(USE_ROCM) && defined(CUDA_VERSION) && CUDA_VERSION >= 12080
1183-
// Fast path: index_add_(0, idx, src) with alpha == 1 is equivalent to
1184-
// self.scatter_add_(0, idx.view({n, 1, ...}).expand_as(src), src). Delegate
1185-
// so scatter_add's own TMA/vectorized eligibility check + dispatch is the
1186-
// single source of truth (see PR #182675). Pattern from
1187-
// pytorch/pytorch#180430.
1188-
// Gated on CUDA >= 12.8: pre-12.8 builds compile out the TMA branch in
1189-
// scatter_add and fall back to its vectorized atomicAdd path, which
1190-
// regresses skewed/high-contention workloads vs indexFunc{Small,Large}Index
1191-
// (warp-per-entry scheduling concentrates atomic contention on hot rows).
1192-
// Older builds therefore stay on the existing indexFunc dispatch.
1193-
// index_add supports {complex64, complex128, ComplexHalf, Bool} that
1194-
// scatter_add does not, so exclude those and let them use indexFunc.
1195-
// The dtype check is ordered FIRST so short-circuit evaluation skips
1196-
// alpha.equal(1) for complex `self`, where alpha may itself be a
1197-
// complex Scalar and the equality comparison would be ill-defined.
1198-
const auto stype = self_.scalar_type();
1199-
const bool dtype_supported_by_scatter_add =
1200-
!c10::isComplexType(stype) && stype != at::kBool;
1201-
if (dtype_supported_by_scatter_add && dim == 0 &&
1202-
alpha.equal(1) && numIndex > 0 &&
1203-
index.dim() <= 1 && indContig) {
1204-
std::vector<int64_t> idx_shape(source_.dim(), 1);
1205-
idx_shape[0] = static_cast<int64_t>(numIndex);
1206-
self_.scatter_add_(0, index.view(idx_shape).expand_as(source_), source_);
1207-
return;
1208-
}
1209-
#endif
1210-
12111182
#define SMALL_INDEX(TENSOR_TYPE, INDICES_TYPE, TYPE, SELF_DIM, SOURCE_DIM, IDX_DIM) \
12121183
indexFuncSmallIndex<TENSOR_TYPE, INDICES_TYPE, TYPE, SELF_DIM, SOURCE_DIM, IDX_DIM> \
12131184
<<<smallIndexGrid, smallIndexBlock, 0, stream>>>( \

‎test/test_indexing.py‎

Lines changed: 0 additions & 139 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,10 @@
2121
expectedFailureMPS,
2222
instantiate_device_type_tests,
2323
onlyCPU,
24-
onlyCUDA,
2524
onlyNativeDeviceTypes,
2625
onlyOn,
2726
skipXLA,
2827
skipXPUIf,
29-
tol,
30-
toleranceOverride,
3128
)
3229
from torch.testing._internal.common_dtype import (
3330
all_mps_types_and,
@@ -2045,142 +2042,6 @@ def test_index_add_deterministic(self, device: torch.device) -> None:
20452042
y_nd = torch.index_add(x, dim, index, src, alpha=alpha)
20462043
self.assertEqual(y_nd, y0, atol=1e-3, rtol=1e-5)
20472044

2048-
@serialTest()
2049-
@onlyCUDA
2050-
@toleranceOverride(
2051-
{
2052-
torch.float32: tol(atol=1e-5, rtol=1e-3),
2053-
torch.float64: tol(atol=1e-5, rtol=1e-3),
2054-
torch.half: tol(atol=5e-2, rtol=5e-2),
2055-
torch.bfloat16: tol(atol=0.5, rtol=0.5),
2056-
}
2057-
)
2058-
@dtypes(torch.float32, torch.float64, torch.half, torch.bfloat16)
2059-
def test_index_add_fast_path(self, device, dtype):
2060-
# Coverage for the index_add_ TMA fast path: one eligible case + five
2061-
# fallback predicates per shape, asserted against a CPU reference.
2062-
# Shapes keep n/m <= 1 so atomicAdd-order noise on bf16/half stays
2063-
# within tolerance; (4096, 1024, 1024) crosses the TMA chunk_elems
2064-
# boundary (D > one chunk).
2065-
def check(out, dim, idx, src, alpha=1.0):
2066-
expected = (
2067-
out.cpu().clone().index_add_(dim, idx.cpu(), src.cpu(), alpha=alpha)
2068-
)
2069-
out.index_add_(dim, idx, src, alpha=alpha)
2070-
self.assertEqual(out.cpu(), expected)
2071-
2072-
for m, n, D in [(1024, 512, 128), (4096, 3072, 128), (4096, 1024, 1024)]:
2073-
torch.cuda.empty_cache()
2074-
for idx_dtype in (torch.int32, torch.int64):
2075-
src = make_tensor((n, D), device=device, dtype=dtype)
2076-
idx = torch.randint(m, (n,), device=device, dtype=idx_dtype)
2077-
2078-
# 1) Eligible -> fast path.
2079-
check(torch.zeros(m, D, device=device, dtype=dtype), 0, idx, src)
2080-
# 2) alpha != 1 -> fallback.
2081-
check(
2082-
torch.zeros(m, D, device=device, dtype=dtype),
2083-
0,
2084-
idx,
2085-
src,
2086-
alpha=2.5,
2087-
)
2088-
# 3) Discontiguous src -> fallback.
2089-
src_strided = torch.empty(n, 2 * D, device=device, dtype=dtype)[
2090-
:, ::2
2091-
].copy_(src)
2092-
check(
2093-
torch.zeros(m, D, device=device, dtype=dtype), 0, idx, src_strided
2094-
)
2095-
# 4) Misaligned self (one-element pointer offset) -> fallback.
2096-
self_mis = (
2097-
torch.empty(m * D + 1, device=device, dtype=dtype)[1:]
2098-
.view(m, D)
2099-
.zero_()
2100-
)
2101-
check(self_mis, 0, idx, src)
2102-
# 5) dim != 0 -> fallback.
2103-
check(
2104-
torch.zeros(D, m, device=device, dtype=dtype),
2105-
1,
2106-
idx,
2107-
make_tensor((D, n), device=device, dtype=dtype),
2108-
)
2109-
# 6) Sliced inner dim (not is_contiguous) -> fallback.
2110-
sl = slice(64, 192)
2111-
check(
2112-
torch.zeros(m, 256, device=device, dtype=dtype)[:, sl],
2113-
0,
2114-
idx,
2115-
make_tensor((n, 256), device=device, dtype=dtype)[:, sl],
2116-
)
2117-
2118-
# 7) Empty index is a no-op.
2119-
out = torch.randn(8, 128, device=device, dtype=dtype)
2120-
expected = out.clone()
2121-
out.index_add_(
2122-
0,
2123-
torch.empty(0, device=device, dtype=torch.int64),
2124-
torch.empty(0, 128, device=device, dtype=dtype),
2125-
)
2126-
self.assertEqual(out, expected)
2127-
2128-
@serialTest()
2129-
@onlyCUDA
2130-
@toleranceOverride(
2131-
{
2132-
# Tolerances follow test_index_add_fast_path: this shape does
2133-
# ~n/m atomic adds per row (~670 here with m=13), and bf16's
2134-
# 7-bit mantissa accumulates noise quickly under non-
2135-
# deterministic atomicAdd ordering. fp32 stays tight.
2136-
torch.float32: tol(atol=1e-4, rtol=1e-3),
2137-
torch.bfloat16: tol(atol=20.0, rtol=0.5),
2138-
}
2139-
)
2140-
@dtypes(torch.float32, torch.bfloat16)
2141-
def test_index_add_smem_stage_alignment_regression(self, device, dtype):
2142-
# Regression for SEV S664741: the original D104669063 was reverted
2143-
# when this delegation surfaced a latent scatter_add TMA smem
2144-
# stage-alignment bug -- chunk_bytes < 128 (or not a multiple of
2145-
# 128) plus multi-iter-per-CTA (M_src > grid_x cap of sm*64) wrote
2146-
# stage 1 of the 2-stage pipeline buffer at a non-128-aligned smem
2147-
# offset, faulting in cp.async.bulk. Fixed in PR #184554 by
2148-
# rounding the stage stride to 128 bytes. This test pins the
2149-
# prod shape (small D + high M_src) at the index_add layer so a
2150-
# future refactor of the delegation re-exposing the same shape
2151-
# class is caught here, not in prod.
2152-
sm = torch.cuda.get_device_properties(0).multi_processor_count
2153-
# D=8 fp32 -> chunk_bytes=32 (< 128). M_src > sm*64 forces every
2154-
# CTA into >= 2 iterations -> stage 1 used. Prod fault was at
2155-
# sm*64=8448 (H100); sm*64 + 256 exposes the regime on any GPU.
2156-
m, n, D = 13, sm * 64 + 256, 8
2157-
src = make_tensor((n, D), device=device, dtype=dtype)
2158-
idx = torch.randint(m, (n,), device=device, dtype=torch.int64)
2159-
out = torch.zeros(m, D, device=device, dtype=dtype)
2160-
expected = out.cpu().clone().index_add_(0, idx.cpu(), src.cpu())
2161-
out.index_add_(0, idx, src)
2162-
self.assertEqual(out.cpu(), expected)
2163-
2164-
@serialTest()
2165-
@onlyCUDA
2166-
@dtypes(torch.complex64, torch.complex128, torch.bool)
2167-
def test_index_add_excluded_dtypes(self, device, dtype):
2168-
# scatter_add_'s CUDA dispatch covers neither complex nor bool, so the
2169-
# fast-path delegation in index_add_cuda_impl excludes these dtypes
2170-
# and lets them fall through to indexFunc{Small,Large}Index. Regression
2171-
# test that an eligible-shape (dim=0, alpha=1, contiguous, aligned)
2172-
# call still produces correct results for these dtypes.
2173-
m, n, D = 1024, 512, 128
2174-
if dtype == torch.bool:
2175-
src = torch.randint(0, 2, (n, D), device=device, dtype=dtype)
2176-
else:
2177-
src = make_tensor((n, D), device=device, dtype=dtype)
2178-
out = torch.zeros(m, D, device=device, dtype=dtype)
2179-
idx = torch.randint(m, (n,), device=device, dtype=torch.int64)
2180-
expected = out.cpu().clone().index_add_(0, idx.cpu(), src.cpu())
2181-
out.index_add_(0, idx, src)
2182-
self.assertEqual(out.cpu(), expected)
2183-
21842045
@onlyNativeDeviceTypes
21852046
@skipXPUIf(True, "https://github.com/intel/torch-xpu-ops/issues/1973")
21862047
def test_index_put_non_accumulate_deterministic(self, device) -> None:

0 commit comments

Comments
 (0)