|
21 | 21 | expectedFailureMPS, |
22 | 22 | instantiate_device_type_tests, |
23 | 23 | onlyCPU, |
24 | | - onlyCUDA, |
25 | 24 | onlyNativeDeviceTypes, |
26 | 25 | onlyOn, |
27 | 26 | skipXLA, |
28 | 27 | skipXPUIf, |
29 | | - tol, |
30 | | - toleranceOverride, |
31 | 28 | ) |
32 | 29 | from torch.testing._internal.common_dtype import ( |
33 | 30 | all_mps_types_and, |
@@ -2045,142 +2042,6 @@ def test_index_add_deterministic(self, device: torch.device) -> None: |
2045 | 2042 | y_nd = torch.index_add(x, dim, index, src, alpha=alpha) |
2046 | 2043 | self.assertEqual(y_nd, y0, atol=1e-3, rtol=1e-5) |
2047 | 2044 |
|
2048 | | - @serialTest() |
2049 | | - @onlyCUDA |
2050 | | - @toleranceOverride( |
2051 | | - { |
2052 | | - torch.float32: tol(atol=1e-5, rtol=1e-3), |
2053 | | - torch.float64: tol(atol=1e-5, rtol=1e-3), |
2054 | | - torch.half: tol(atol=5e-2, rtol=5e-2), |
2055 | | - torch.bfloat16: tol(atol=0.5, rtol=0.5), |
2056 | | - } |
2057 | | - ) |
2058 | | - @dtypes(torch.float32, torch.float64, torch.half, torch.bfloat16) |
2059 | | - def test_index_add_fast_path(self, device, dtype): |
2060 | | - # Coverage for the index_add_ TMA fast path: one eligible case + five |
2061 | | - # fallback predicates per shape, asserted against a CPU reference. |
2062 | | - # Shapes keep n/m <= 1 so atomicAdd-order noise on bf16/half stays |
2063 | | - # within tolerance; (4096, 1024, 1024) crosses the TMA chunk_elems |
2064 | | - # boundary (D > one chunk). |
2065 | | - def check(out, dim, idx, src, alpha=1.0): |
2066 | | - expected = ( |
2067 | | - out.cpu().clone().index_add_(dim, idx.cpu(), src.cpu(), alpha=alpha) |
2068 | | - ) |
2069 | | - out.index_add_(dim, idx, src, alpha=alpha) |
2070 | | - self.assertEqual(out.cpu(), expected) |
2071 | | - |
2072 | | - for m, n, D in [(1024, 512, 128), (4096, 3072, 128), (4096, 1024, 1024)]: |
2073 | | - torch.cuda.empty_cache() |
2074 | | - for idx_dtype in (torch.int32, torch.int64): |
2075 | | - src = make_tensor((n, D), device=device, dtype=dtype) |
2076 | | - idx = torch.randint(m, (n,), device=device, dtype=idx_dtype) |
2077 | | - |
2078 | | - # 1) Eligible -> fast path. |
2079 | | - check(torch.zeros(m, D, device=device, dtype=dtype), 0, idx, src) |
2080 | | - # 2) alpha != 1 -> fallback. |
2081 | | - check( |
2082 | | - torch.zeros(m, D, device=device, dtype=dtype), |
2083 | | - 0, |
2084 | | - idx, |
2085 | | - src, |
2086 | | - alpha=2.5, |
2087 | | - ) |
2088 | | - # 3) Discontiguous src -> fallback. |
2089 | | - src_strided = torch.empty(n, 2 * D, device=device, dtype=dtype)[ |
2090 | | - :, ::2 |
2091 | | - ].copy_(src) |
2092 | | - check( |
2093 | | - torch.zeros(m, D, device=device, dtype=dtype), 0, idx, src_strided |
2094 | | - ) |
2095 | | - # 4) Misaligned self (one-element pointer offset) -> fallback. |
2096 | | - self_mis = ( |
2097 | | - torch.empty(m * D + 1, device=device, dtype=dtype)[1:] |
2098 | | - .view(m, D) |
2099 | | - .zero_() |
2100 | | - ) |
2101 | | - check(self_mis, 0, idx, src) |
2102 | | - # 5) dim != 0 -> fallback. |
2103 | | - check( |
2104 | | - torch.zeros(D, m, device=device, dtype=dtype), |
2105 | | - 1, |
2106 | | - idx, |
2107 | | - make_tensor((D, n), device=device, dtype=dtype), |
2108 | | - ) |
2109 | | - # 6) Sliced inner dim (not is_contiguous) -> fallback. |
2110 | | - sl = slice(64, 192) |
2111 | | - check( |
2112 | | - torch.zeros(m, 256, device=device, dtype=dtype)[:, sl], |
2113 | | - 0, |
2114 | | - idx, |
2115 | | - make_tensor((n, 256), device=device, dtype=dtype)[:, sl], |
2116 | | - ) |
2117 | | - |
2118 | | - # 7) Empty index is a no-op. |
2119 | | - out = torch.randn(8, 128, device=device, dtype=dtype) |
2120 | | - expected = out.clone() |
2121 | | - out.index_add_( |
2122 | | - 0, |
2123 | | - torch.empty(0, device=device, dtype=torch.int64), |
2124 | | - torch.empty(0, 128, device=device, dtype=dtype), |
2125 | | - ) |
2126 | | - self.assertEqual(out, expected) |
2127 | | - |
2128 | | - @serialTest() |
2129 | | - @onlyCUDA |
2130 | | - @toleranceOverride( |
2131 | | - { |
2132 | | - # Tolerances follow test_index_add_fast_path: this shape does |
2133 | | - # ~n/m atomic adds per row (~670 here with m=13), and bf16's |
2134 | | - # 7-bit mantissa accumulates noise quickly under non- |
2135 | | - # deterministic atomicAdd ordering. fp32 stays tight. |
2136 | | - torch.float32: tol(atol=1e-4, rtol=1e-3), |
2137 | | - torch.bfloat16: tol(atol=20.0, rtol=0.5), |
2138 | | - } |
2139 | | - ) |
2140 | | - @dtypes(torch.float32, torch.bfloat16) |
2141 | | - def test_index_add_smem_stage_alignment_regression(self, device, dtype): |
2142 | | - # Regression for SEV S664741: the original D104669063 was reverted |
2143 | | - # when this delegation surfaced a latent scatter_add TMA smem |
2144 | | - # stage-alignment bug -- chunk_bytes < 128 (or not a multiple of |
2145 | | - # 128) plus multi-iter-per-CTA (M_src > grid_x cap of sm*64) wrote |
2146 | | - # stage 1 of the 2-stage pipeline buffer at a non-128-aligned smem |
2147 | | - # offset, faulting in cp.async.bulk. Fixed in PR #184554 by |
2148 | | - # rounding the stage stride to 128 bytes. This test pins the |
2149 | | - # prod shape (small D + high M_src) at the index_add layer so a |
2150 | | - # future refactor of the delegation re-exposing the same shape |
2151 | | - # class is caught here, not in prod. |
2152 | | - sm = torch.cuda.get_device_properties(0).multi_processor_count |
2153 | | - # D=8 fp32 -> chunk_bytes=32 (< 128). M_src > sm*64 forces every |
2154 | | - # CTA into >= 2 iterations -> stage 1 used. Prod fault was at |
2155 | | - # sm*64=8448 (H100); sm*64 + 256 exposes the regime on any GPU. |
2156 | | - m, n, D = 13, sm * 64 + 256, 8 |
2157 | | - src = make_tensor((n, D), device=device, dtype=dtype) |
2158 | | - idx = torch.randint(m, (n,), device=device, dtype=torch.int64) |
2159 | | - out = torch.zeros(m, D, device=device, dtype=dtype) |
2160 | | - expected = out.cpu().clone().index_add_(0, idx.cpu(), src.cpu()) |
2161 | | - out.index_add_(0, idx, src) |
2162 | | - self.assertEqual(out.cpu(), expected) |
2163 | | - |
2164 | | - @serialTest() |
2165 | | - @onlyCUDA |
2166 | | - @dtypes(torch.complex64, torch.complex128, torch.bool) |
2167 | | - def test_index_add_excluded_dtypes(self, device, dtype): |
2168 | | - # scatter_add_'s CUDA dispatch covers neither complex nor bool, so the |
2169 | | - # fast-path delegation in index_add_cuda_impl excludes these dtypes |
2170 | | - # and lets them fall through to indexFunc{Small,Large}Index. Regression |
2171 | | - # test that an eligible-shape (dim=0, alpha=1, contiguous, aligned) |
2172 | | - # call still produces correct results for these dtypes. |
2173 | | - m, n, D = 1024, 512, 128 |
2174 | | - if dtype == torch.bool: |
2175 | | - src = torch.randint(0, 2, (n, D), device=device, dtype=dtype) |
2176 | | - else: |
2177 | | - src = make_tensor((n, D), device=device, dtype=dtype) |
2178 | | - out = torch.zeros(m, D, device=device, dtype=dtype) |
2179 | | - idx = torch.randint(m, (n,), device=device, dtype=torch.int64) |
2180 | | - expected = out.cpu().clone().index_add_(0, idx.cpu(), src.cpu()) |
2181 | | - out.index_add_(0, idx, src) |
2182 | | - self.assertEqual(out.cpu(), expected) |
2183 | | - |
2184 | 2045 | @onlyNativeDeviceTypes |
2185 | 2046 | @skipXPUIf(True, "https://github.com/intel/torch-xpu-ops/issues/1973") |
2186 | 2047 | def test_index_put_non_accumulate_deterministic(self, device) -> None: |
|
0 commit comments