Skip to content

Commit 7661cd9

Browse files
pytorchbotIsalia20atalman
authored
[MPS] Fix SDPA wrong output for permuted q/k/v with B > 1 (#181886)
[MPS] Fix SDPA wrong output for permuted q/k/v with B > 1 (#181151) Fixes #181133 Pull Request resolved: #181151 Approved by: https://github.com/malfet (cherry picked from commit 1ca2f17) Co-authored-by: Isalia20 <[email protected]> Co-authored-by: Andrey Talman <[email protected]>
1 parent 9da6087 commit 7661cd9

3 files changed

Lines changed: 107 additions & 72 deletions

File tree

‎aten/src/ATen/native/mps/kernels/Attention.metal‎

Lines changed: 72 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ template <typename T, int D, int V = D>
2020
const device bool* mask [[buffer(9)]],
2121
const constant uint3& mask_strides [[buffer(10)]],
2222
const constant bool& has_mask [[buffer(11)]],
23+
const constant uint4& qkv_batch_strides_heads [[buffer(12)]],
2324
uint3 tid [[threadgroup_position_in_grid]],
2425
uint3 tpg [[threadgroups_per_grid]],
2526
uint simd_gid [[simdgroup_index_in_threadgroup]],
@@ -30,10 +31,14 @@ template <typename T, int D, int V = D>
3031
constexpr uint v_per_thread = V / BD;
3132
const uint q_head_stride = qkv_head_strides.x;
3233
const uint q_seq_stride = qkv_seq_strides.x;
34+
const uint q_batch_stride = qkv_batch_strides_heads.x;
3335
const uint k_head_stride = qkv_head_strides.y;
3436
const uint k_seq_stride = qkv_seq_strides.y;
37+
const uint k_batch_stride = qkv_batch_strides_heads.y;
3538
const uint v_head_stride = qkv_head_strides.z;
3639
const uint v_seq_stride = qkv_seq_strides.z;
40+
const uint v_batch_stride = qkv_batch_strides_heads.z;
41+
const uint num_heads = qkv_batch_strides_heads.w;
3742
const uint mask_kv_seq_stride = mask_strides.x;
3843
const uint mask_q_seq_stride = mask_strides.y;
3944
const uint mask_head_stride = mask_strides.z;
@@ -50,21 +55,22 @@ template <typename T, int D, int V = D>
5055
threadgroup U max_scores[BN];
5156
threadgroup U sum_exp_scores[BN];
5257

53-
// Adjust positions
54-
const int head_idx = tid.x;
58+
const int bh_idx = tid.x;
59+
const int batch_idx = bh_idx / int(num_heads);
60+
const int head_idx = bh_idx - batch_idx * int(num_heads);
5561
const int q_seq_idx = tid.y;
5662
const int kv_head_idx = head_idx / gqa_factor;
5763
const int Q = tpg.y;
58-
const int group_offset = head_idx * Q + q_seq_idx;
64+
const int group_offset = bh_idx * Q + q_seq_idx;
5965
const int o_offset = group_offset;
60-
queries += head_idx * q_head_stride + q_seq_idx * q_seq_stride +
61-
simd_lid * qk_per_thread;
62-
keys += kv_head_idx * k_head_stride + simd_gid * k_seq_stride +
63-
simd_lid * qk_per_thread;
64-
values += kv_head_idx * v_head_stride + simd_gid * v_seq_stride +
65-
simd_lid * v_per_thread;
66+
queries += batch_idx * q_batch_stride + head_idx * q_head_stride +
67+
q_seq_idx * q_seq_stride + simd_lid * qk_per_thread;
68+
keys += batch_idx * k_batch_stride + kv_head_idx * k_head_stride +
69+
simd_gid * k_seq_stride + simd_lid * qk_per_thread;
70+
values += batch_idx * v_batch_stride + kv_head_idx * v_head_stride +
71+
simd_gid * v_seq_stride + simd_lid * v_per_thread;
6672
if (has_mask) {
67-
mask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +
73+
mask += bh_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +
6874
q_seq_idx * mask_q_seq_stride;
6975
}
7076

@@ -164,6 +170,7 @@ template <typename T, int D, int V = D>
164170
const device bool* mask [[buffer(11)]],
165171
const constant uint3& mask_strides [[buffer(12)]],
166172
const constant bool& has_mask [[buffer(13)]],
173+
const constant uint4& qkv_batch_strides_heads [[buffer(14)]],
167174
uint3 tid [[threadgroup_position_in_grid]],
168175
uint3 tpg [[threadgroups_per_grid]],
169176
uint simd_gid [[simdgroup_index_in_threadgroup]],
@@ -174,10 +181,14 @@ template <typename T, int D, int V = D>
174181
constexpr int v_per_thread = V / BD;
175182
const int q_head_stride = qkv_head_strides.x;
176183
const int q_seq_stride = qkv_seq_strides.x;
184+
const int q_batch_stride = qkv_batch_strides_heads.x;
177185
const int k_head_stride = qkv_head_strides.y;
178186
const int k_seq_stride = qkv_seq_strides.y;
187+
const int k_batch_stride = qkv_batch_strides_heads.y;
179188
const int v_head_stride = qkv_head_strides.z;
180189
const int v_seq_stride = qkv_seq_strides.z;
190+
const int v_batch_stride = qkv_batch_strides_heads.z;
191+
const int num_heads = qkv_batch_strides_heads.w;
181192
const int mask_kv_seq_stride = mask_strides.x;
182193
const int mask_q_seq_stride = mask_strides.y;
183194
const int mask_head_stride = mask_strides.z;
@@ -195,22 +206,23 @@ template <typename T, int D, int V = D>
195206
threadgroup U max_scores[BN];
196207
threadgroup U sum_exp_scores[BN];
197208

198-
// Adjust positions
199209
const int block_idx = tid.z;
200-
const int head_idx = tid.x;
210+
const int bh_idx = tid.x;
211+
const int batch_idx = bh_idx / int(num_heads);
212+
const int head_idx = bh_idx - batch_idx * int(num_heads);
201213
const int q_seq_idx = tid.y;
202-
const int o_offset = head_idx * tpg.y + q_seq_idx;
214+
const int o_offset = bh_idx * tpg.y + q_seq_idx;
203215
const int kv_head_idx = head_idx / gqa_factor;
204216

205-
queries += head_idx * q_head_stride + q_seq_idx * q_seq_stride +
206-
simd_lid * qk_per_thread;
207-
keys += kv_head_idx * k_head_stride +
217+
queries += batch_idx * q_batch_stride + head_idx * q_head_stride +
218+
q_seq_idx * q_seq_stride + simd_lid * qk_per_thread;
219+
keys += batch_idx * k_batch_stride + kv_head_idx * k_head_stride +
208220
(block_idx * BN + simd_gid) * k_seq_stride + simd_lid * qk_per_thread;
209-
values += kv_head_idx * v_head_stride +
221+
values += batch_idx * v_batch_stride + kv_head_idx * v_head_stride +
210222
(block_idx * BN + simd_gid) * v_seq_stride + simd_lid * v_per_thread;
211223
out += o_offset * blocks * V + block_idx * V + simd_lid * v_per_thread;
212224
if (has_mask) {
213-
mask += head_idx * mask_head_stride +
225+
mask += bh_idx * mask_head_stride +
214226
(block_idx * BN + simd_gid) * mask_kv_seq_stride +
215227
q_seq_idx * mask_q_seq_stride;
216228
}
@@ -524,48 +536,50 @@ kernel void attention(
524536
}
525537
}
526538

527-
#define INSTANTIATE_SDPA_VECTOR(DTYPE, QK_DIM, VALUE_DIM) \
528-
template [[host_name("sdpa_vector_" #DTYPE "_" #QK_DIM \
529-
"_" #VALUE_DIM)]] kernel void \
530-
sdpa_vector<DTYPE, QK_DIM, VALUE_DIM>( \
531-
const device DTYPE* queries [[buffer(0)]], \
532-
const device DTYPE* keys [[buffer(1)]], \
533-
const device DTYPE* values [[buffer(2)]], \
534-
device DTYPE* out [[buffer(3)]], \
535-
const constant uint& gqa_factor [[buffer(4)]], \
536-
const constant uint& N [[buffer(5)]], \
537-
const constant uint3& qkv_head_strides [[buffer(6)]], \
538-
const constant uint3& qkv_seq_strides [[buffer(7)]], \
539-
const constant float& scale [[buffer(8)]], \
540-
const device bool* mask [[buffer(9)]], \
541-
const constant uint3& mask_strides [[buffer(10)]], \
542-
const constant bool& has_mask [[buffer(11)]], \
543-
uint3 tid [[threadgroup_position_in_grid]], \
544-
uint3 tpg [[threadgroups_per_grid]], \
545-
uint simd_gid [[simdgroup_index_in_threadgroup]], \
539+
#define INSTANTIATE_SDPA_VECTOR(DTYPE, QK_DIM, VALUE_DIM) \
540+
template [[host_name("sdpa_vector_" #DTYPE "_" #QK_DIM \
541+
"_" #VALUE_DIM)]] kernel void \
542+
sdpa_vector<DTYPE, QK_DIM, VALUE_DIM>( \
543+
const device DTYPE* queries [[buffer(0)]], \
544+
const device DTYPE* keys [[buffer(1)]], \
545+
const device DTYPE* values [[buffer(2)]], \
546+
device DTYPE* out [[buffer(3)]], \
547+
const constant uint& gqa_factor [[buffer(4)]], \
548+
const constant uint& N [[buffer(5)]], \
549+
const constant uint3& qkv_head_strides [[buffer(6)]], \
550+
const constant uint3& qkv_seq_strides [[buffer(7)]], \
551+
const constant float& scale [[buffer(8)]], \
552+
const device bool* mask [[buffer(9)]], \
553+
const constant uint3& mask_strides [[buffer(10)]], \
554+
const constant bool& has_mask [[buffer(11)]], \
555+
const constant uint4& qkv_batch_strides_heads [[buffer(12)]], \
556+
uint3 tid [[threadgroup_position_in_grid]], \
557+
uint3 tpg [[threadgroups_per_grid]], \
558+
uint simd_gid [[simdgroup_index_in_threadgroup]], \
546559
uint simd_lid [[thread_index_in_simdgroup]]);
547560

548-
#define INSTANTIATE_SDPA_VECTOR_2PASS_1(DTYPE, QK_DIM, VALUE_DIM) \
549-
template [[host_name("sdpa_vector_2pass_1_" #DTYPE "_" #QK_DIM \
550-
"_" #VALUE_DIM)]] kernel void \
551-
sdpa_vector_2pass_1<DTYPE, QK_DIM, VALUE_DIM>( \
552-
const device DTYPE* queries [[buffer(0)]], \
553-
const device DTYPE* keys [[buffer(1)]], \
554-
const device DTYPE* values [[buffer(2)]], \
555-
device DTYPE* out [[buffer(3)]], \
556-
device float* sums [[buffer(4)]], \
557-
device float* maxs [[buffer(5)]], \
558-
const constant uint& gqa_factor [[buffer(6)]], \
559-
const constant uint& N [[buffer(7)]], \
560-
const constant uint3& qkv_head_strides [[buffer(8)]], \
561-
const constant uint3& qkv_seq_strides [[buffer(9)]], \
562-
const constant float& scale [[buffer(10)]], \
563-
const device bool* mask [[buffer(11)]], \
564-
const constant uint3& mask_strides [[buffer(12)]], \
565-
const constant bool& has_mask [[buffer(13)]], \
566-
uint3 tid [[threadgroup_position_in_grid]], \
567-
uint3 tpg [[threadgroups_per_grid]], \
568-
uint simd_gid [[simdgroup_index_in_threadgroup]], \
561+
#define INSTANTIATE_SDPA_VECTOR_2PASS_1(DTYPE, QK_DIM, VALUE_DIM) \
562+
template [[host_name("sdpa_vector_2pass_1_" #DTYPE "_" #QK_DIM \
563+
"_" #VALUE_DIM)]] kernel void \
564+
sdpa_vector_2pass_1<DTYPE, QK_DIM, VALUE_DIM>( \
565+
const device DTYPE* queries [[buffer(0)]], \
566+
const device DTYPE* keys [[buffer(1)]], \
567+
const device DTYPE* values [[buffer(2)]], \
568+
device DTYPE* out [[buffer(3)]], \
569+
device float* sums [[buffer(4)]], \
570+
device float* maxs [[buffer(5)]], \
571+
const constant uint& gqa_factor [[buffer(6)]], \
572+
const constant uint& N [[buffer(7)]], \
573+
const constant uint3& qkv_head_strides [[buffer(8)]], \
574+
const constant uint3& qkv_seq_strides [[buffer(9)]], \
575+
const constant float& scale [[buffer(10)]], \
576+
const device bool* mask [[buffer(11)]], \
577+
const constant uint3& mask_strides [[buffer(12)]], \
578+
const constant bool& has_mask [[buffer(13)]], \
579+
const constant uint4& qkv_batch_strides_heads [[buffer(14)]], \
580+
uint3 tid [[threadgroup_position_in_grid]], \
581+
uint3 tpg [[threadgroups_per_grid]], \
582+
uint simd_gid [[simdgroup_index_in_threadgroup]], \
569583
uint simd_lid [[thread_index_in_simdgroup]]);
570584

571585
#define INSTANTIATE_SDPA_VECTOR_AGGREGATION(DTYPE, VALUE_DIM) \

‎aten/src/ATen/native/mps/operations/Attention.mm‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -194,10 +194,13 @@
194194
uint maxSeqLength = k_.size(2);
195195
uint N = k_.size(2);
196196
uint B = q_.size(0) * q_.size(1);
197+
uint q_batch_stride = q_.stride(0);
197198
uint q_head_stride = q_.stride(1);
198199
uint q_seq_stride = q_.stride(2);
200+
uint k_batch_stride = k_.stride(0);
199201
uint k_head_stride = k_.stride(1);
200202
uint k_seq_stride = k_.stride(2);
203+
uint v_batch_stride = v_.stride(0);
201204
uint v_head_stride = v_.stride(1);
202205
uint v_seq_stride = v_.stride(2);
203206

@@ -235,7 +238,8 @@
235238
mtl_setArgs<9>(
236239
computeEncoder, mask_.value(), std::array<uint32_t, 3>{kv_seq_stride, q_seq_stride, head_stride});
237240
}
238-
mtl_setArgs<11>(computeEncoder, has_mask);
241+
mtl_setArgs<11>(
242+
computeEncoder, has_mask, std::array<uint32_t, 4>{q_batch_stride, k_batch_stride, v_batch_stride, num_head});
239243
[computeEncoder dispatchThreadgroups:grid_dims threadsPerThreadgroup:group_dims];
240244
}
241245
});
@@ -273,10 +277,13 @@
273277
uint B = batchSize * num_heads;
274278
uint gqa_factor = q_.size(1) / k_.size(1);
275279

280+
uint q_batch_stride = q_.stride(0);
276281
uint q_head_stride = q_.stride(1);
277282
uint q_seq_stride = q_.stride(2);
283+
uint k_batch_stride = k_.stride(0);
278284
uint k_head_stride = k_.stride(1);
279285
uint k_seq_stride = k_.stride(2);
286+
uint v_batch_stride = v_.stride(0);
280287
uint v_head_stride = v_.stride(1);
281288
uint v_seq_stride = v_.stride(2);
282289

@@ -324,7 +331,8 @@
324331
uint head_stride = (nd >= 3 && mask.size(nd - 3) > 1) ? mask.stride(nd - 3) : 0;
325332
mtl_setArgs<11>(computeEncoder, mask, std::array<uint32_t, 3>{kv_seq_stride, q_seq_stride, head_stride});
326333
}
327-
mtl_setArgs<13>(computeEncoder, has_mask);
334+
mtl_setArgs<13>(
335+
computeEncoder, has_mask, std::array<uint32_t, 4>{q_batch_stride, k_batch_stride, v_batch_stride, num_heads});
328336
[computeEncoder dispatchThreadgroups:grid_dims threadsPerThreadgroup:group_dims];
329337
// 2nd pass
330338
[computeEncoder setComputePipelineState:sdpa_vector_pass2PSO];
@@ -515,19 +523,12 @@
515523
return sdpa_general_mps(q_, k_, v_, mask_, dropout_p, is_causal, dropout_mask, scale, query, unsqueezed);
516524
}
517525

518-
// dispatch to the fast SDPA implementation
519-
auto is_contiguous_or_head_seq_transposed = [](const Tensor& t) -> bool {
520-
if (t.is_contiguous())
521-
return true;
522-
auto sizes = t.sizes();
523-
auto strides = t.strides();
524-
return (strides[3] == 1) && (strides[2] == sizes[3] * sizes[1]) && (strides[1] == sizes[3]) &&
525-
(strides[0] == strides[2] * sizes[2]);
526-
};
526+
// Kernels load head-dim elements linearly, so stride(-1) == 1 is the only hard requirement
527+
auto can_use_kernel_strides = [](const Tensor& t) -> bool { return t.stride(-1) == 1; };
527528

528-
Tensor q_contig = is_contiguous_or_head_seq_transposed(q_) ? q_ : q_.contiguous();
529-
Tensor k_contig = k_.contiguous();
530-
Tensor v_contig = v_.contiguous();
529+
Tensor q_contig = can_use_kernel_strides(q_) ? q_ : q_.contiguous();
530+
Tensor k_contig = can_use_kernel_strides(k_) ? k_ : k_.contiguous();
531+
Tensor v_contig = can_use_kernel_strides(v_) ? v_ : v_.contiguous();
531532

532533
// for short sequences, differentiate based on key sequence length
533534
if ((k_.size(2) >= 1024) || (k_.size(1) < q_.size(1) && k_.size(2) >= 4096)) {

‎test/test_mps.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10201,6 +10201,26 @@ def test_fast_vector_attention_2pass(self, dtype: torch.dtype, layout: str, with
1020110201
q, k, v = self.generate_qkv(batch, NH, q_len, s_len, head_dim, layout, dtype)
1020210202
self.run_fast_attention_test(q, k, v, with_mask)
1020310203

10204+
def test_fast_vector_permuted_inputs_regression(self):
10205+
# Regression test for https://github.com/pytorch/pytorch/issues/181133
10206+
torch.manual_seed(0)
10207+
B, S, H, D = 84, 3, 3, 64
10208+
dtype = torch.float32
10209+
q = torch.randn(B, S, H, D, dtype=dtype)
10210+
k = torch.randn(B, S, H, D, dtype=dtype)
10211+
v = torch.randn(B, S, H, D, dtype=dtype)
10212+
10213+
q_mps = q.to("mps").permute(0, 2, 1, 3)
10214+
k_mps = k.to("mps").permute(0, 2, 1, 3)
10215+
v_mps = v.to("mps").permute(0, 2, 1, 3)
10216+
q_cpu = q.permute(0, 2, 1, 3)
10217+
k_cpu = k.permute(0, 2, 1, 3)
10218+
v_cpu = v.permute(0, 2, 1, 3)
10219+
10220+
y_mps = F.scaled_dot_product_attention(q_mps, k_mps, v_mps)
10221+
y_cpu = F.scaled_dot_product_attention(q_cpu, k_cpu, v_cpu)
10222+
self._compare_tensors(y_mps.cpu(), y_cpu)
10223+
1020410224
@unittest.skip("Full attention fast kernel not implemented yet")
1020510225
@parametrize("dtype", [torch.float16, torch.float32])
1020610226
@parametrize("layout", ["contiguous", "mT"])

0 commit comments

Comments
 (0)