@@ -20,6 +20,7 @@ template <typename T, int D, int V = D>
2020 const device bool * mask [[buffer(9)]] ,
2121 const constant uint3& mask_strides [[buffer(10)]] ,
2222 const constant bool & has_mask [[buffer(11)]] ,
23+ const constant uint4& qkv_batch_strides_heads [[buffer(12)]] ,
2324 uint3 tid [[threadgroup_position_in_grid]] ,
2425 uint3 tpg [[threadgroups_per_grid]] ,
2526 uint simd_gid [[simdgroup_index_in_threadgroup]] ,
@@ -30,10 +31,14 @@ template <typename T, int D, int V = D>
3031 constexpr uint v_per_thread = V / BD ;
3132 const uint q_head_stride = qkv_head_strides.x ;
3233 const uint q_seq_stride = qkv_seq_strides.x ;
34+ const uint q_batch_stride = qkv_batch_strides_heads.x ;
3335 const uint k_head_stride = qkv_head_strides.y ;
3436 const uint k_seq_stride = qkv_seq_strides.y ;
37+ const uint k_batch_stride = qkv_batch_strides_heads.y ;
3538 const uint v_head_stride = qkv_head_strides.z ;
3639 const uint v_seq_stride = qkv_seq_strides.z ;
40+ const uint v_batch_stride = qkv_batch_strides_heads.z ;
41+ const uint num_heads = qkv_batch_strides_heads.w ;
3742 const uint mask_kv_seq_stride = mask_strides.x ;
3843 const uint mask_q_seq_stride = mask_strides.y ;
3944 const uint mask_head_stride = mask_strides.z ;
@@ -50,21 +55,22 @@ template <typename T, int D, int V = D>
5055 threadgroup U max_scores[BN ];
5156 threadgroup U sum_exp_scores[BN ];
5257
53- // Adjust positions
54- const int head_idx = tid.x ;
58+ const int bh_idx = tid.x ;
59+ const int batch_idx = bh_idx / int (num_heads);
60+ const int head_idx = bh_idx - batch_idx * int (num_heads);
5561 const int q_seq_idx = tid.y ;
5662 const int kv_head_idx = head_idx / gqa_factor;
5763 const int Q = tpg.y ;
58- const int group_offset = head_idx * Q + q_seq_idx;
64+ const int group_offset = bh_idx * Q + q_seq_idx;
5965 const int o_offset = group_offset;
60- queries += head_idx * q_head_stride + q_seq_idx * q_seq_stride +
61- simd_lid * qk_per_thread;
62- keys += kv_head_idx * k_head_stride + simd_gid * k_seq_stride +
63- simd_lid * qk_per_thread;
64- values += kv_head_idx * v_head_stride + simd_gid * v_seq_stride +
65- simd_lid * v_per_thread;
66+ queries += batch_idx * q_batch_stride + head_idx * q_head_stride +
67+ q_seq_idx * q_seq_stride + simd_lid * qk_per_thread;
68+ keys += batch_idx * k_batch_stride + kv_head_idx * k_head_stride +
69+ simd_gid * k_seq_stride + simd_lid * qk_per_thread;
70+ values += batch_idx * v_batch_stride + kv_head_idx * v_head_stride +
71+ simd_gid * v_seq_stride + simd_lid * v_per_thread;
6672 if (has_mask) {
67- mask += head_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +
73+ mask += bh_idx * mask_head_stride + simd_gid * mask_kv_seq_stride +
6874 q_seq_idx * mask_q_seq_stride;
6975 }
7076
@@ -164,6 +170,7 @@ template <typename T, int D, int V = D>
164170 const device bool * mask [[buffer(11)]] ,
165171 const constant uint3& mask_strides [[buffer(12)]] ,
166172 const constant bool & has_mask [[buffer(13)]] ,
173+ const constant uint4& qkv_batch_strides_heads [[buffer(14)]] ,
167174 uint3 tid [[threadgroup_position_in_grid]] ,
168175 uint3 tpg [[threadgroups_per_grid]] ,
169176 uint simd_gid [[simdgroup_index_in_threadgroup]] ,
@@ -174,10 +181,14 @@ template <typename T, int D, int V = D>
174181 constexpr int v_per_thread = V / BD ;
175182 const int q_head_stride = qkv_head_strides.x ;
176183 const int q_seq_stride = qkv_seq_strides.x ;
184+ const int q_batch_stride = qkv_batch_strides_heads.x ;
177185 const int k_head_stride = qkv_head_strides.y ;
178186 const int k_seq_stride = qkv_seq_strides.y ;
187+ const int k_batch_stride = qkv_batch_strides_heads.y ;
179188 const int v_head_stride = qkv_head_strides.z ;
180189 const int v_seq_stride = qkv_seq_strides.z ;
190+ const int v_batch_stride = qkv_batch_strides_heads.z ;
191+ const int num_heads = qkv_batch_strides_heads.w ;
181192 const int mask_kv_seq_stride = mask_strides.x ;
182193 const int mask_q_seq_stride = mask_strides.y ;
183194 const int mask_head_stride = mask_strides.z ;
@@ -195,22 +206,23 @@ template <typename T, int D, int V = D>
195206 threadgroup U max_scores[BN ];
196207 threadgroup U sum_exp_scores[BN ];
197208
198- // Adjust positions
199209 const int block_idx = tid.z ;
200- const int head_idx = tid.x ;
210+ const int bh_idx = tid.x ;
211+ const int batch_idx = bh_idx / int (num_heads);
212+ const int head_idx = bh_idx - batch_idx * int (num_heads);
201213 const int q_seq_idx = tid.y ;
202- const int o_offset = head_idx * tpg.y + q_seq_idx;
214+ const int o_offset = bh_idx * tpg.y + q_seq_idx;
203215 const int kv_head_idx = head_idx / gqa_factor;
204216
205- queries += head_idx * q_head_stride + q_seq_idx * q_seq_stride +
206- simd_lid * qk_per_thread;
207- keys += kv_head_idx * k_head_stride +
217+ queries += batch_idx * q_batch_stride + head_idx * q_head_stride +
218+ q_seq_idx * q_seq_stride + simd_lid * qk_per_thread;
219+ keys += batch_idx * k_batch_stride + kv_head_idx * k_head_stride +
208220 (block_idx * BN + simd_gid) * k_seq_stride + simd_lid * qk_per_thread;
209- values += kv_head_idx * v_head_stride +
221+ values += batch_idx * v_batch_stride + kv_head_idx * v_head_stride +
210222 (block_idx * BN + simd_gid) * v_seq_stride + simd_lid * v_per_thread;
211223 out += o_offset * blocks * V + block_idx * V + simd_lid * v_per_thread;
212224 if (has_mask) {
213- mask += head_idx * mask_head_stride +
225+ mask += bh_idx * mask_head_stride +
214226 (block_idx * BN + simd_gid) * mask_kv_seq_stride +
215227 q_seq_idx * mask_q_seq_stride;
216228 }
@@ -524,48 +536,50 @@ kernel void attention(
524536 }
525537}
526538
527- #define INSTANTIATE_SDPA_VECTOR (DTYPE, QK_DIM, VALUE_DIM ) \
528- template [[host_name(" sdpa_vector_" #DTYPE " _" #QK_DIM \
529- " _" #VALUE_DIM )]] kernel void \
530- sdpa_vector<DTYPE , QK_DIM , VALUE_DIM >( \
531- const device DTYPE * queries [[buffer(0)]] , \
532- const device DTYPE * keys [[buffer(1)]] , \
533- const device DTYPE * values [[buffer(2)]] , \
534- device DTYPE * out [[buffer(3)]] , \
535- const constant uint& gqa_factor [[buffer(4)]] , \
536- const constant uint& N [[buffer(5)]] , \
537- const constant uint3& qkv_head_strides [[buffer(6)]] , \
538- const constant uint3& qkv_seq_strides [[buffer(7)]] , \
539- const constant float & scale [[buffer(8)]] , \
540- const device bool * mask [[buffer(9)]] , \
541- const constant uint3& mask_strides [[buffer(10)]] , \
542- const constant bool & has_mask [[buffer(11)]] , \
543- uint3 tid [[threadgroup_position_in_grid]] , \
544- uint3 tpg [[threadgroups_per_grid]] , \
545- uint simd_gid [[simdgroup_index_in_threadgroup]] , \
539+ #define INSTANTIATE_SDPA_VECTOR (DTYPE, QK_DIM, VALUE_DIM ) \
540+ template [[host_name(" sdpa_vector_" #DTYPE " _" #QK_DIM \
541+ " _" #VALUE_DIM )]] kernel void \
542+ sdpa_vector<DTYPE , QK_DIM , VALUE_DIM >( \
543+ const device DTYPE * queries [[buffer(0)]] , \
544+ const device DTYPE * keys [[buffer(1)]] , \
545+ const device DTYPE * values [[buffer(2)]] , \
546+ device DTYPE * out [[buffer(3)]] , \
547+ const constant uint& gqa_factor [[buffer(4)]] , \
548+ const constant uint& N [[buffer(5)]] , \
549+ const constant uint3& qkv_head_strides [[buffer(6)]] , \
550+ const constant uint3& qkv_seq_strides [[buffer(7)]] , \
551+ const constant float & scale [[buffer(8)]] , \
552+ const device bool * mask [[buffer(9)]] , \
553+ const constant uint3& mask_strides [[buffer(10)]] , \
554+ const constant bool & has_mask [[buffer(11)]] , \
555+ const constant uint4& qkv_batch_strides_heads [[buffer(12)]] , \
556+ uint3 tid [[threadgroup_position_in_grid]] , \
557+ uint3 tpg [[threadgroups_per_grid]] , \
558+ uint simd_gid [[simdgroup_index_in_threadgroup]] , \
546559 uint simd_lid [[thread_index_in_simdgroup]] );
547560
548- #define INSTANTIATE_SDPA_VECTOR_2PASS_1 (DTYPE, QK_DIM, VALUE_DIM ) \
549- template [[host_name(" sdpa_vector_2pass_1_" #DTYPE " _" #QK_DIM \
550- " _" #VALUE_DIM )]] kernel void \
551- sdpa_vector_2pass_1<DTYPE , QK_DIM , VALUE_DIM >( \
552- const device DTYPE * queries [[buffer(0)]] , \
553- const device DTYPE * keys [[buffer(1)]] , \
554- const device DTYPE * values [[buffer(2)]] , \
555- device DTYPE * out [[buffer(3)]] , \
556- device float * sums [[buffer(4)]] , \
557- device float * maxs [[buffer(5)]] , \
558- const constant uint& gqa_factor [[buffer(6)]] , \
559- const constant uint& N [[buffer(7)]] , \
560- const constant uint3& qkv_head_strides [[buffer(8)]] , \
561- const constant uint3& qkv_seq_strides [[buffer(9)]] , \
562- const constant float & scale [[buffer(10)]] , \
563- const device bool * mask [[buffer(11)]] , \
564- const constant uint3& mask_strides [[buffer(12)]] , \
565- const constant bool & has_mask [[buffer(13)]] , \
566- uint3 tid [[threadgroup_position_in_grid]] , \
567- uint3 tpg [[threadgroups_per_grid]] , \
568- uint simd_gid [[simdgroup_index_in_threadgroup]] , \
561+ #define INSTANTIATE_SDPA_VECTOR_2PASS_1 (DTYPE, QK_DIM, VALUE_DIM ) \
562+ template [[host_name(" sdpa_vector_2pass_1_" #DTYPE " _" #QK_DIM \
563+ " _" #VALUE_DIM )]] kernel void \
564+ sdpa_vector_2pass_1<DTYPE , QK_DIM , VALUE_DIM >( \
565+ const device DTYPE * queries [[buffer(0)]] , \
566+ const device DTYPE * keys [[buffer(1)]] , \
567+ const device DTYPE * values [[buffer(2)]] , \
568+ device DTYPE * out [[buffer(3)]] , \
569+ device float * sums [[buffer(4)]] , \
570+ device float * maxs [[buffer(5)]] , \
571+ const constant uint& gqa_factor [[buffer(6)]] , \
572+ const constant uint& N [[buffer(7)]] , \
573+ const constant uint3& qkv_head_strides [[buffer(8)]] , \
574+ const constant uint3& qkv_seq_strides [[buffer(9)]] , \
575+ const constant float & scale [[buffer(10)]] , \
576+ const device bool * mask [[buffer(11)]] , \
577+ const constant uint3& mask_strides [[buffer(12)]] , \
578+ const constant bool & has_mask [[buffer(13)]] , \
579+ const constant uint4& qkv_batch_strides_heads [[buffer(14)]] , \
580+ uint3 tid [[threadgroup_position_in_grid]] , \
581+ uint3 tpg [[threadgroups_per_grid]] , \
582+ uint simd_gid [[simdgroup_index_in_threadgroup]] , \
569583 uint simd_lid [[thread_index_in_simdgroup]] );
570584
571585#define INSTANTIATE_SDPA_VECTOR_AGGREGATION (DTYPE, VALUE_DIM ) \
0 commit comments