Skip to content

Commit f1c2a0b

Browse files
committed
perf: add contiguous vectorized masked scatter kernel
1 parent e70c1fc commit f1c2a0b

2 files changed

Lines changed: 68 additions & 7 deletions

File tree

‎mlx/backend/cuda/device/scatter.cuh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,4 +114,48 @@ __global__ void masked_scatter_fused(
114114
out[index] = dst_val;
115115
}
116116

117+
template <typename T, typename IdxT, int N_READS>
118+
__global__ void masked_scatter_fused_vec_contiguous(
119+
const T* dst,
120+
const bool* mask,
121+
const int32_t* scatter_offsets,
122+
const T* src,
123+
T* out,
124+
IdxT size,
125+
IdxT src_batch_size,
126+
IdxT mask_batch_size,
127+
const __grid_constant__ Shape,
128+
const __grid_constant__ Strides,
129+
int32_t,
130+
const __grid_constant__ Shape,
131+
const __grid_constant__ Strides,
132+
int32_t) {
133+
IdxT vec_index = cg::this_grid().thread_rank();
134+
IdxT base = vec_index * N_READS;
135+
if (base >= size) {
136+
return;
137+
}
138+
139+
auto out_vec = load_vector<N_READS>(dst, vec_index, size, static_cast<T>(0));
140+
auto mask_vec = load_vector<N_READS>(mask, vec_index, size, false);
141+
auto offset_vec = load_vector<N_READS>(scatter_offsets, vec_index, size, 0);
142+
143+
#pragma unroll
144+
for (int i = 0; i < N_READS; ++i) {
145+
IdxT index = base + i;
146+
if (index >= size) {
147+
break;
148+
}
149+
if (mask_vec[i]) {
150+
IdxT src_index = static_cast<IdxT>(offset_vec[i]);
151+
if (src_index < src_batch_size) {
152+
IdxT batch_idx = index / mask_batch_size;
153+
out_vec[i] = src[batch_idx * src_batch_size + src_index];
154+
}
155+
}
156+
}
157+
158+
store_vector<N_READS>(out, vec_index, out_vec, size);
159+
}
160+
117161
} // namespace mlx::core::cu

‎mlx/backend/cuda/indexing.cpp‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,14 @@ void MaskedScatter::eval_gpu(const std::vector<array>& inputs, array& out) {
491491
}
492492
}
493493
}
494+
for (int use_large = 0; use_large <= 1; ++use_large) {
495+
kernel_names.push_back(
496+
fmt::format(
497+
"mlx::core::cu::masked_scatter_fused_vec_contiguous<{}, {}, {}>",
498+
dtype_to_cuda_type(out.dtype()),
499+
use_large ? "int64_t" : "int32_t",
500+
16));
501+
}
494502
return std::make_tuple(false, jit_source_scatter, std::move(kernel_names));
495503
});
496504

@@ -522,14 +530,23 @@ void MaskedScatter::eval_gpu(const std::vector<array>& inputs, array& out) {
522530
encoder.set_input_array(src);
523531
encoder.set_output_array(out);
524532

525-
std::string kernel_name = fmt::format(
526-
"mlx::core::cu::masked_scatter_fused<{}, {}, {}, {}>",
527-
dtype_to_cuda_type(out.dtype()),
528-
src.flags().row_contiguous ? "true" : "false",
529-
dst.flags().row_contiguous ? "true" : "false",
530-
large ? "int64_t" : "int32_t");
533+
bool vectorized = src.flags().row_contiguous && dst.flags().row_contiguous;
534+
std::string kernel_name = vectorized
535+
? fmt::format(
536+
"mlx::core::cu::masked_scatter_fused_vec_contiguous<{}, {}, {}>",
537+
dtype_to_cuda_type(out.dtype()),
538+
large ? "int64_t" : "int32_t",
539+
16)
540+
: fmt::format(
541+
"mlx::core::cu::masked_scatter_fused<{}, {}, {}, {}>",
542+
dtype_to_cuda_type(out.dtype()),
543+
src.flags().row_contiguous ? "true" : "false",
544+
dst.flags().row_contiguous ? "true" : "false",
545+
large ? "int64_t" : "int32_t");
531546
auto kernel = mod.get_kernel(kernel_name);
532-
auto [num_blocks, block_dims] = get_launch_args(mask_flat, large);
547+
auto [num_blocks, block_dims] = vectorized
548+
? get_launch_args(mask_flat, large, 16, 256)
549+
: get_launch_args(mask_flat, large);
533550
encoder.add_kernel_node_raw(
534551
kernel, num_blocks, block_dims, {}, 0, args.args());
535552
}

0 commit comments

Comments
 (0)