@@ -491,6 +491,14 @@ void MaskedScatter::eval_gpu(const std::vector<array>& inputs, array& out) {
491491 }
492492 }
493493 }
494+ for (int use_large = 0 ; use_large <= 1 ; ++use_large) {
495+ kernel_names.push_back (
496+ fmt::format (
497+ " mlx::core::cu::masked_scatter_fused_vec_contiguous<{}, {}, {}>" ,
498+ dtype_to_cuda_type (out.dtype ()),
499+ use_large ? " int64_t" : " int32_t" ,
500+ 16 ));
501+ }
494502 return std::make_tuple (false , jit_source_scatter, std::move (kernel_names));
495503 });
496504
@@ -522,14 +530,23 @@ void MaskedScatter::eval_gpu(const std::vector<array>& inputs, array& out) {
522530 encoder.set_input_array (src);
523531 encoder.set_output_array (out);
524532
525- std::string kernel_name = fmt::format (
526- " mlx::core::cu::masked_scatter_fused<{}, {}, {}, {}>" ,
527- dtype_to_cuda_type (out.dtype ()),
528- src.flags ().row_contiguous ? " true" : " false" ,
529- dst.flags ().row_contiguous ? " true" : " false" ,
530- large ? " int64_t" : " int32_t" );
533+ bool vectorized = src.flags ().row_contiguous && dst.flags ().row_contiguous ;
534+ std::string kernel_name = vectorized
535+ ? fmt::format (
536+ " mlx::core::cu::masked_scatter_fused_vec_contiguous<{}, {}, {}>" ,
537+ dtype_to_cuda_type (out.dtype ()),
538+ large ? " int64_t" : " int32_t" ,
539+ 16 )
540+ : fmt::format (
541+ " mlx::core::cu::masked_scatter_fused<{}, {}, {}, {}>" ,
542+ dtype_to_cuda_type (out.dtype ()),
543+ src.flags ().row_contiguous ? " true" : " false" ,
544+ dst.flags ().row_contiguous ? " true" : " false" ,
545+ large ? " int64_t" : " int32_t" );
531546 auto kernel = mod.get_kernel (kernel_name);
532- auto [num_blocks, block_dims] = get_launch_args (mask_flat, large);
547+ auto [num_blocks, block_dims] = vectorized
548+ ? get_launch_args (mask_flat, large, 16 , 256 )
549+ : get_launch_args (mask_flat, large);
533550 encoder.add_kernel_node_raw (
534551 kernel, num_blocks, block_dims, {}, 0 , args.args ());
535552}
0 commit comments