@@ -296,8 +296,17 @@ class NCCLGroup : public GroupImpl {
296296 }
297297
298298 void all_gather (const array& input, array& output, Stream stream) override {
299- throw std::runtime_error (
300- " [nccl] All gather not supported in NCCL backend." );
299+ detail::dispatch_dtype (input, [&](auto type_tag, ncclDataType_t dt) {
300+ using T = typename decltype (type_tag)::type;
301+ auto & encoder = cu::get_command_encoder (stream);
302+ CHECK_NCCL (ncclAllGather (
303+ input.data <T>(),
304+ output.data <T>(),
305+ input.size (),
306+ dt,
307+ comm_,
308+ encoder.stream ()));
309+ });
301310 }
302311
303312 void send (const array& input, int dst, Stream stream) override {
@@ -309,11 +318,24 @@ class NCCLGroup : public GroupImpl {
309318 }
310319
311320 void all_max (const array& input, array& output, Stream stream) override {
312- throw std::runtime_error (" [nccl] All max not supported in NCCL backend." );
321+ detail::dispatch_dtype (input, [&](auto type_tag, ncclDataType_t dt) {
322+ using T = typename decltype (type_tag)::type;
323+ all_reduce_impl<T>(input, output, stream, dt, ncclMax);
324+ });
313325 }
314326
315327 void all_min (const array& input, array& output, Stream stream) override {
316- throw std::runtime_error (" [nccl] All min not supported in NCCL backend." );
328+ detail::dispatch_dtype (input, [&](auto type_tag, ncclDataType_t dt) {
329+ using T = typename decltype (type_tag)::type;
330+ all_reduce_impl<T>(input, output, stream, dt, ncclMin);
331+ });
332+ }
333+
334+ void sum_scatter (const array& input, array& output, Stream stream) override {
335+ detail::dispatch_dtype (input, [&](auto type_tag, ncclDataType_t dt) {
336+ using T = typename decltype (type_tag)::type;
337+ reduce_scatter_impl<T>(input, output, stream, dt, ncclSum);
338+ });
317339 }
318340
319341 template <typename T>
@@ -335,6 +357,25 @@ class NCCLGroup : public GroupImpl {
335357 encoder.stream ()));
336358 }
337359
360+ template <typename T>
361+ void reduce_scatter_impl (
362+ const array& input,
363+ array& output,
364+ Stream stream,
365+ ncclDataType_t dt,
366+ ncclRedOp_t op) {
367+ auto & encoder = cu::get_command_encoder (stream);
368+
369+ CHECK_NCCL (ncclReduceScatter (
370+ input.data <T>(),
371+ output.data <T>(),
372+ output.size (),
373+ dt,
374+ op,
375+ comm_,
376+ encoder.stream ()));
377+ }
378+
338379 int rank_, size_;
339380 std::string initMethod_;
340381 ncclUniqueId uniqueId_;
0 commit comments