Skip to content

Commit 2777815

Browse files
nastya236awni
andauthored
Nccl reduce scatter, all gather (#2727)
* Added reduce scatter and all gather for nccl * fix unused import, delete unused file * small fix * deleted useless condition * fixed comments * fix bug in eval_gpu, renamed to sum_scatter, fix docs * final fix docs * remove and * Update mlx/distributed/mpi/mpi.cpp Co-authored-by: Awni Hannun <[email protected]> * fix broken set input output * fixes set output * typo * fix typo * no cpu, no gpu for reduce scatter --------- Co-authored-by: Awni Hannun <[email protected]>
1 parent 761f901 commit 2777815

19 files changed

Lines changed: 352 additions & 312 deletions

‎mlx/backend/cpu/distributed.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,4 +95,9 @@ void Recv::eval_cpu(
9595
distributed::detail::recv(group(), outputs[0], src_, stream());
9696
}
9797

98+
void ReduceScatter::eval_cpu(
99+
const std::vector<array>& inputs,
100+
std::vector<array>& outputs) {
101+
throw std::runtime_error("[ReduceScatter] Not implemented yet.");
102+
}
98103
} // namespace mlx::core::distributed

‎mlx/backend/cuda/distributed.cu‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,4 +53,69 @@ void AllReduce::eval_gpu(
5353
"Only all reduce sum, max, and min are supported.");
5454
}
5555
}
56+
57+
void AllGather::eval_gpu(
58+
const std::vector<array>& inputs,
59+
std::vector<array>& outputs) {
60+
assert(inputs.size() == 1);
61+
assert(outputs.size() == 1);
62+
63+
auto& s = stream();
64+
auto& encoder = cu::get_command_encoder(s);
65+
66+
auto ensure_contiguous = [&s, &encoder](const array& x) {
67+
if (x.flags().row_contiguous) {
68+
return x;
69+
} else {
70+
array x_copy = contiguous_copy_gpu(x, s);
71+
encoder.add_temporary(x_copy);
72+
return x_copy;
73+
}
74+
};
75+
76+
auto input = ensure_contiguous(inputs[0]);
77+
outputs[0].set_data(allocator::malloc(outputs[0].nbytes()));
78+
79+
encoder.set_input_array(input);
80+
encoder.set_output_array(outputs[0]);
81+
82+
auto capture = encoder.capture_context();
83+
distributed::detail::all_gather(group(), input, outputs[0], s);
84+
}
85+
86+
void ReduceScatter::eval_gpu(
87+
const std::vector<array>& inputs,
88+
std::vector<array>& outputs) {
89+
assert(inputs.size() == 1);
90+
assert(outputs.size() == 1);
91+
92+
auto& s = stream();
93+
auto& encoder = cu::get_command_encoder(s);
94+
95+
auto ensure_contiguous = [&s, &encoder](const array& x) {
96+
if (x.flags().row_contiguous) {
97+
return x;
98+
} else {
99+
array x_copy = contiguous_copy_gpu(x, s);
100+
encoder.add_temporary(x_copy);
101+
return x_copy;
102+
}
103+
};
104+
105+
auto input = ensure_contiguous(inputs[0]);
106+
outputs[0].set_data(allocator::malloc(outputs[0].nbytes()));
107+
108+
encoder.set_input_array(input);
109+
encoder.set_output_array(outputs[0]);
110+
111+
auto capture = encoder.capture_context();
112+
113+
switch (reduce_type_) {
114+
case Sum:
115+
distributed::detail::sum_scatter(group(), input, outputs[0], s);
116+
break;
117+
default:
118+
throw std::runtime_error("Only sum scatter is supported. ");
119+
}
120+
}
56121
} // namespace mlx::core::distributed

‎mlx/backend/cuda/primitives.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ NO_GPU_MULTI(Eig)
4040
NO_GPU_MULTI(Eigh)
4141

4242
namespace distributed {
43-
NO_GPU_MULTI(AllGather)
4443
NO_GPU_MULTI(Send)
4544
NO_GPU_MULTI(Recv)
4645
} // namespace distributed

‎mlx/backend/metal/distributed.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,9 @@ void Recv::eval_gpu(const std::vector<array>&, std::vector<array>&) {
3030
throw std::runtime_error("[Recv::eval_gpu] has no GPU implementation.");
3131
}
3232

33+
void ReduceScatter::eval_gpu(const std::vector<array>&, std::vector<array>&) {
34+
throw std::runtime_error(
35+
"[ReduceScatter::eval_gpu] has no GPU implementation.");
36+
}
37+
3338
} // namespace mlx::core::distributed

‎mlx/backend/no_cpu/primitives.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ NO_CPU_MULTI(AllReduce)
138138
NO_CPU_MULTI(AllGather)
139139
NO_CPU_MULTI(Send)
140140
NO_CPU_MULTI(Recv)
141+
NO_CPU_MULTI(ReduceScatter)
141142
} // namespace distributed
142143

143144
} // namespace mlx::core

‎mlx/backend/no_gpu/primitives.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ NO_GPU_MULTI(AllReduce)
164164
NO_GPU_MULTI(AllGather)
165165
NO_GPU_MULTI(Send)
166166
NO_GPU_MULTI(Recv)
167+
NO_GPU_MULTI(ReduceScatter)
167168
} // namespace distributed
168169

169170
} // namespace mlx::core

‎mlx/distributed/distributed.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,14 @@ void recv(Group group, array& out, int src, Stream stream) {
4141
group.raw_group()->recv(out, src, stream);
4242
}
4343

44+
void sum_scatter(
45+
Group group,
46+
const array& input,
47+
array& output,
48+
Stream stream) {
49+
group.raw_group()->sum_scatter(input, output, stream);
50+
}
51+
4452
class EmptyGroup : public GroupImpl {
4553
public:
4654
Stream communication_stream(StreamOrDevice s) override {
@@ -85,6 +93,10 @@ class EmptyGroup : public GroupImpl {
8593
throw std::runtime_error(
8694
"Communication not implemented in an empty distributed group.");
8795
}
96+
void sum_scatter(const array&, array&, Stream) override {
97+
throw std::runtime_error(
98+
"Communication not implemented in an empty distributed group.");
99+
}
88100
};
89101

90102
} // namespace detail

‎mlx/distributed/distributed_impl.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ class GroupImpl {
2828
virtual void recv(array& out, int src, Stream stream) = 0;
2929
virtual void all_max(const array& input, array& output, Stream stream) = 0;
3030
virtual void all_min(const array& input, array& output, Stream stream) = 0;
31+
virtual void
32+
sum_scatter(const array& input, array& output, Stream stream) = 0;
3133
};
3234

3335
/* Define the MLX stream that the communication should happen in. */
@@ -51,4 +53,7 @@ void all_max(Group group, const array& input, array& output, Stream stream);
5153
/** Min reduction */
5254
void all_min(Group group, const array& input, array& output, Stream stream);
5355

56+
/** Reduce scatter with average operation */
57+
void sum_scatter(Group group, const array& input, array& output, Stream stream);
58+
5459
} // namespace mlx::core::distributed::detail

‎mlx/distributed/mpi/mpi.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -465,6 +465,10 @@ class MPIGroup : public GroupImpl {
465465
});
466466
}
467467

468+
void sum_scatter(const array& input, array& output, Stream stream) override {
469+
throw std::runtime_error("[mpi] sum_scatter not yet implemented.");
470+
}
471+
468472
private:
469473
MPI_Comm comm_;
470474
bool global_;

‎mlx/distributed/nccl/nccl.cpp‎

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -296,8 +296,17 @@ class NCCLGroup : public GroupImpl {
296296
}
297297

298298
void all_gather(const array& input, array& output, Stream stream) override {
299-
throw std::runtime_error(
300-
"[nccl] All gather not supported in NCCL backend.");
299+
detail::dispatch_dtype(input, [&](auto type_tag, ncclDataType_t dt) {
300+
using T = typename decltype(type_tag)::type;
301+
auto& encoder = cu::get_command_encoder(stream);
302+
CHECK_NCCL(ncclAllGather(
303+
input.data<T>(),
304+
output.data<T>(),
305+
input.size(),
306+
dt,
307+
comm_,
308+
encoder.stream()));
309+
});
301310
}
302311

303312
void send(const array& input, int dst, Stream stream) override {
@@ -309,11 +318,24 @@ class NCCLGroup : public GroupImpl {
309318
}
310319

311320
void all_max(const array& input, array& output, Stream stream) override {
312-
throw std::runtime_error("[nccl] All max not supported in NCCL backend.");
321+
detail::dispatch_dtype(input, [&](auto type_tag, ncclDataType_t dt) {
322+
using T = typename decltype(type_tag)::type;
323+
all_reduce_impl<T>(input, output, stream, dt, ncclMax);
324+
});
313325
}
314326

315327
void all_min(const array& input, array& output, Stream stream) override {
316-
throw std::runtime_error("[nccl] All min not supported in NCCL backend.");
328+
detail::dispatch_dtype(input, [&](auto type_tag, ncclDataType_t dt) {
329+
using T = typename decltype(type_tag)::type;
330+
all_reduce_impl<T>(input, output, stream, dt, ncclMin);
331+
});
332+
}
333+
334+
void sum_scatter(const array& input, array& output, Stream stream) override {
335+
detail::dispatch_dtype(input, [&](auto type_tag, ncclDataType_t dt) {
336+
using T = typename decltype(type_tag)::type;
337+
reduce_scatter_impl<T>(input, output, stream, dt, ncclSum);
338+
});
317339
}
318340

319341
template <typename T>
@@ -335,6 +357,25 @@ class NCCLGroup : public GroupImpl {
335357
encoder.stream()));
336358
}
337359

360+
template <typename T>
361+
void reduce_scatter_impl(
362+
const array& input,
363+
array& output,
364+
Stream stream,
365+
ncclDataType_t dt,
366+
ncclRedOp_t op) {
367+
auto& encoder = cu::get_command_encoder(stream);
368+
369+
CHECK_NCCL(ncclReduceScatter(
370+
input.data<T>(),
371+
output.data<T>(),
372+
output.size(),
373+
dt,
374+
op,
375+
comm_,
376+
encoder.stream()));
377+
}
378+
338379
int rank_, size_;
339380
std::string initMethod_;
340381
ncclUniqueId uniqueId_;

0 commit comments

Comments
 (0)