Skip to content

Commit df58b41

Browse files
authored
[CUDA] Reduce use of managed memory (#2725)
* Use async cuda malloc managed with cuda 13 * add pool threshold * refactor for regular cuda malloc * load eval gpu for cuda * remove use of cuda pool, use cuda free async * fix * fix * fix * fix * fix + comment
1 parent 2777815 commit df58b41

79 files changed

Lines changed: 795 additions & 515 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/build-cuda/action.yml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,6 @@ runs:
2626
CMAKE_ARGS: -DMLX_BUILD_CUDA=ON -DCMAKE_COMPILE_WARNING_AS_ERROR=ON -DCMAKE_CUDA_COMPILER=${{ inputs.nvcc-location }}
2727
run: pip install -e ".[dev]" -v
2828

29-
- name: Check if build actually worked
30-
shell: bash
31-
run: python -c "import mlx.core"
32-
3329
- name: Run Python tests - CPU
3430
if: inputs.run-tests == 'true'
3531
shell: bash

‎mlx/allocator.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ class Buffer {
1414
void* ptr_;
1515

1616
public:
17-
Buffer(void* ptr) : ptr_(ptr) {};
17+
explicit Buffer(void* ptr) : ptr_(ptr) {};
1818

1919
// Get the raw data pointer from the buffer
2020
void* raw_ptr();

‎mlx/array.cpp‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ array array::unsafe_weak_copy(const array& other) {
6464
other.strides(),
6565
other.flags(),
6666
[](auto) {});
67-
cpy.array_desc_->data_ptr = other.array_desc_->data_ptr;
67+
cpy.array_desc_->offset = other.array_desc_->offset;
6868
return cpy;
6969
}
7070

@@ -141,7 +141,7 @@ bool array::is_tracer() const {
141141

142142
void array::set_data(allocator::Buffer buffer, Deleter d) {
143143
array_desc_->data = std::make_shared<Data>(buffer, d);
144-
array_desc_->data_ptr = buffer.raw_ptr();
144+
array_desc_->offset = 0;
145145
array_desc_->data_size = size();
146146
array_desc_->flags.contiguous = true;
147147
array_desc_->flags.row_contiguous = true;
@@ -156,7 +156,7 @@ void array::set_data(
156156
Flags flags,
157157
Deleter d) {
158158
array_desc_->data = std::make_shared<Data>(buffer, d);
159-
array_desc_->data_ptr = buffer.raw_ptr();
159+
array_desc_->offset = 0;
160160
array_desc_->data_size = data_size;
161161
array_desc_->strides = std::move(strides);
162162
array_desc_->flags = flags;
@@ -172,9 +172,8 @@ void array::copy_shared_buffer(
172172
array_desc_->strides = strides;
173173
array_desc_->flags = flags;
174174
array_desc_->data_size = data_size;
175-
auto char_offset = sizeof(char) * itemsize() * offset;
176-
array_desc_->data_ptr = static_cast<void*>(
177-
static_cast<char*>(other.array_desc_->data_ptr) + char_offset);
175+
array_desc_->offset =
176+
sizeof(char) * itemsize() * offset + other.array_desc_->offset;
178177
}
179178

180179
void array::copy_shared_buffer(const array& other) {

‎mlx/array.h‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -354,15 +354,23 @@ class array {
354354
return array_desc_->data;
355355
}
356356

357-
// Return a raw pointer to the arrays data
357+
// Return a raw pointer to the arrays data. This function may do a copy if
358+
// the underlying buffer is not accessible on the CPU. When accessing the
359+
// data for GPU kernels, be sure to use the correct method / function for the
360+
// given backend to access the GPU pointer.
358361
template <typename T>
359362
T* data() {
360-
return static_cast<T*>(array_desc_->data_ptr);
363+
return reinterpret_cast<T*>(
364+
(static_cast<char*>(buffer().raw_ptr()) + array_desc_->offset));
361365
}
362366

363367
template <typename T>
364368
const T* data() const {
365-
return static_cast<T*>(array_desc_->data_ptr);
369+
return const_cast<array&>(*this).data<T>();
370+
}
371+
372+
int64_t offset() const {
373+
return array_desc_->offset;
366374
}
367375

368376
enum Status {
@@ -466,8 +474,8 @@ class array {
466474
// can share the underlying data buffer.
467475
std::shared_ptr<Data> data;
468476

469-
// Properly offset data pointer
470-
void* data_ptr{nullptr};
477+
// Offset from beginning of data pointer
478+
int64_t offset{0};
471479

472480
// The size in elements of the data buffer the array accesses
473481
size_t data_size;

‎mlx/backend/common/binary.h‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,20 +38,20 @@ inline void set_binary_op_output_data(
3838
const array& a,
3939
const array& b,
4040
array& out,
41-
BinaryOpType bopt) {
41+
BinaryOpType bopt,
42+
std::function<allocator::Buffer(size_t)> mallocfn = allocator::malloc) {
4243
bool b_donatable = is_donatable(b, out);
4344
bool a_donatable = is_donatable(a, out);
4445
switch (bopt) {
4546
case BinaryOpType::ScalarScalar:
46-
out.set_data(
47-
allocator::malloc(out.itemsize()), 1, a.strides(), a.flags());
47+
out.set_data(mallocfn(out.itemsize()), 1, a.strides(), a.flags());
4848
break;
4949
case BinaryOpType::ScalarVector:
5050
if (b_donatable) {
5151
out.copy_shared_buffer(b);
5252
} else {
5353
out.set_data(
54-
allocator::malloc(b.data_size() * out.itemsize()),
54+
mallocfn(b.data_size() * out.itemsize()),
5555
b.data_size(),
5656
b.strides(),
5757
b.flags());
@@ -62,7 +62,7 @@ inline void set_binary_op_output_data(
6262
out.copy_shared_buffer(a);
6363
} else {
6464
out.set_data(
65-
allocator::malloc(a.data_size() * out.itemsize()),
65+
mallocfn(a.data_size() * out.itemsize()),
6666
a.data_size(),
6767
a.strides(),
6868
a.flags());
@@ -75,7 +75,7 @@ inline void set_binary_op_output_data(
7575
out.copy_shared_buffer(b);
7676
} else {
7777
out.set_data(
78-
allocator::malloc(a.data_size() * out.itemsize()),
78+
mallocfn(a.data_size() * out.itemsize()),
7979
a.data_size(),
8080
a.strides(),
8181
a.flags());
@@ -88,7 +88,7 @@ inline void set_binary_op_output_data(
8888
b_donatable && b.flags().row_contiguous && b.size() == out.size()) {
8989
out.copy_shared_buffer(b);
9090
} else {
91-
out.set_data(allocator::malloc(out.nbytes()));
91+
out.set_data(mallocfn(out.nbytes()));
9292
}
9393
break;
9494
}

‎mlx/backend/common/broadcasting.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ namespace mlx::core {
66

77
void broadcast(const array& in, array& out) {
88
if (out.size() == 0) {
9-
out.set_data(nullptr);
9+
out.set_data(allocator::malloc(0));
1010
return;
1111
}
1212
Strides strides(out.ndim(), 0);

‎mlx/backend/common/compiled.cpp‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,9 @@ void compiled_allocate_outputs(
114114
const std::vector<array>& inputs,
115115
std::vector<array>& outputs,
116116
const std::function<bool(size_t)>& is_constant,
117-
bool contiguous) {
117+
bool contiguous,
118+
const std::function<allocator::Buffer(size_t)>&
119+
mallocfn /* = allocator::malloc */) {
118120
if (contiguous) {
119121
int o = 0;
120122
Strides strides;
@@ -140,7 +142,7 @@ void compiled_allocate_outputs(
140142
}
141143
for (; o < outputs.size(); ++o) {
142144
outputs[o].set_data(
143-
allocator::malloc(data_size * outputs[o].itemsize()),
145+
mallocfn(data_size * outputs[o].itemsize()),
144146
data_size,
145147
strides,
146148
flags);
@@ -163,7 +165,7 @@ void compiled_allocate_outputs(
163165
}
164166
}
165167
for (; o < outputs.size(); ++o) {
166-
outputs[o].set_data(allocator::malloc(outputs[o].nbytes()));
168+
outputs[o].set_data(mallocfn(outputs[o].nbytes()));
167169
}
168170
}
169171
}

‎mlx/backend/common/compiled.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,9 @@ void compiled_allocate_outputs(
5858
const std::vector<array>& inputs,
5959
std::vector<array>& outputs,
6060
const std::function<bool(size_t)>& is_constant,
61-
bool contiguous);
61+
bool contiguous,
62+
const std::function<allocator::Buffer(size_t)>& mallocfn =
63+
allocator::malloc);
6264

6365
// Collapse contiguous dims ignoring scalars and constants.
6466
std::tuple<bool, Shape, std::vector<Strides>> compiled_collapse_contiguous_dims(

‎mlx/backend/common/copy.h‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,11 @@ enum class CopyType {
2222
GeneralGeneral
2323
};
2424

25-
inline bool set_copy_output_data(const array& in, array& out, CopyType ctype) {
25+
inline bool set_copy_output_data(
26+
const array& in,
27+
array& out,
28+
CopyType ctype,
29+
std::function<allocator::Buffer(size_t)> mallocfn = allocator::malloc) {
2630
if (ctype == CopyType::Vector) {
2731
// If the input is donateable, we are doing a vector copy and the types
2832
// have the same size, then the input buffer can hold the output.
@@ -31,14 +35,14 @@ inline bool set_copy_output_data(const array& in, array& out, CopyType ctype) {
3135
return true;
3236
} else {
3337
out.set_data(
34-
allocator::malloc(in.data_size() * out.itemsize()),
38+
mallocfn(in.data_size() * out.itemsize()),
3539
in.data_size(),
3640
in.strides(),
3741
in.flags());
3842
return false;
3943
}
4044
} else {
41-
out.set_data(allocator::malloc(out.nbytes()));
45+
out.set_data(mallocfn(out.nbytes()));
4246
return false;
4347
}
4448
}

‎mlx/backend/common/slicing.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ void slice(
4545
const Shape& start_indices,
4646
const Shape& strides) {
4747
if (out.size() == 0) {
48-
out.set_data(nullptr);
48+
out.set_data(allocator::malloc(0));
4949
return;
5050
}
5151

0 commit comments

Comments
 (0)