Skip to content

Commit 5dbe999

Browse files
authored
Fix foreign buffer ownership (ml-explore#4634)
1 parent 6553c47 commit 5dbe999

27 files changed

Lines changed: 232 additions & 193 deletions

‎mlx/allocator.h‎

Lines changed: 51 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,20 @@
33
#pragma once
44

55
#include <cstdlib>
6+
#include <functional>
7+
#include <utility>
68

79
#include "mlx/api.h"
810

11+
namespace mlx::core::allocator {
12+
class Data;
13+
} // namespace mlx::core::allocator
14+
15+
namespace mlx::core::cu {
16+
class CommandEncoder;
17+
allocator::Data malloc_async(size_t size, CommandEncoder& encoder);
18+
} // namespace mlx::core::cu
19+
920
namespace mlx::core::allocator {
1021

1122
// Simple wrapper around buffer pointers
@@ -51,12 +62,47 @@ class MLX_API Allocator {
5162

5263
MLX_API Allocator& allocator();
5364

54-
inline Buffer malloc(size_t size) {
55-
return allocator().malloc(size);
56-
}
65+
using Deleter = std::function<void(Buffer)>;
66+
67+
// The allocator owns data from malloc
68+
class Data {
69+
public:
70+
Data(Buffer buffer, Deleter d) : buffer_(buffer), deleter_(std::move(d)) {}
71+
Data(const Data& other) = delete;
72+
Data& operator=(const Data& other) = delete;
73+
Data(Data&& other) noexcept
74+
: buffer_(std::exchange(other.buffer_, Buffer(nullptr))),
75+
deleter_(std::exchange(other.deleter_, nullptr)),
76+
owned_(std::exchange(other.owned_, false)) {}
77+
~Data() {
78+
if (owned_) {
79+
allocator().free(buffer_);
80+
} else if (deleter_) {
81+
deleter_(buffer_);
82+
}
83+
}
84+
85+
Buffer buffer() const& {
86+
return buffer_;
87+
}
88+
Buffer buffer() const&& = delete;
89+
90+
bool is_owned() const {
91+
return owned_;
92+
}
93+
94+
private:
95+
Buffer buffer_;
96+
Deleter deleter_;
97+
bool owned_{false};
98+
99+
explicit Data(Buffer buffer) : buffer_(buffer), owned_(true) {}
100+
friend Data malloc(size_t size);
101+
friend Data cu::malloc_async(size_t size, cu::CommandEncoder& encoder);
102+
};
57103

58-
inline void free(Buffer buffer) {
59-
allocator().free(buffer);
104+
inline Data malloc(size_t size) {
105+
return Data(allocator().malloc(size));
60106
}
61107

62108
// Make a Buffer from a raw pointer of the given size without a copy. If a

‎mlx/array.cpp‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -60,13 +60,11 @@ std::vector<array> array::make_arrays(
6060
array array::unsafe_weak_copy(const array& other) {
6161
auto cpy = array(other.shape(), other.dtype(), nullptr, {});
6262
cpy.set_data(
63-
other.buffer(),
63+
Data(other.buffer(), nullptr),
6464
other.data_size(),
6565
other.strides(),
6666
other.flags(),
67-
0,
68-
[](auto) {});
69-
cpy.array_desc_->offset = other.array_desc_->offset;
67+
other.offset());
7068
return cpy;
7169
}
7270

@@ -104,14 +102,13 @@ array::array(
104102
allocator::release(buffer);
105103
return deleter(ptr);
106104
};
107-
set_data(buffer, std::move(wrapped_deleter));
105+
set_data(Data(buffer, std::move(wrapped_deleter)));
108106
}
109107
}
110108

111-
/* Build an array from a shared buffer */
112-
array::array(allocator::Buffer data, Shape shape, Dtype dtype, Deleter deleter)
109+
array::array(allocator::Data data, Shape shape, Dtype dtype)
113110
: array_desc_(std::make_shared<ArrayDesc>(std::move(shape), dtype)) {
114-
set_data(data, deleter);
111+
set_data(std::move(data));
115112
}
116113

117114
void array::detach() {
@@ -166,8 +163,8 @@ bool array::is_tracer() const {
166163
detail::retain_graph();
167164
}
168165

169-
void array::set_data(allocator::Buffer buffer, Deleter d) {
170-
array_desc_->data = std::make_shared<Data>(buffer, d);
166+
void array::set_data(allocator::Data data) {
167+
array_desc_->data = std::make_shared<Data>(std::move(data));
171168
array_desc_->offset = 0;
172169
array_desc_->data_size = size();
173170
array_desc_->flags.contiguous = true;
@@ -177,13 +174,12 @@ void array::set_data(allocator::Buffer buffer, Deleter d) {
177174
}
178175

179176
void array::set_data(
180-
allocator::Buffer buffer,
177+
allocator::Data data,
181178
size_t data_size,
182179
Strides strides,
183180
Flags flags,
184-
int64_t offset,
185-
Deleter d) {
186-
array_desc_->data = std::make_shared<Data>(buffer, d);
181+
int64_t offset) {
182+
array_desc_->data = std::make_shared<Data>(std::move(data));
187183
array_desc_->offset = offset;
188184
array_desc_->data_size = data_size;
189185
array_desc_->strides = std::move(strides);

‎mlx/array.h‎

Lines changed: 10 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ namespace mlx::core {
1919
// Forward declaration
2020
class Primitive;
2121

22-
using Deleter = std::function<void(allocator::Buffer)>;
2322
using ShapeElem = int32_t;
2423
using Shape = SmallVector<ShapeElem>;
2524
using Strides = SmallVector<int64_t>;
@@ -69,12 +68,8 @@ class MLX_API array {
6968
Dtype dtype,
7069
const std::function<void(void*)>& deleter);
7170

72-
/* Build an array from a buffer */
73-
explicit array(
74-
allocator::Buffer data,
75-
Shape shape,
76-
Dtype dtype,
77-
Deleter deleter = allocator::free);
71+
/* Build an array from data */
72+
explicit array(allocator::Data data, Shape shape, Dtype dtype);
7873

7974
/** Assignment to rvalue does not compile. */
8075
array& operator=(const array& other) && = delete;
@@ -238,22 +233,7 @@ class MLX_API array {
238233
return reinterpret_cast<std::uintptr_t>(array_desc_->primitive.get());
239234
}
240235

241-
struct Data {
242-
allocator::Buffer buffer;
243-
Deleter d;
244-
Data(allocator::Buffer buffer, Deleter d = allocator::free)
245-
: buffer(buffer), d(d) {}
246-
// Not copyable
247-
Data(const Data& d) = delete;
248-
Data& operator=(const Data& d) = delete;
249-
Data(Data&& o) : buffer(o.buffer), d(o.d) {
250-
o.buffer = allocator::Buffer(nullptr);
251-
o.d = [](allocator::Buffer) {};
252-
}
253-
~Data() {
254-
d(buffer);
255-
}
256-
};
236+
using Data = allocator::Data;
257237

258238
struct Flags {
259239
// True iff there are no gaps in the underlying data. Each item
@@ -302,7 +282,8 @@ class MLX_API array {
302282

303283
/** True indicates the arrays buffer is safe to reuse */
304284
bool is_donatable() const {
305-
return array_desc_.use_count() == 1 && (array_desc_->data.use_count() == 1);
285+
return array_desc_.use_count() == 1 &&
286+
(array_desc_->data.use_count() == 1) && array_desc_->data->is_owned();
306287
}
307288

308289
/** The array's siblings. */
@@ -359,11 +340,8 @@ class MLX_API array {
359340
return array_desc_->data_size;
360341
}
361342

362-
allocator::Buffer& buffer() {
363-
return array_desc_->data->buffer;
364-
}
365-
const allocator::Buffer& buffer() const {
366-
return array_desc_->data->buffer;
343+
allocator::Buffer buffer() const {
344+
return array_desc_->data->buffer();
367345
}
368346

369347
size_t buffer_size() const {
@@ -447,24 +425,14 @@ class MLX_API array {
447425
// Check if the array is a tracer array
448426
bool is_tracer() const;
449427

450-
void set_data(allocator::Buffer buffer, Deleter d = allocator::free);
428+
void set_data(allocator::Data data);
451429

452430
void set_data(
453-
allocator::Buffer buffer,
431+
allocator::Data data,
454432
size_t data_size,
455433
Strides strides,
456434
Flags flags,
457-
Deleter d) {
458-
set_data(buffer, data_size, std::move(strides), flags, 0, std::move(d));
459-
}
460-
461-
void set_data(
462-
allocator::Buffer buffer,
463-
size_t data_size,
464-
Strides strides,
465-
Flags flags,
466-
int64_t offset = 0,
467-
Deleter d = allocator::free);
435+
int64_t offset = 0);
468436

469437
void copy_shared_buffer(
470438
const array& other,

‎mlx/backend/common/binary.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ inline void set_binary_op_output_data(
3939
const array& b,
4040
array& out,
4141
BinaryOpType bopt,
42-
std::function<allocator::Buffer(size_t)> mallocfn = allocator::malloc) {
42+
std::function<allocator::Data(size_t)> mallocfn = allocator::malloc) {
4343
bool b_donatable = is_donatable(b, out);
4444
bool a_donatable = is_donatable(a, out);
4545
switch (bopt) {

‎mlx/backend/common/compiled.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ void compiled_allocate_outputs(
110110
std::vector<array>& outputs,
111111
const std::function<bool(size_t)>& is_constant,
112112
bool contiguous,
113-
const std::function<allocator::Buffer(size_t)>&
113+
const std::function<allocator::Data(size_t)>&
114114
mallocfn /* = allocator::malloc */) {
115115
if (contiguous) {
116116
int o = 0;

‎mlx/backend/common/compiled.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,7 @@ void compiled_allocate_outputs(
7878
std::vector<array>& outputs,
7979
const std::function<bool(size_t)>& is_constant,
8080
bool contiguous,
81-
const std::function<allocator::Buffer(size_t)>& mallocfn =
82-
allocator::malloc);
81+
const std::function<allocator::Data(size_t)>& mallocfn = allocator::malloc);
8382

8483
// Collapse contiguous dims ignoring scalars and constants.
8584
std::tuple<bool, bool, Shape, std::vector<Strides>>

‎mlx/backend/common/copy.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ inline bool set_copy_output_data(
2626
const array& in,
2727
array& out,
2828
CopyType ctype,
29-
std::function<allocator::Buffer(size_t)> mallocfn = allocator::malloc) {
29+
std::function<allocator::Data(size_t)> mallocfn = allocator::malloc) {
3030
if (ctype == CopyType::Vector) {
3131
// If the input is donateable, we are doing a vector copy and the types
3232
// have the same size, then the input buffer can hold the output.

‎mlx/backend/common/ternary.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ inline void set_ternary_op_output_data(
4747
const array& c,
4848
array& out,
4949
TernaryOpType topt,
50-
std::function<allocator::Buffer(size_t)> mallocfn = allocator::malloc) {
50+
std::function<allocator::Data(size_t)> mallocfn = allocator::malloc) {
5151
auto maybe_donate = [&out](const array& x) {
5252
if (is_donatable(x, out)) {
5353
out.copy_shared_buffer(x);

‎mlx/backend/common/unary.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ namespace mlx::core {
1010
inline void set_unary_output_data(
1111
const array& in,
1212
array& out,
13-
std::function<allocator::Buffer(size_t)> mallocfn = allocator::malloc) {
13+
std::function<allocator::Data(size_t)> mallocfn = allocator::malloc) {
1414
if (in.flags().contiguous) {
1515
if (is_donatable(in, out)) {
1616
out.copy_shared_buffer(in);

‎mlx/backend/cpu/eig.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,12 +63,12 @@ struct EigWork<
6363
}
6464

6565
void run(T* a, O* values, O* vectors) {
66-
auto eig_tmp = static_cast<T*>(buffers[0].buffer.raw_ptr());
66+
auto eig_tmp = static_cast<T*>(buffers[0].buffer().raw_ptr());
6767
T* vec_tmp = nullptr;
6868
if (vectors) {
69-
vec_tmp = static_cast<T*>(buffers[1].buffer.raw_ptr());
69+
vec_tmp = static_cast<T*>(buffers[1].buffer().raw_ptr());
7070
}
71-
auto work = static_cast<T*>(buffers.back().buffer.raw_ptr());
71+
auto work = static_cast<T*>(buffers.back().buffer().raw_ptr());
7272

7373
int n_vecs_l = vectors ? N : 1;
7474
int n_vecs_r = 1;
@@ -166,9 +166,9 @@ struct EigWork<std::complex<float>> {
166166
&n_vecs_l,
167167
nullptr,
168168
&n_vecs_r,
169-
static_cast<T*>(buffers[0].buffer.raw_ptr()),
169+
static_cast<T*>(buffers[0].buffer().raw_ptr()),
170170
&lwork,
171-
static_cast<R*>(buffers[1].buffer.raw_ptr()),
171+
static_cast<R*>(buffers[1].buffer().raw_ptr()),
172172
&info);
173173
}
174174
};

0 commit comments

Comments
 (0)