Skip to content

Commit 7e9bcd2

Browse files
notfilippoLiaCastaneda
authored andcommitted
feat(memory-tracking): implement arrow_buffer::MemoryPool for MemoryPool
Closes apache#18926
1 parent 49776a6 commit 7e9bcd2

4 files changed

Lines changed: 113 additions & 0 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/execution/Cargo.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,14 @@ default = ["sql"]
4646
parquet_encryption = [
4747
"parquet/encryption",
4848
]
49+
arrow_buffer_pool = [
50+
"arrow-buffer/pool"
51+
]
4952
sql = []
5053

5154
[dependencies]
5255
arrow = { workspace = true }
56+
arrow-buffer = { workspace = true }
5357
async-trait = { workspace = true }
5458
chrono = { workspace = true }
5559
dashmap = { workspace = true }
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
//! Adapter for integrating DataFusion's [`MemoryPool`] with Arrow's memory tracking APIs.
2+
3+
use crate::memory_pool::{MemoryConsumer, MemoryLimit, MemoryPool, MemoryReservation};
4+
use std::fmt::Debug;
5+
use std::sync::Arc;
6+
7+
/// An adapter that implements Arrow's [`arrow_buffer::MemoryPool`] trait
8+
/// by wrapping a DataFusion [`MemoryPool`].
9+
///
10+
/// This allows DataFusion's memory management system to be used with Arrow's
11+
/// memory allocation APIs. Each reservation made through this pool will be
12+
/// tracked using the provided [`MemoryConsumer`], enabling DataFusion to
13+
/// monitor and limit memory usage across Arrow operations.
14+
///
15+
/// This is useful when you want Arrow operations (such as array builders
16+
/// or compute kernels) to participate in DataFusion's memory management
17+
/// and respect the same memory limits as DataFusion operators.
18+
#[derive(Debug)]
19+
pub struct ArrowMemoryPool {
20+
inner: Arc<dyn MemoryPool>,
21+
consumer: MemoryConsumer,
22+
}
23+
24+
impl ArrowMemoryPool {
25+
/// Creates a new [`ArrowMemoryPool`] that wraps the given DataFusion [`MemoryPool`]
26+
/// and tracks allocations under the specified [`MemoryConsumer`].
27+
pub fn new(inner: Arc<dyn MemoryPool>, consumer: MemoryConsumer) -> Self {
28+
Self { inner, consumer }
29+
}
30+
}
31+
32+
impl arrow_buffer::MemoryReservation for MemoryReservation {
33+
fn size(&self) -> usize {
34+
MemoryReservation::size(self)
35+
}
36+
37+
fn resize(&mut self, new_size: usize) {
38+
MemoryReservation::resize(self, new_size)
39+
}
40+
}
41+
42+
impl arrow_buffer::MemoryPool for ArrowMemoryPool {
43+
fn reserve(&self, size: usize) -> Box<dyn arrow_buffer::MemoryReservation> {
44+
let consumer = self.consumer.clone_with_new_id();
45+
let mut reservation = consumer.register(&self.inner);
46+
reservation.grow(size);
47+
48+
Box::new(reservation)
49+
}
50+
51+
fn available(&self) -> isize {
52+
// The pool may be overfilled, so this method might return a negative value.
53+
(self.capacity() as i128 - self.used() as i128)
54+
.try_into()
55+
.unwrap_or(isize::MIN)
56+
}
57+
58+
fn used(&self) -> usize {
59+
self.inner.reserved()
60+
}
61+
62+
fn capacity(&self) -> usize {
63+
match self.inner.memory_limit() {
64+
MemoryLimit::Infinite | MemoryLimit::Unknown => usize::MAX,
65+
MemoryLimit::Finite(capacity) => capacity,
66+
}
67+
}
68+
}
69+
70+
#[cfg(test)]
71+
mod tests {
72+
use super::*;
73+
use crate::memory_pool::UnboundedMemoryPool;
74+
use arrow::array::{Array, Int32Array};
75+
use arrow_buffer::MemoryPool;
76+
77+
#[test]
78+
pub fn can_claim_array() {
79+
let pool = Arc::new(UnboundedMemoryPool::default());
80+
81+
let consumer = MemoryConsumer::new("arrow");
82+
let arrow_pool = ArrowMemoryPool::new(pool, consumer);
83+
84+
// Until https://github.com/apache/arrow-rs/pull/8918 lands, we need to iterate all the
85+
// buffer in the array. Change once the PR is released.
86+
fn claim_array(array: &dyn Array, pool: &dyn MemoryPool) {
87+
for buffer in array.to_data().buffers() {
88+
buffer.claim(pool);
89+
}
90+
}
91+
92+
let array = Int32Array::from(vec![1, 2, 3, 4, 5]);
93+
claim_array(&array, &arrow_pool);
94+
95+
assert_eq!(arrow_pool.used(), array.get_buffer_memory_size());
96+
97+
let slice = array.slice(0, 2);
98+
99+
// This should be a no-op
100+
claim_array(&slice, &arrow_pool);
101+
102+
assert_eq!(arrow_pool.used(), array.get_buffer_memory_size());
103+
}
104+
}

‎datafusion/execution/src/memory_pool/mod.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ use std::hash::{Hash, Hasher};
2323
use std::{cmp::Ordering, sync::Arc, sync::atomic};
2424

2525
mod pool;
26+
27+
#[cfg(feature = "arrow_buffer_pool")]
28+
pub mod arrow;
29+
2630
pub mod proxy {
2731
pub use datafusion_common::utils::proxy::{HashTableAllocExt, VecAllocExt};
2832
}

0 commit comments

Comments
 (0)