Skip to content

Commit 488a584

Browse files
Track allocator-level memory vs MemoryPool during SLTs to prevent OOMs (apache#22626)
## Which issue does this PR close? Closes apache#22627. ## What & why A test opts in with `SET datafusion.runtime.memory_limit = 'N'`; the wrapping `AccountingMemoryPool` then panics the query if real heap allocations exceed `N * 1.10`. Catches the silent-OOM bug class where DF's voluntary `MemoryPool` undercounts. Off by default. ## Changes **Upstream** (`execution/src/memory_pool/`, `runtime_env.rs`) - New `MemoryPool::try_resize(usize) -> Result<()>`, default `Err(NotImplemented)`. `GreedyMemoryPool` (now `AtomicUsize` pool_size) and `TrackConsumersPool` override. - `RuntimeEnvBuilder::with_memory_limit` tries `try_resize` before wholesale replacement so wrappers survive `SET memory_limit`. **SLT runner** (`datafusion/sqllogictest/`) - `AccountingAllocator`: global allocator, per-file account (`HashMap<ContextId, AtomicIsize>`). Context-id stamped on workers via per-file Tokio runtime + `on_thread_start`. - `AccountingMemoryPool` wraps DF's pool. `try_resize` retunes the account to `N * 1.10`. Renders the operator-set default as `Infinite` so un-opted tests' `SHOW ALL` is unchanged. - One CLI flag, `--default-pool-size-mb` (replaces `--total-memory-mb` + `--datafusion-memory-fraction`). - README + contributor-guide updated. ## Testing - `memory_pool` and SLT-accounting unit tests pass. - Full 475-file SLT corpus passes under `--default-pool-size-mb 16384`. ## User-facing API non-breaking. `--total-memory-mb` and `--datafusion-memory-fraction` replaced by `--default-pool-size-mb`. End-user query behavior unchanged. --- ``` [aggregate.slt] killed by allocator overdraft: account balance = -110983311 bytes, df-pool reserved = 0 MB; sql = "SELECT array_agg(c13 ORDER BY c13) FROM ..." ``` `df-pool reserved = 0 MB` next to a deeply-negative balance is the discrepancy this PR surfaces. --------- Co-authored-by: Claude Opus 4.7 (1M context) <[email protected]>
1 parent 00c35d0 commit 488a584

13 files changed

Lines changed: 932 additions & 18 deletions

File tree

‎.github/workflows/rust.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -471,7 +471,7 @@ jobs:
471471
export RUST_MIN_STACK=20971520
472472
export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data`
473473
cargo test plan_q --package datafusion-benchmarks --profile ci --features=ci -- --test-threads=1
474-
INCLUDE_TPCH=true cargo test --features backtrace,parquet_encryption,substrait --profile ci --package datafusion-sqllogictest --test sqllogictests
474+
INCLUDE_TPCH=true cargo test --features backtrace,parquet_encryption,substrait,memory-accounting --profile ci --package datafusion-sqllogictest --test sqllogictests -- --default-pool-size-mb 16384
475475
- name: Verify Working Directory Clean
476476
run: git diff --exit-code
477477

‎datafusion/execution/src/memory_pool/mod.rs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
//! [`MemoryPool`] for memory management during query execution, [`proxy`] for
1919
//! help with allocation accounting.
2020
21-
use datafusion_common::{Result, internal_datafusion_err};
21+
use datafusion_common::{Result, internal_datafusion_err, not_impl_err};
2222
use std::any::Any;
2323
use std::fmt::Display;
2424
use std::hash::{Hash, Hasher};
@@ -223,6 +223,16 @@ pub trait MemoryPool: Any + Send + Sync + std::fmt::Debug + Display {
223223
fn memory_limit(&self) -> MemoryLimit {
224224
MemoryLimit::Unknown
225225
}
226+
227+
/// Attempt to update this pool's limit in place to `new_limit` bytes.
228+
///
229+
/// Default impl returns `Err`. Callers that route through
230+
/// [`crate::runtime_env::RuntimeEnvBuilder::with_memory_limit`] fall
231+
/// back to replacing the pool wholesale on `Err`, preserving historical
232+
/// behavior for pools that can't be resized in place.
233+
fn try_resize(&self, _new_limit: usize) -> Result<()> {
234+
not_impl_err!("{} does not support resize", self.name())
235+
}
226236
}
227237

228238
impl dyn MemoryPool {

‎datafusion/execution/src/memory_pool/pool.rs‎

Lines changed: 62 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,15 @@ impl Display for UnboundedMemoryPool {
7373
/// This pool works well for queries that do not need to spill or have
7474
/// a single spillable operator. See [`FairSpillPool`] if there are
7575
/// multiple spillable operators that all will spill.
76+
///
77+
/// Supports [`MemoryPool::try_resize`] for in-place limit adjustment, so
78+
/// callers routing through
79+
/// [`RuntimeEnvBuilder::with_memory_limit`](crate::runtime_env::RuntimeEnvBuilder::with_memory_limit)
80+
/// can keep the existing pool (and any wrappers around it) rather than
81+
/// replacing it on every change.
7682
#[derive(Debug)]
7783
pub struct GreedyMemoryPool {
78-
pool_size: usize,
84+
pool_size: AtomicUsize,
7985
used: AtomicUsize,
8086
}
8187

@@ -84,7 +90,7 @@ impl GreedyMemoryPool {
8490
pub fn new(pool_size: usize) -> Self {
8591
debug!("Created new GreedyMemoryPool(pool_size={pool_size})");
8692
Self {
87-
pool_size,
93+
pool_size: AtomicUsize::new(pool_size),
8894
used: AtomicUsize::new(0),
8995
}
9096
}
@@ -104,16 +110,17 @@ impl MemoryPool for GreedyMemoryPool {
104110
}
105111

106112
fn try_grow(&self, reservation: &MemoryReservation, additional: usize) -> Result<()> {
113+
let pool_size = self.pool_size.load(Ordering::Relaxed);
107114
self.used
108115
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |used| {
109116
let new_used = used + additional;
110-
(new_used <= self.pool_size).then_some(new_used)
117+
(new_used <= pool_size).then_some(new_used)
111118
})
112119
.map_err(|used| {
113120
insufficient_capacity_err(
114121
reservation,
115122
additional,
116-
self.pool_size.saturating_sub(used),
123+
pool_size.saturating_sub(used),
117124
self,
118125
)
119126
})?;
@@ -125,19 +132,25 @@ impl MemoryPool for GreedyMemoryPool {
125132
}
126133

127134
fn memory_limit(&self) -> MemoryLimit {
128-
MemoryLimit::Finite(self.pool_size)
135+
MemoryLimit::Finite(self.pool_size.load(Ordering::Relaxed))
136+
}
137+
138+
fn try_resize(&self, new_limit: usize) -> Result<()> {
139+
self.pool_size.store(new_limit, Ordering::Relaxed);
140+
Ok(())
129141
}
130142
}
131143

132144
impl Display for GreedyMemoryPool {
133145
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
134146
let used = self.used.load(Ordering::Relaxed);
147+
let pool_size = self.pool_size.load(Ordering::Relaxed);
135148
write!(
136149
f,
137150
"{}(used: {}, pool_size: {})",
138151
&self.name(),
139152
human_readable_size(used),
140-
human_readable_size(self.pool_size)
153+
human_readable_size(pool_size)
141154
)
142155
}
143156
}
@@ -600,6 +613,10 @@ impl<I: MemoryPool> MemoryPool for TrackConsumersPool<I> {
600613
fn memory_limit(&self) -> MemoryLimit {
601614
self.inner.memory_limit()
602615
}
616+
617+
fn try_resize(&self, new_limit: usize) -> Result<()> {
618+
self.inner.try_resize(new_limit)
619+
}
603620
}
604621

605622
fn provide_top_memory_consumers_to_error_msg(
@@ -1046,4 +1063,43 @@ mod tests {
10461063
"TrackConsumersPool<UnboundedMemoryPool> Display"
10471064
);
10481065
}
1066+
1067+
#[test]
1068+
fn test_greedy_try_resize_in_place() {
1069+
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(100));
1070+
let r = MemoryConsumer::new("r").register(&pool);
1071+
1072+
// Fill the pool, then verify it rejects further growth.
1073+
r.try_grow(100).unwrap();
1074+
r.try_grow(1).unwrap_err();
1075+
1076+
// Resize *up*: previously-rejected growth now succeeds.
1077+
pool.try_resize(200).unwrap();
1078+
assert!(matches!(pool.memory_limit(), MemoryLimit::Finite(200)));
1079+
r.try_grow(50).unwrap();
1080+
assert_eq!(pool.reserved(), 150);
1081+
1082+
// Resize *down* below current usage: subsequent grows fail because
1083+
// reserved (150) already exceeds the new limit (120). Already-issued
1084+
// reservations are not retroactively shrunk.
1085+
pool.try_resize(120).unwrap();
1086+
assert!(matches!(pool.memory_limit(), MemoryLimit::Finite(120)));
1087+
r.try_grow(1).unwrap_err();
1088+
}
1089+
1090+
#[test]
1091+
fn test_track_consumers_try_resize_forwards() {
1092+
let pool: Arc<dyn MemoryPool> = Arc::new(TrackConsumersPool::new(
1093+
GreedyMemoryPool::new(100),
1094+
NonZeroUsize::new(3).unwrap(),
1095+
));
1096+
pool.try_resize(500).unwrap();
1097+
assert!(matches!(pool.memory_limit(), MemoryLimit::Finite(500)));
1098+
}
1099+
1100+
#[test]
1101+
fn test_unbounded_try_resize_returns_err() {
1102+
let pool: Arc<dyn MemoryPool> = Arc::new(UnboundedMemoryPool::default());
1103+
assert!(pool.try_resize(100).is_err());
1104+
}
10491105
}

‎datafusion/execution/src/runtime_env.rs‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -409,12 +409,23 @@ impl RuntimeEnvBuilder {
409409
/// Specify the total memory to use while running the DataFusion
410410
/// plan to `max_memory * memory_fraction` in bytes.
411411
///
412-
/// This defaults to using [`GreedyMemoryPool`] wrapped in the
413-
/// [`TrackConsumersPool`] with a maximum of 5 consumers.
412+
/// If a memory pool is already configured on this builder, this first
413+
/// attempts to resize it in place via [`MemoryPool::try_resize`]. Pools
414+
/// that support resize (e.g. [`GreedyMemoryPool`]) keep their identity
415+
/// — useful for any wrapper that needs to observe limit changes (e.g.
416+
/// to retune external accounting). Pools whose [`MemoryPool::try_resize`]
417+
/// returns `Err` (the default) fall back to wholesale replacement
418+
/// with a [`TrackConsumersPool`]-wrapped [`GreedyMemoryPool`] (top 5
419+
/// consumers), preserving the historical behavior.
414420
///
415421
/// Note DataFusion does not yet respect this limit in all cases.
416422
pub fn with_memory_limit(self, max_memory: usize, memory_fraction: f64) -> Self {
417423
let pool_size = (max_memory as f64 * memory_fraction) as usize;
424+
if let Some(existing) = &self.memory_pool
425+
&& existing.try_resize(pool_size).is_ok()
426+
{
427+
return self;
428+
}
418429
self.with_memory_pool(Arc::new(TrackConsumersPool::new(
419430
GreedyMemoryPool::new(pool_size),
420431
NonZeroUsize::new(5).unwrap(),
@@ -562,3 +573,48 @@ impl RuntimeEnvBuilder {
562573
docs
563574
}
564575
}
576+
577+
#[cfg(test)]
578+
mod tests {
579+
use super::*;
580+
use crate::memory_pool::{GreedyMemoryPool, MemoryLimit, UnboundedMemoryPool};
581+
582+
#[test]
583+
fn with_memory_limit_resizes_in_place_when_pool_supports_it() {
584+
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(100));
585+
let pool_ptr = Arc::as_ptr(&pool);
586+
587+
let env = RuntimeEnvBuilder::new()
588+
.with_memory_pool(Arc::clone(&pool))
589+
.with_memory_limit(500, 1.0)
590+
.build()
591+
.unwrap();
592+
593+
// Same Arc as before — wrapper-or-other-resize-capable pools survive.
594+
assert!(std::ptr::eq(Arc::as_ptr(&env.memory_pool), pool_ptr));
595+
assert!(matches!(
596+
env.memory_pool.memory_limit(),
597+
MemoryLimit::Finite(500)
598+
));
599+
}
600+
601+
#[test]
602+
fn with_memory_limit_falls_back_to_replace_when_resize_unsupported() {
603+
let pool: Arc<dyn MemoryPool> = Arc::new(UnboundedMemoryPool::default());
604+
let pool_ptr = Arc::as_ptr(&pool);
605+
606+
let env = RuntimeEnvBuilder::new()
607+
.with_memory_pool(Arc::clone(&pool))
608+
.with_memory_limit(500, 1.0)
609+
.build()
610+
.unwrap();
611+
612+
// Different Arc — wholesale replacement happened because Unbounded's
613+
// default `try_resize` returns Err.
614+
assert!(!std::ptr::eq(Arc::as_ptr(&env.memory_pool), pool_ptr));
615+
assert!(matches!(
616+
env.memory_pool.memory_limit(),
617+
MemoryLimit::Finite(500)
618+
));
619+
}
620+
}

‎datafusion/sqllogictest/Cargo.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,10 @@ tokio-postgres = { version = "0.7.17", optional = true }
7070
[features]
7171
avro = ["datafusion/avro"]
7272
backtrace = ["datafusion/backtrace"]
73+
# Enable the `AccountingAllocator` `GlobalAlloc` wrapper and its thread-local
74+
# byte counter. The binary still has to declare `#[global_allocator]` for it
75+
# to actually take effect — building with this feature on alone is harmless.
76+
memory-accounting = []
7377
postgres = [
7478
"bytes",
7579
"chrono",

‎datafusion/sqllogictest/README.md‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,35 @@ For focusing on one specific failing test, a file:line filter can be used:
360360
cargo test --test sqllogictests -- --substrait-round-trip binary.slt:23
361361
```
362362

363+
## Running tests: allocator-level memory accounting
364+
365+
Build with `--features memory-accounting` to install a global allocator
366+
wrapper that tracks actual bytes allocated per SLT file and reconciles them
367+
against DataFusion's voluntary `MemoryPool` tracking. The point isn't to
368+
enforce a process-wide budget — it's to catch DataFusion lying about how
369+
much memory it's using. If `MemoryPool` reports 1 MB while the allocator
370+
sees 100 MB go by, _that gap is the bug_.
371+
372+
```shell
373+
cargo test --features memory-accounting --test sqllogictests -- \
374+
--default-pool-size-mb 16384
375+
```
376+
377+
`--default-pool-size-mb` seeds each per-file SLT context's MemoryPool with
378+
the given size in MB and arms the bank as a no-op until a test opts in.
379+
380+
**Opting an individual test in.** Add `SET datafusion.runtime.memory_limit = 'N'` at the top of the `.slt`. The wrapping `AccountingMemoryPool` then
381+
tightens its allocator-level bank to `N * 1.10` (10% headroom). If the test
382+
allocates more than that — including bytes DataFusion's tracker didn't see
383+
— the test panics with an `OverdraftPanic` reporting the actual balance at
384+
panic time. SLTs without a `SET` of `memory_limit` see no change in
385+
behavior; the bank stays loose and `SHOW ALL` continues to render the limit
386+
as `unlimited`.
387+
388+
Inside the runner each file gets its own multi-thread Tokio runtime so
389+
context-ids stamped onto worker threads stay stable for the allocator
390+
hook, and per-file accounts in the bank are isolated from each other.
391+
363392
## `.slt` file format
364393

365394
[`sqllogictest`] was originally written for SQLite to verify the

‎datafusion/sqllogictest/bin/sqllogictests.rs‎

Lines changed: 67 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
#[cfg(feature = "memory-accounting")]
19+
#[global_allocator]
20+
static GLOBAL: datafusion_sqllogictest::AccountingAllocator =
21+
datafusion_sqllogictest::AccountingAllocator::system();
22+
1823
use clap::{ColorChoice, Parser};
1924
use datafusion::common::instant::Instant;
2025
use datafusion::common::utils::get_available_parallelism;
@@ -137,6 +142,19 @@ async fn run_tests() -> Result<()> {
137142

138143
options.warn_on_ignored();
139144

145+
#[cfg(feature = "memory-accounting")]
146+
if let Some(pool_mb) = options.default_pool_size_mb {
147+
let pool_bytes = pool_mb.saturating_mul(1024 * 1024);
148+
// Same value drives the inner MemoryPool's size and the bank's
149+
// default budget. The wrapper renders this value as `unlimited` in
150+
// `SHOW ALL` (sentinel for "no SET has happened"); once a test
151+
// calls `SET datafusion.runtime.memory_limit`, the wrapper retunes
152+
// the bank to that limit + 10% headroom.
153+
datafusion_sqllogictest::set_memory_tracker_limit(pool_bytes);
154+
datafusion_sqllogictest::set_default_budget(pool_bytes as isize);
155+
log::info!("memory-accounting on: default pool size = {pool_mb} MB");
156+
}
157+
140158
// Print parallelism info for debugging CI performance
141159
eprintln!(
142160
"Running with {} test threads (available parallelism: {})",
@@ -209,7 +227,7 @@ async fn run_tests() -> Result<()> {
209227
let currently_running_sql_tracker_clone =
210228
currently_running_sql_tracker.clone();
211229
let file_start = Instant::now();
212-
SpawnedTask::spawn(async move {
230+
let body = async move {
213231
let result = match (
214232
options.postgres_runner,
215233
options.complete,
@@ -282,9 +300,41 @@ async fn run_tests() -> Result<()> {
282300
}
283301

284302
(result, elapsed)
285-
})
286-
.join()
287-
.map(move |result| {
303+
};
304+
// Each file gets its own multi-thread runtime so a stable per-file
305+
// context-id (stamped via `on_thread_start`) is readable from the
306+
// global allocator hook. Bank accounting and SET-driven limit
307+
// retuning will key off this id in later steps. The outer
308+
// orchestration runtime hosts this via `spawn_blocking` so its
309+
// worker threads aren't blocked by the per-file `block_on`.
310+
//
311+
// Worker count matches `SLT_TARGET_PARTITIONS` so a query's
312+
// partition streams each get a worker rather than contending.
313+
#[cfg(feature = "memory-accounting")]
314+
let spawned = {
315+
let context_id = datafusion_sqllogictest::next_context_id();
316+
SpawnedTask::spawn_blocking(move || {
317+
// Stamp this thread too — `block_on` polls `body` here, so
318+
// statements that don't suspend (e.g. `SET memory_limit`,
319+
// pool construction) run on this thread, not a worker.
320+
datafusion_sqllogictest::set_thread_context_id(context_id);
321+
let runtime = tokio::runtime::Builder::new_multi_thread()
322+
.enable_all()
323+
.worker_threads(datafusion_sqllogictest::SLT_TARGET_PARTITIONS)
324+
.thread_name(format!("slt-file-{context_id}"))
325+
.on_thread_start(move || {
326+
datafusion_sqllogictest::set_thread_context_id(context_id);
327+
})
328+
.build()
329+
.expect("build per-file Tokio runtime");
330+
let out = runtime.block_on(body);
331+
runtime.shutdown_background();
332+
out
333+
})
334+
};
335+
#[cfg(not(feature = "memory-accounting"))]
336+
let spawned = SpawnedTask::spawn(body);
337+
spawned.join().map(move |result| {
288338
let elapsed = match &result {
289339
Ok((_, elapsed)) => *elapsed,
290340
Err(_) => Duration::ZERO,
@@ -910,6 +960,19 @@ struct Options {
910960
default_value_t = ColorChoice::Auto
911961
)]
912962
color: ColorChoice,
963+
964+
#[clap(
965+
long,
966+
help = "Default MemoryPool size in MB for each per-file SLT context. \
967+
The pool is wrapped in AccountingMemoryPool, which doubles \
968+
this value as the 'no SET has happened yet' sentinel — until \
969+
an SLT calls `SET datafusion.runtime.memory_limit`, SHOW ALL \
970+
renders the limit as 'unlimited' and the allocator bank \
971+
stays loose. Once a test SETs a limit, the bank tightens to \
972+
that limit + 10% headroom. Requires the memory-accounting \
973+
feature; ignored without it."
974+
)]
975+
default_pool_size_mb: Option<usize>,
913976
}
914977

915978
impl Options {

0 commit comments

Comments
 (0)