Skip to content

Commit 666f862

Browse files
Revert custom allocator auditing of MemoryPool tracking in SLTs (#22860)
## Which issue does this PR close? No issue, just responding to discussion in #22723 ## Rationale for this change Agreed this was an overly strict approach. ## What changes are included in this PR? Reverting previous work ## Are these changes tested? By definition ## Are there any user-facing changes? Contributors won't be held to higher standards in SLTs than in the contributor guide.
1 parent a851dfd commit 666f862

13 files changed

Lines changed: 18 additions & 949 deletions

File tree

‎.github/workflows/rust.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -471,7 +471,7 @@ jobs:
471471
export RUST_MIN_STACK=20971520
472472
export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data`
473473
cargo test plan_q --package datafusion-benchmarks --profile ci --features=ci -- --test-threads=1
474-
INCLUDE_TPCH=true cargo test --features backtrace,parquet_encryption,substrait,memory-accounting --profile ci --package datafusion-sqllogictest --test sqllogictests -- --default-pool-size-mb 16384
474+
INCLUDE_TPCH=true cargo test --features backtrace,parquet_encryption,substrait --profile ci --package datafusion-sqllogictest --test sqllogictests
475475
- name: Verify Working Directory Clean
476476
run: git diff --exit-code
477477

‎datafusion/execution/src/memory_pool/mod.rs‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
//! [`MemoryPool`] for memory management during query execution, [`proxy`] for
1919
//! help with allocation accounting.
2020
21-
use datafusion_common::{Result, internal_datafusion_err, not_impl_err};
21+
use datafusion_common::{Result, internal_datafusion_err};
2222
use std::any::Any;
2323
use std::fmt::Display;
2424
use std::hash::{Hash, Hasher};
@@ -223,16 +223,6 @@ pub trait MemoryPool: Any + Send + Sync + std::fmt::Debug + Display {
223223
fn memory_limit(&self) -> MemoryLimit {
224224
MemoryLimit::Unknown
225225
}
226-
227-
/// Attempt to update this pool's limit in place to `new_limit` bytes.
228-
///
229-
/// Default impl returns `Err`. Callers that route through
230-
/// [`crate::runtime_env::RuntimeEnvBuilder::with_memory_limit`] fall
231-
/// back to replacing the pool wholesale on `Err`, preserving historical
232-
/// behavior for pools that can't be resized in place.
233-
fn try_resize(&self, _new_limit: usize) -> Result<()> {
234-
not_impl_err!("{} does not support resize", self.name())
235-
}
236226
}
237227

238228
impl dyn MemoryPool {

‎datafusion/execution/src/memory_pool/pool.rs‎

Lines changed: 6 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -73,15 +73,9 @@ impl Display for UnboundedMemoryPool {
7373
/// This pool works well for queries that do not need to spill or have
7474
/// a single spillable operator. See [`FairSpillPool`] if there are
7575
/// multiple spillable operators that all will spill.
76-
///
77-
/// Supports [`MemoryPool::try_resize`] for in-place limit adjustment, so
78-
/// callers routing through
79-
/// [`RuntimeEnvBuilder::with_memory_limit`](crate::runtime_env::RuntimeEnvBuilder::with_memory_limit)
80-
/// can keep the existing pool (and any wrappers around it) rather than
81-
/// replacing it on every change.
8276
#[derive(Debug)]
8377
pub struct GreedyMemoryPool {
84-
pool_size: AtomicUsize,
78+
pool_size: usize,
8579
used: AtomicUsize,
8680
}
8781

@@ -90,7 +84,7 @@ impl GreedyMemoryPool {
9084
pub fn new(pool_size: usize) -> Self {
9185
debug!("Created new GreedyMemoryPool(pool_size={pool_size})");
9286
Self {
93-
pool_size: AtomicUsize::new(pool_size),
87+
pool_size,
9488
used: AtomicUsize::new(0),
9589
}
9690
}
@@ -110,17 +104,16 @@ impl MemoryPool for GreedyMemoryPool {
110104
}
111105

112106
fn try_grow(&self, reservation: &MemoryReservation, additional: usize) -> Result<()> {
113-
let pool_size = self.pool_size.load(Ordering::Relaxed);
114107
self.used
115108
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |used| {
116109
let new_used = used + additional;
117-
(new_used <= pool_size).then_some(new_used)
110+
(new_used <= self.pool_size).then_some(new_used)
118111
})
119112
.map_err(|used| {
120113
insufficient_capacity_err(
121114
reservation,
122115
additional,
123-
pool_size.saturating_sub(used),
116+
self.pool_size.saturating_sub(used),
124117
self,
125118
)
126119
})?;
@@ -132,25 +125,19 @@ impl MemoryPool for GreedyMemoryPool {
132125
}
133126

134127
fn memory_limit(&self) -> MemoryLimit {
135-
MemoryLimit::Finite(self.pool_size.load(Ordering::Relaxed))
136-
}
137-
138-
fn try_resize(&self, new_limit: usize) -> Result<()> {
139-
self.pool_size.store(new_limit, Ordering::Relaxed);
140-
Ok(())
128+
MemoryLimit::Finite(self.pool_size)
141129
}
142130
}
143131

144132
impl Display for GreedyMemoryPool {
145133
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
146134
let used = self.used.load(Ordering::Relaxed);
147-
let pool_size = self.pool_size.load(Ordering::Relaxed);
148135
write!(
149136
f,
150137
"{}(used: {}, pool_size: {})",
151138
&self.name(),
152139
human_readable_size(used),
153-
human_readable_size(pool_size)
140+
human_readable_size(self.pool_size)
154141
)
155142
}
156143
}
@@ -613,10 +600,6 @@ impl<I: MemoryPool> MemoryPool for TrackConsumersPool<I> {
613600
fn memory_limit(&self) -> MemoryLimit {
614601
self.inner.memory_limit()
615602
}
616-
617-
fn try_resize(&self, new_limit: usize) -> Result<()> {
618-
self.inner.try_resize(new_limit)
619-
}
620603
}
621604

622605
fn provide_top_memory_consumers_to_error_msg(
@@ -1063,43 +1046,4 @@ mod tests {
10631046
"TrackConsumersPool<UnboundedMemoryPool> Display"
10641047
);
10651048
}
1066-
1067-
#[test]
1068-
fn test_greedy_try_resize_in_place() {
1069-
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(100));
1070-
let r = MemoryConsumer::new("r").register(&pool);
1071-
1072-
// Fill the pool, then verify it rejects further growth.
1073-
r.try_grow(100).unwrap();
1074-
r.try_grow(1).unwrap_err();
1075-
1076-
// Resize *up*: previously-rejected growth now succeeds.
1077-
pool.try_resize(200).unwrap();
1078-
assert!(matches!(pool.memory_limit(), MemoryLimit::Finite(200)));
1079-
r.try_grow(50).unwrap();
1080-
assert_eq!(pool.reserved(), 150);
1081-
1082-
// Resize *down* below current usage: subsequent grows fail because
1083-
// reserved (150) already exceeds the new limit (120). Already-issued
1084-
// reservations are not retroactively shrunk.
1085-
pool.try_resize(120).unwrap();
1086-
assert!(matches!(pool.memory_limit(), MemoryLimit::Finite(120)));
1087-
r.try_grow(1).unwrap_err();
1088-
}
1089-
1090-
#[test]
1091-
fn test_track_consumers_try_resize_forwards() {
1092-
let pool: Arc<dyn MemoryPool> = Arc::new(TrackConsumersPool::new(
1093-
GreedyMemoryPool::new(100),
1094-
NonZeroUsize::new(3).unwrap(),
1095-
));
1096-
pool.try_resize(500).unwrap();
1097-
assert!(matches!(pool.memory_limit(), MemoryLimit::Finite(500)));
1098-
}
1099-
1100-
#[test]
1101-
fn test_unbounded_try_resize_returns_err() {
1102-
let pool: Arc<dyn MemoryPool> = Arc::new(UnboundedMemoryPool::default());
1103-
assert!(pool.try_resize(100).is_err());
1104-
}
11051049
}

‎datafusion/execution/src/runtime_env.rs‎

Lines changed: 2 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -409,23 +409,12 @@ impl RuntimeEnvBuilder {
409409
/// Specify the total memory to use while running the DataFusion
410410
/// plan to `max_memory * memory_fraction` in bytes.
411411
///
412-
/// If a memory pool is already configured on this builder, this first
413-
/// attempts to resize it in place via [`MemoryPool::try_resize`]. Pools
414-
/// that support resize (e.g. [`GreedyMemoryPool`]) keep their identity
415-
/// — useful for any wrapper that needs to observe limit changes (e.g.
416-
/// to retune external accounting). Pools whose [`MemoryPool::try_resize`]
417-
/// returns `Err` (the default) fall back to wholesale replacement
418-
/// with a [`TrackConsumersPool`]-wrapped [`GreedyMemoryPool`] (top 5
419-
/// consumers), preserving the historical behavior.
412+
/// This defaults to using [`GreedyMemoryPool`] wrapped in the
413+
/// [`TrackConsumersPool`] with a maximum of 5 consumers.
420414
///
421415
/// Note DataFusion does not yet respect this limit in all cases.
422416
pub fn with_memory_limit(self, max_memory: usize, memory_fraction: f64) -> Self {
423417
let pool_size = (max_memory as f64 * memory_fraction) as usize;
424-
if let Some(existing) = &self.memory_pool
425-
&& existing.try_resize(pool_size).is_ok()
426-
{
427-
return self;
428-
}
429418
self.with_memory_pool(Arc::new(TrackConsumersPool::new(
430419
GreedyMemoryPool::new(pool_size),
431420
NonZeroUsize::new(5).unwrap(),
@@ -573,48 +562,3 @@ impl RuntimeEnvBuilder {
573562
docs
574563
}
575564
}
576-
577-
#[cfg(test)]
578-
mod tests {
579-
use super::*;
580-
use crate::memory_pool::{GreedyMemoryPool, MemoryLimit, UnboundedMemoryPool};
581-
582-
#[test]
583-
fn with_memory_limit_resizes_in_place_when_pool_supports_it() {
584-
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(100));
585-
let pool_ptr = Arc::as_ptr(&pool);
586-
587-
let env = RuntimeEnvBuilder::new()
588-
.with_memory_pool(Arc::clone(&pool))
589-
.with_memory_limit(500, 1.0)
590-
.build()
591-
.unwrap();
592-
593-
// Same Arc as before — wrapper-or-other-resize-capable pools survive.
594-
assert!(std::ptr::eq(Arc::as_ptr(&env.memory_pool), pool_ptr));
595-
assert!(matches!(
596-
env.memory_pool.memory_limit(),
597-
MemoryLimit::Finite(500)
598-
));
599-
}
600-
601-
#[test]
602-
fn with_memory_limit_falls_back_to_replace_when_resize_unsupported() {
603-
let pool: Arc<dyn MemoryPool> = Arc::new(UnboundedMemoryPool::default());
604-
let pool_ptr = Arc::as_ptr(&pool);
605-
606-
let env = RuntimeEnvBuilder::new()
607-
.with_memory_pool(Arc::clone(&pool))
608-
.with_memory_limit(500, 1.0)
609-
.build()
610-
.unwrap();
611-
612-
// Different Arc — wholesale replacement happened because Unbounded's
613-
// default `try_resize` returns Err.
614-
assert!(!std::ptr::eq(Arc::as_ptr(&env.memory_pool), pool_ptr));
615-
assert!(matches!(
616-
env.memory_pool.memory_limit(),
617-
MemoryLimit::Finite(500)
618-
));
619-
}
620-
}

‎datafusion/sqllogictest/Cargo.toml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,10 +70,6 @@ tokio-postgres = { version = "0.7.17", optional = true }
7070
[features]
7171
avro = ["datafusion/avro"]
7272
backtrace = ["datafusion/backtrace"]
73-
# Enable the `AccountingAllocator` `GlobalAlloc` wrapper and its thread-local
74-
# byte counter. The binary still has to declare `#[global_allocator]` for it
75-
# to actually take effect — building with this feature on alone is harmless.
76-
memory-accounting = []
7773
postgres = [
7874
"bytes",
7975
"chrono",

‎datafusion/sqllogictest/README.md‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -360,35 +360,6 @@ For focusing on one specific failing test, a file:line filter can be used:
360360
cargo test --test sqllogictests -- --substrait-round-trip binary.slt:23
361361
```
362362

363-
## Running tests: allocator-level memory accounting
364-
365-
Build with `--features memory-accounting` to install a global allocator
366-
wrapper that tracks actual bytes allocated per SLT file and reconciles them
367-
against DataFusion's voluntary `MemoryPool` tracking. The point isn't to
368-
enforce a process-wide budget — it's to catch DataFusion lying about how
369-
much memory it's using. If `MemoryPool` reports 1 MB while the allocator
370-
sees 100 MB go by, _that gap is the bug_.
371-
372-
```shell
373-
cargo test --features memory-accounting --test sqllogictests -- \
374-
--default-pool-size-mb 16384
375-
```
376-
377-
`--default-pool-size-mb` seeds each per-file SLT context's MemoryPool with
378-
the given size in MB and arms the bank as a no-op until a test opts in.
379-
380-
**Opting an individual test in.** Add `SET datafusion.runtime.memory_limit = 'N'` at the top of the `.slt`. The wrapping `AccountingMemoryPool` then
381-
tightens its allocator-level bank to `N * 1.10` (10% headroom). If the test
382-
allocates more than that — including bytes DataFusion's tracker didn't see
383-
— the test panics with an `OverdraftPanic` reporting the actual balance at
384-
panic time. SLTs without a `SET` of `memory_limit` see no change in
385-
behavior; the bank stays loose and `SHOW ALL` continues to render the limit
386-
as `unlimited`.
387-
388-
Inside the runner each file gets its own multi-thread Tokio runtime so
389-
context-ids stamped onto worker threads stay stable for the allocator
390-
hook, and per-file accounts in the bank are isolated from each other.
391-
392363
## `.slt` file format
393364

394365
[`sqllogictest`] was originally written for SQLite to verify the

‎datafusion/sqllogictest/bin/sqllogictests.rs‎

Lines changed: 4 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,6 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
#[cfg(feature = "memory-accounting")]
19-
#[global_allocator]
20-
static GLOBAL: datafusion_sqllogictest::AccountingAllocator =
21-
datafusion_sqllogictest::AccountingAllocator::system();
22-
2318
use clap::{ColorChoice, Parser};
2419
use datafusion::common::instant::Instant;
2520
use datafusion::common::utils::get_available_parallelism;
@@ -143,19 +138,6 @@ async fn run_tests() -> Result<()> {
143138

144139
options.warn_on_ignored();
145140

146-
#[cfg(feature = "memory-accounting")]
147-
if let Some(pool_mb) = options.default_pool_size_mb {
148-
let pool_bytes = pool_mb.saturating_mul(1024 * 1024);
149-
// Same value drives the inner MemoryPool's size and the bank's
150-
// default budget. The wrapper renders this value as `unlimited` in
151-
// `SHOW ALL` (sentinel for "no SET has happened"); once a test
152-
// calls `SET datafusion.runtime.memory_limit`, the wrapper retunes
153-
// the bank to that limit + 10% headroom.
154-
datafusion_sqllogictest::set_memory_tracker_limit(pool_bytes);
155-
datafusion_sqllogictest::set_default_budget(pool_bytes as isize);
156-
log::info!("memory-accounting on: default pool size = {pool_mb} MB");
157-
}
158-
159141
// Print parallelism info for debugging CI performance
160142
eprintln!(
161143
"Running with {} test threads (available parallelism: {})",
@@ -228,7 +210,7 @@ async fn run_tests() -> Result<()> {
228210
let currently_running_sql_tracker_clone =
229211
currently_running_sql_tracker.clone();
230212
let file_start = Instant::now();
231-
let body = async move {
213+
SpawnedTask::spawn(async move {
232214
let result = match (
233215
options.postgres_runner,
234216
options.complete,
@@ -301,41 +283,9 @@ async fn run_tests() -> Result<()> {
301283
}
302284

303285
(result, elapsed)
304-
};
305-
// Each file gets its own multi-thread runtime so a stable per-file
306-
// context-id (stamped via `on_thread_start`) is readable from the
307-
// global allocator hook. Bank accounting and SET-driven limit
308-
// retuning will key off this id in later steps. The outer
309-
// orchestration runtime hosts this via `spawn_blocking` so its
310-
// worker threads aren't blocked by the per-file `block_on`.
311-
//
312-
// Worker count matches `SLT_TARGET_PARTITIONS` so a query's
313-
// partition streams each get a worker rather than contending.
314-
#[cfg(feature = "memory-accounting")]
315-
let spawned = {
316-
let context_id = datafusion_sqllogictest::next_context_id();
317-
SpawnedTask::spawn_blocking(move || {
318-
// Stamp this thread too — `block_on` polls `body` here, so
319-
// statements that don't suspend (e.g. `SET memory_limit`,
320-
// pool construction) run on this thread, not a worker.
321-
datafusion_sqllogictest::set_thread_context_id(context_id);
322-
let runtime = tokio::runtime::Builder::new_multi_thread()
323-
.enable_all()
324-
.worker_threads(datafusion_sqllogictest::SLT_TARGET_PARTITIONS)
325-
.thread_name(format!("slt-file-{context_id}"))
326-
.on_thread_start(move || {
327-
datafusion_sqllogictest::set_thread_context_id(context_id);
328-
})
329-
.build()
330-
.expect("build per-file Tokio runtime");
331-
let out = runtime.block_on(body);
332-
runtime.shutdown_background();
333-
out
334-
})
335-
};
336-
#[cfg(not(feature = "memory-accounting"))]
337-
let spawned = SpawnedTask::spawn(body);
338-
spawned.join().map(move |result| {
286+
})
287+
.join()
288+
.map(move |result| {
339289
let elapsed = match &result {
340290
Ok((_, elapsed)) => *elapsed,
341291
Err(_) => Duration::ZERO,
@@ -965,19 +915,6 @@ struct Options {
965915
default_value_t = ColorChoice::Auto
966916
)]
967917
color: ColorChoice,
968-
969-
#[clap(
970-
long,
971-
help = "Default MemoryPool size in MB for each per-file SLT context. \
972-
The pool is wrapped in AccountingMemoryPool, which doubles \
973-
this value as the 'no SET has happened yet' sentinel — until \
974-
an SLT calls `SET datafusion.runtime.memory_limit`, SHOW ALL \
975-
renders the limit as 'unlimited' and the allocator bank \
976-
stays loose. Once a test SETs a limit, the bank tightens to \
977-
that limit + 10% headroom. Requires the memory-accounting \
978-
feature; ignored without it."
979-
)]
980-
default_pool_size_mb: Option<usize>,
981918
}
982919

983920
impl Options {

0 commit comments

Comments
 (0)