Skip to content

Commit 956f9a0

Browse files
committed
fix: address review feedback on memory drift logging
- Add MemoryDriftTracker::sample to observe drift without a reservation change - Test the log threshold and re-arm logic - Document that AllocatedBytesFn must be cheap and what DriftSample pool/consumer do and do not mean - Re-wrap the pool after SET datafusion.runtime.memory_limit in SLT - Add --memory-drift-log-threshold / SLT_MEMORY_DRIFT_LOG_THRESHOLD - Warn that attribution can be wrong when files run concurrently and document --test-threads 1 for investigation - Correct the CountingAllocator flush error comment
1 parent 44c87fb commit 956f9a0

5 files changed

Lines changed: 168 additions & 18 deletions

File tree

‎datafusion/execution/src/memory_pool/drift.rs‎

Lines changed: 88 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@ use parking_lot::Mutex;
4545
use super::{MemoryConsumer, MemoryLimit, MemoryPool, MemoryReservation};
4646

4747
/// Returns the number of bytes currently allocated by the process.
48+
///
49+
/// This is called on every reservation change (`grow`, `try_grow` and
50+
/// `shrink`) of every pool that reports to the tracker, so it must be cheap,
51+
/// e.g. a single atomic load. Do not read allocator statistics that need a
52+
/// refresh on each call (such as jemalloc's `epoch`).
4853
pub type AllocatedBytesFn = Arc<dyn Fn() -> usize + Send + Sync>;
4954

5055
/// Default rise in drift, in bytes, needed before another line is logged.
@@ -99,9 +104,14 @@ pub struct MemoryDriftTracker {
99104
/// One observation of drift, recorded by [`MemoryDriftTracker`].
100105
#[derive(Debug, Clone, PartialEq, Eq)]
101106
pub struct DriftSample {
102-
/// Label of the [`DriftLoggingPool`] that made the observation.
107+
/// Label of the [`DriftLoggingPool`] whose reservation change took this
108+
/// sample, or the source passed to [`MemoryDriftTracker::sample`]. When
109+
/// several pools share a tracker, the untracked memory can come from any
110+
/// of them.
103111
pub pool: String,
104-
/// Consumer whose reservation change triggered the observation.
112+
/// Consumer whose reservation change took this sample (empty for
113+
/// [`MemoryDriftTracker::sample`]). This shows when drift was sampled,
114+
/// not what caused it.
105115
pub consumer: String,
106116
/// Bytes reserved across all pools reporting to the tracker.
107117
pub reserved: usize,
@@ -159,6 +169,17 @@ impl MemoryDriftTracker {
159169
self.peak.lock().clone()
160170
}
161171

172+
/// Compare the current allocated bytes with the reserved total now,
173+
/// without a reservation change.
174+
///
175+
/// Drift is otherwise only sampled when a reservation changes, so memory
176+
/// allocated by code that reserves little can go unseen until some other
177+
/// reservation changes. Calling this periodically, e.g. from a timer,
178+
/// closes that gap. `source` is recorded as the pool label.
179+
pub fn sample(&self, source: &str) {
180+
self.observe(source, "", self.reserved());
181+
}
182+
162183
fn grew(&self, pool: &str, consumer: &str, additional: usize) {
163184
let reserved =
164185
self.reserved.fetch_add(additional, Ordering::Relaxed) + additional;
@@ -170,7 +191,8 @@ impl MemoryDriftTracker {
170191
self.observe(pool, consumer, reserved);
171192
}
172193

173-
fn observe(&self, pool: &str, consumer: &str, reserved: usize) {
194+
/// Returns `true` if a line was logged.
195+
fn observe(&self, pool: &str, consumer: &str, reserved: usize) -> bool {
174196
let allocated = (self.allocated)();
175197
let drift = allocated as isize - reserved as isize;
176198

@@ -198,15 +220,17 @@ impl MemoryDriftTracker {
198220
let last = self.last_logged.load(Ordering::Relaxed);
199221
let rose = untracked >= last.saturating_add(self.log_threshold as isize);
200222
if !rose && untracked >= last {
201-
return;
223+
return false;
202224
}
203225
let updated = self
204226
.last_logged
205227
.compare_exchange(last, untracked, Ordering::Relaxed, Ordering::Relaxed)
206228
.is_ok();
207-
if updated && rose {
229+
let logged = updated && rose;
230+
if logged {
208231
log::info!("memory drift: {}", sample());
209232
}
233+
logged
210234
}
211235
}
212236

@@ -421,6 +445,65 @@ mod tests {
421445
);
422446
}
423447

448+
#[test]
449+
fn logs_each_rise_by_the_threshold_and_rearms_after_a_fall() {
450+
let allocated = Arc::new(AtomicUsize::new(0));
451+
let source = Arc::clone(&allocated);
452+
let tracker =
453+
MemoryDriftTracker::new(Arc::new(move || source.load(Ordering::Relaxed)))
454+
.with_log_threshold(1000);
455+
let observe = |bytes: usize| {
456+
allocated.store(bytes, Ordering::Relaxed);
457+
tracker.observe("p", "c", 0)
458+
};
459+
460+
// Below the threshold: nothing logged.
461+
assert!(!observe(999));
462+
// Reaching the threshold logs and moves the baseline to 1000.
463+
assert!(observe(1000));
464+
assert!(!observe(1999));
465+
assert!(observe(2000));
466+
// Falling drift is not logged but lowers the baseline to 500...
467+
assert!(!observe(500));
468+
// ...so a rise of the threshold from there logs again.
469+
assert!(!observe(1499));
470+
assert!(observe(1500));
471+
}
472+
473+
#[test]
474+
fn negative_drift_counts_as_zero_for_logging() {
475+
let (allocated, tracker) = tracker();
476+
let tracker = Arc::into_inner(tracker).unwrap().with_log_threshold(1000);
477+
478+
assert!(!tracker.observe("p", "c", 5000));
479+
allocated.store(999, Ordering::Relaxed);
480+
assert!(!tracker.observe("p", "c", 0));
481+
allocated.store(1000, Ordering::Relaxed);
482+
assert!(tracker.observe("p", "c", 0));
483+
}
484+
485+
#[test]
486+
fn sample_observes_without_a_reservation_change() {
487+
let (allocated, tracker) = tracker();
488+
let pool = pool(&tracker, "q1");
489+
let reservation = MemoryConsumer::new("a").register(&pool);
490+
reservation.grow(100);
491+
492+
allocated.store(5000, Ordering::Relaxed);
493+
tracker.sample("timer");
494+
495+
assert_eq!(
496+
tracker.peak_drift(),
497+
Some(DriftSample {
498+
pool: "timer".to_string(),
499+
consumer: String::new(),
500+
reserved: 100,
501+
allocated: 5000,
502+
drift: 4900,
503+
})
504+
);
505+
}
506+
424507
#[test]
425508
fn delegates_to_the_wrapped_pool() {
426509
let (_allocated, tracker) = tracker();

‎datafusion/sqllogictest/README.md‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,16 +109,32 @@ allocated, to find operators whose memory is not tracked by the pool (see
109109
default and prints the largest drift seen at the end of the run. It only logs
110110
and never fails a test.
111111

112-
Test files run concurrently, so the comparison is process-wide: allocated bytes
113-
across the whole process against reservations summed across all files. Files
114-
that `SET datafusion.runtime.memory_limit` replace their pool and drop out of
115-
the reserved total.
112+
Test files run concurrently by default, so the comparison is process-wide:
113+
allocated bytes across the whole process against reservations summed across all
114+
files. In this mode, the file in the output can be wrong. To attribute drift to
115+
one file, run one file at a time:
116116

117117
```shell
118-
# Log each 64 MB rise in drift, with the file and consumer that triggered it
118+
cargo test --test sqllogictests -- --test-threads 1
119+
```
120+
121+
In both modes, the consumer in the output is the reservation change that took
122+
the sample, not necessarily the code that allocated the untracked memory. Most
123+
drift at this scale is memory that `MemoryPool` does not track by design (for
124+
example, in-flight batches and Parquet read buffers), so a large drift is a lead
125+
to investigate, not necessarily a bug.
126+
127+
```shell
128+
# Log each 64 MB rise in drift, with the file and consumer that took the sample
119129
RUST_LOG=datafusion_execution::memory_pool=info cargo test --test sqllogictests
120130
```
121131

132+
```shell
133+
# Log each 4 MB rise instead (also settable via SLT_MEMORY_DRIFT_LOG_THRESHOLD)
134+
RUST_LOG=datafusion_execution::memory_pool=info cargo test --test sqllogictests -- \
135+
--test-threads 1 --memory-drift-log-threshold 4194304
136+
```
137+
122138
```shell
123139
# Disable
124140
cargo test --test sqllogictests -- --memory-drift false

‎datafusion/sqllogictest/bin/sqllogictests.rs‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use clap::{ColorChoice, Parser};
1919
use datafusion::common::instant::Instant;
2020
use datafusion::common::utils::get_available_parallelism;
2121
use datafusion::common::{DataFusionError, Result, exec_datafusion_err, exec_err};
22+
use datafusion::execution::memory_pool::DEFAULT_DRIFT_LOG_THRESHOLD;
2223
#[cfg(feature = "substrait")]
2324
use datafusion_sqllogictest::DataFusionSubstraitRoundTrip;
2425
use datafusion_sqllogictest::TestFile;
@@ -146,7 +147,7 @@ async fn run_tests() -> Result<()> {
146147
options.warn_on_ignored();
147148

148149
if options.memory_drift {
149-
enable_memory_drift_logging();
150+
enable_memory_drift_logging(options.memory_drift_log_threshold);
150151
}
151152

152153
// Print parallelism info for debugging CI performance
@@ -398,6 +399,12 @@ async fn run_tests() -> Result<()> {
398399

399400
if let Some(peak) = memory_drift_tracker().and_then(|t| t.peak_drift()) {
400401
eprintln!("Peak memory drift: {peak}");
402+
if options.test_threads > 1 {
403+
eprintln!(
404+
"Test files ran concurrently, so the file and consumer above can be wrong. \
405+
Run with --test-threads 1 to attribute drift to one file."
406+
);
407+
}
401408
}
402409

403410
#[cfg(feature = "postgres")]
@@ -1129,6 +1136,15 @@ struct Options {
11291136
)]
11301137
memory_drift: bool,
11311138

1139+
#[clap(
1140+
long,
1141+
env = "SLT_MEMORY_DRIFT_LOG_THRESHOLD",
1142+
value_name = "BYTES",
1143+
default_value_t = DEFAULT_DRIFT_LOG_THRESHOLD,
1144+
help = "Rise in memory drift, in bytes, needed before another line is logged"
1145+
)]
1146+
memory_drift_log_threshold: usize,
1147+
11321148
#[clap(
11331149
long,
11341150
value_name = "MODE",

‎datafusion/sqllogictest/src/engines/datafusion_engine/runner.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ use super::{DFSqlLogicTestError, error::Result, normalize};
2424
use crate::engines::currently_executed_sql::CurrentlyExecutingSqlTracker;
2525
use crate::engines::output::{DFColumnType, DFOutput};
2626
use crate::is_spark_path;
27+
use crate::memory_drift::rewrap_replaced_pool;
2728
use arrow::record_batch::RecordBatch;
2829
use async_trait::async_trait;
2930
use datafusion::physical_plan::common::collect;
@@ -156,6 +157,7 @@ impl sqllogictest::AsyncDB for DataFusion {
156157
let start = Instant::now();
157158
let result = run_query(&self.ctx, is_spark_path(&self.relative_path), sql).await;
158159
let duration = start.elapsed();
160+
rewrap_replaced_pool(&self.ctx, &self.relative_path.display().to_string());
159161

160162
self.currently_executing_sql_tracker.remove_sql(tracked_sql);
161163

‎datafusion/sqllogictest/src/memory_drift.rs‎

Lines changed: 40 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323
//! [`MemoryDriftTracker`], which compares the sum of all reservations with the
2424
//! bytes counted by [`CountingAllocator`].
2525
//!
26-
//! Files that `SET datafusion.runtime.memory_limit` replace their pool, so
27-
//! their reservations after that point are not included in the total.
26+
//! Files that `SET datafusion.runtime.memory_limit` replace their pool; the
27+
//! runner wraps the replacement too (see [`rewrap_replaced_pool`]).
2828
//!
2929
//! This only logs. It never fails a test.
3030
@@ -37,9 +37,12 @@ use std::{
3737
},
3838
};
3939

40+
use datafusion::execution::SessionStateBuilder;
4041
use datafusion::execution::memory_pool::{
4142
DriftLoggingPool, MemoryDriftTracker, MemoryPool,
4243
};
44+
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
45+
use datafusion::prelude::SessionContext;
4346

4447
static ALLOCATED: AtomicIsize = AtomicIsize::new(0);
4548
static COUNTING: AtomicBool = AtomicBool::new(false);
@@ -63,7 +66,9 @@ impl<A> CountingAllocator<A> {
6366

6467
/// Per-thread count is flushed to [`ALLOCATED`] once it moves this far, so
6568
/// threads do not contend on one atomic for every allocation. The global count
66-
/// is therefore accurate to within `threads * FLUSH_BYTES`.
69+
/// is therefore off by up to `FLUSH_BYTES` per live thread. The unflushed
70+
/// count of a thread that exits (e.g. an idle Tokio blocking thread) is lost,
71+
/// so this error can grow during a long run.
6772
const FLUSH_BYTES: isize = 256 * 1024;
6873

6974
thread_local! {
@@ -130,11 +135,16 @@ pub fn allocated_bytes() -> usize {
130135
}
131136

132137
/// Start counting allocations and wrap the memory pool of every test file
133-
/// created from now on. Only has an effect if [`CountingAllocator`] is the
134-
/// global allocator.
135-
pub fn enable_memory_drift_logging() {
138+
/// created from now on, logging each rise in drift of `log_threshold` bytes.
139+
/// Only has an effect if [`CountingAllocator`] is the global allocator.
140+
pub fn enable_memory_drift_logging(log_threshold: usize) {
136141
COUNTING.store(true, Ordering::Relaxed);
137-
TRACKER.get_or_init(|| Arc::new(MemoryDriftTracker::new(Arc::new(allocated_bytes))));
142+
TRACKER.get_or_init(|| {
143+
Arc::new(
144+
MemoryDriftTracker::new(Arc::new(allocated_bytes))
145+
.with_log_threshold(log_threshold),
146+
)
147+
});
138148
}
139149

140150
/// The process-wide tracker, if [`enable_memory_drift_logging`] was called.
@@ -152,3 +162,26 @@ pub fn wrap_pool(pool: Arc<dyn MemoryPool>, label: &str) -> Arc<dyn MemoryPool>
152162
None => pool,
153163
}
154164
}
165+
166+
/// Wrap the memory pool of `ctx` again if a statement replaced it, e.g.
167+
/// `SET datafusion.runtime.memory_limit`, so its reservations keep being
168+
/// counted. Does nothing if drift logging is not enabled.
169+
pub(crate) fn rewrap_replaced_pool(ctx: &SessionContext, label: &str) {
170+
if memory_drift_tracker().is_none() {
171+
return;
172+
}
173+
let state = ctx.state_ref();
174+
let mut state = state.write();
175+
let runtime = state.runtime_env();
176+
if runtime.memory_pool.is::<DriftLoggingPool>() {
177+
return;
178+
}
179+
let pool = wrap_pool(Arc::clone(&runtime.memory_pool), label);
180+
let runtime = RuntimeEnvBuilder::from_runtime_env(runtime)
181+
.with_memory_pool(pool)
182+
.build_arc()
183+
.expect("rebuilding an existing runtime succeeds");
184+
*state = SessionStateBuilder::from(state.clone())
185+
.with_runtime_env(runtime)
186+
.build();
187+
}

0 commit comments

Comments
 (0)