Skip to content

Commit 15a8ebc

Browse files
committed
test: reproduce the RepartitionExec spill pool deadlock with a SQL query
Add a test that runs the query from issue apache#24883 under a 4 MB memory limit, with two partitions and 64-row batches. The query is a grouped COUNT(DISTINCT) on a Utf8View column. It sends `RepartitionExec` into its spill path. On the current code, the first attempt does not complete. The test stops the attempt after 20 seconds and fails. The test does 12 attempts. One attempt that completes is not sufficient for a pass. The test uses a memory limit that is too small for the query. This is necessary to make `RepartitionExec` spill. Once in a few thousand attempts, the greedy memory pool refuses an allocation of an aggregate, because the repartition reservations hold the pool. The test accepts a resources-exhausted error. That error is a correct result of the limit and not the deadlock. Ref: apache#24883 Co-Authored-By: Claude Fable 5.1 <[email protected]> Claude-Session: https://claude.ai/code/session_01RauD5PqPCp4y92RUZUD6C9
1 parent da89c7c commit 15a8ebc

1 file changed

Lines changed: 112 additions & 0 deletions

File tree

‎datafusion/core/tests/memory_limit/repartition_mem_limit.rs‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,18 @@
1616
// under the License.
1717

1818
use std::sync::Arc;
19+
use std::time::Duration;
1920

2021
use arrow::array::{ArrayRef, Int32Array, RecordBatch};
2122
use datafusion::{
2223
assert_batches_sorted_eq,
24+
execution::session_state::SessionStateBuilder,
2325
prelude::{SessionConfig, SessionContext},
2426
};
2527
use datafusion_catalog::MemTable;
2628
use datafusion_common::tree_node::{Transformed, TreeNode};
29+
use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};
30+
use datafusion_execution::memory_pool::{GreedyMemoryPool, MemoryPool};
2731
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
2832
use datafusion_physical_plan::{ExecutionPlanProperties, repartition::RepartitionExec};
2933
use futures::TryStreamExt;
@@ -117,3 +121,111 @@ async fn test_repartition_memory_limit() {
117121
];
118122
assert_batches_sorted_eq!(expected, &all_batches);
119123
}
124+
125+
/// Regression test for <https://github.com/apache/datafusion/issues/24883>.
126+
///
127+
/// In non-preserve-order mode every input task of a `RepartitionExec` shares one
128+
/// spill pool per output partition. When two input tasks spilled concurrently the
129+
/// pool could end up with two open spill files while the reader only ever drained
130+
/// the head file: it parked on a drained-but-unfinished head file even though the
131+
/// batch it was waiting for had been written to the second file. Once every
132+
/// distributor channel was non-empty the gate closed, both input tasks parked in
133+
/// `send`, no sink was dropped, and nothing could wake anybody.
134+
mod spill_pool_deadlock {
135+
use super::*;
136+
137+
/// Memory limit that puts `RepartitionExec` into its spilling path for this
138+
/// query without failing the aggregation outright.
139+
const MEMORY_LIMIT: usize = 4 * 1024 * 1024;
140+
141+
/// Number of attempts. On an unfixed tree the first attempt deadlocks; the
142+
/// budget is only there so a fix cannot pass by luck.
143+
const ATTEMPTS: usize = 12;
144+
145+
/// Generous per-attempt budget: a healthy run of this query takes well under
146+
/// a second.
147+
const PER_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(20);
148+
149+
async fn run_once() -> datafusion::error::Result<usize> {
150+
let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(MEMORY_LIMIT));
151+
let runtime = RuntimeEnvBuilder::new()
152+
// Spilling must be possible: with the disk manager disabled the query
153+
// fails with a resource error instead of deadlocking.
154+
.with_disk_manager_builder(
155+
DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory),
156+
)
157+
.with_memory_pool(pool)
158+
.build_arc()?;
159+
160+
let config = SessionConfig::new()
161+
// Two input partitions feeding one hash RepartitionExec is the
162+
// smallest configuration with two concurrent writers per spill pool.
163+
.with_target_partitions(2)
164+
// Small batches so many small batches reach the repartition spill
165+
// path while the spill file stays far below
166+
// `max_spill_file_size_bytes` and so never rotates.
167+
.with_batch_size(64);
168+
169+
let state = SessionStateBuilder::new()
170+
.with_config(config)
171+
.with_runtime_env(runtime)
172+
.with_default_features()
173+
.build();
174+
let ctx = SessionContext::new_with_state(state);
175+
176+
ctx.sql(
177+
"create table trace_events as
178+
select v % 13 as g,
179+
case when v % 29 = 0 then null
180+
else md5(cast(v % 337 as varchar)) end as trace_id
181+
from generate_series(1, 40000) as t(v)",
182+
)
183+
.await?
184+
.collect()
185+
.await?;
186+
187+
ctx.sql(
188+
"create view tv as
189+
select g, arrow_cast(trace_id, 'Utf8View') as trace_id from trace_events",
190+
)
191+
.await?
192+
.collect()
193+
.await?;
194+
195+
let batches = ctx
196+
.sql("select g, count(distinct trace_id) as n from tv group by g")
197+
.await?
198+
.collect()
199+
.await?;
200+
201+
Ok(batches.iter().map(|b| b.num_rows()).sum())
202+
}
203+
204+
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
205+
async fn repartition_spill_pool_does_not_deadlock() {
206+
for attempt in 0..ATTEMPTS {
207+
match tokio::time::timeout(PER_ATTEMPT_TIMEOUT, run_once()).await {
208+
Err(elapsed) => panic!(
209+
"attempt {attempt}: grouped COUNT(DISTINCT Utf8View) never completed \
210+
within {PER_ATTEMPT_TIMEOUT:?} ({elapsed}); RepartitionExec spill \
211+
pool deadlock"
212+
),
213+
Ok(Ok(rows)) => {
214+
assert_eq!(rows, 13, "attempt {attempt}: wrong row count")
215+
}
216+
// The budget is deliberately far too small for the query, and the
217+
// greedy pool hands memory out first come first served, so once in
218+
// a few thousand attempts an aggregate's allocation is refused
219+
// while the repartition reservations hold the pool. That is a
220+
// legitimate outcome of the limit, not the hang this test guards
221+
// against.
222+
Ok(Err(e))
223+
if matches!(
224+
e.find_root(),
225+
datafusion::error::DataFusionError::ResourcesExhausted(_)
226+
) => {}
227+
Ok(Err(e)) => panic!("attempt {attempt}: query failed: {e}"),
228+
}
229+
}
230+
}
231+
}

0 commit comments

Comments
 (0)