|
16 | 16 | // under the License. |
17 | 17 |
|
18 | 18 | use std::sync::Arc; |
| 19 | +use std::time::Duration; |
19 | 20 |
|
20 | 21 | use arrow::array::{ArrayRef, Int32Array, RecordBatch}; |
21 | 22 | use datafusion::{ |
22 | 23 | assert_batches_sorted_eq, |
| 24 | + execution::session_state::SessionStateBuilder, |
23 | 25 | prelude::{SessionConfig, SessionContext}, |
24 | 26 | }; |
25 | 27 | use datafusion_catalog::MemTable; |
26 | 28 | use datafusion_common::tree_node::{Transformed, TreeNode}; |
| 29 | +use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; |
| 30 | +use datafusion_execution::memory_pool::{GreedyMemoryPool, MemoryPool}; |
27 | 31 | use datafusion_execution::runtime_env::RuntimeEnvBuilder; |
28 | | -use datafusion_physical_plan::{ExecutionPlanProperties, repartition::RepartitionExec}; |
| 32 | +use datafusion_physical_plan::{ |
| 33 | + ExecutionPlanProperties, collect, repartition::RepartitionExec, |
| 34 | +}; |
29 | 35 | use futures::TryStreamExt; |
30 | 36 | use itertools::Itertools; |
31 | 37 |
|
@@ -117,3 +123,142 @@ async fn test_repartition_memory_limit() { |
117 | 123 | ]; |
118 | 124 | assert_batches_sorted_eq!(expected, &all_batches); |
119 | 125 | } |
| 126 | + |
| 127 | +/// Regression test for <https://github.com/apache/datafusion/issues/24883>. |
| 128 | +/// |
| 129 | +/// In non-preserve-order mode every input task of a `RepartitionExec` shares one |
| 130 | +/// spill pool per output partition. When two input tasks spilled concurrently the |
| 131 | +/// pool could end up with two open spill files while the reader only ever drained |
| 132 | +/// the head file: it parked on a drained-but-unfinished head file even though the |
| 133 | +/// batch it was waiting for had been written to the second file. Once every |
| 134 | +/// distributor channel was non-empty the gate closed, both input tasks parked in |
| 135 | +/// `send`, no sink was dropped, and nothing could wake anybody. |
| 136 | +mod spill_pool_deadlock { |
| 137 | + use super::*; |
| 138 | + |
| 139 | + /// Memory limit that puts `RepartitionExec` into its spilling path for this |
| 140 | + /// query without failing the aggregation outright. |
| 141 | + const MEMORY_LIMIT: usize = 4 * 1024 * 1024; |
| 142 | + |
| 143 | + /// Number of attempts. On an unfixed tree the first attempt deadlocks; the |
| 144 | + /// budget is only there so a fix cannot pass by luck. |
| 145 | + const ATTEMPTS: usize = 12; |
| 146 | + |
| 147 | + /// Generous per-attempt budget: a healthy run of this query takes well under |
| 148 | + /// a second. |
| 149 | + const PER_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(20); |
| 150 | + |
| 151 | + /// Runs the query once. Returns the number of output rows and the number |
| 152 | + /// of spills the `RepartitionExec`s did. |
| 153 | + async fn run_once() -> datafusion::error::Result<(usize, usize)> { |
| 154 | + let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(MEMORY_LIMIT)); |
| 155 | + let runtime = RuntimeEnvBuilder::new() |
| 156 | + // Spilling must be possible: with the disk manager disabled the query |
| 157 | + // fails with a resource error instead of deadlocking. |
| 158 | + .with_disk_manager_builder( |
| 159 | + DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory), |
| 160 | + ) |
| 161 | + .with_memory_pool(pool) |
| 162 | + .build_arc()?; |
| 163 | + |
| 164 | + let config = SessionConfig::new() |
| 165 | + // Two input partitions feeding one hash RepartitionExec is the |
| 166 | + // smallest configuration with two concurrent writers per spill pool. |
| 167 | + .with_target_partitions(2) |
| 168 | + // Small batches so many small batches reach the repartition spill |
| 169 | + // path while the spill file stays far below |
| 170 | + // `max_spill_file_size_bytes` and so never rotates. |
| 171 | + .with_batch_size(64); |
| 172 | + |
| 173 | + let state = SessionStateBuilder::new() |
| 174 | + .with_config(config) |
| 175 | + .with_runtime_env(runtime) |
| 176 | + .with_default_features() |
| 177 | + .build(); |
| 178 | + let ctx = SessionContext::new_with_state(state); |
| 179 | + |
| 180 | + ctx.sql( |
| 181 | + "create table trace_events as |
| 182 | + select v % 13 as g, |
| 183 | + case when v % 29 = 0 then null |
| 184 | + else md5(cast(v % 337 as varchar)) end as trace_id |
| 185 | + from generate_series(1, 40000) as t(v)", |
| 186 | + ) |
| 187 | + .await? |
| 188 | + .collect() |
| 189 | + .await?; |
| 190 | + |
| 191 | + ctx.sql( |
| 192 | + "create view tv as |
| 193 | + select g, arrow_cast(trace_id, 'Utf8View') as trace_id from trace_events", |
| 194 | + ) |
| 195 | + .await? |
| 196 | + .collect() |
| 197 | + .await?; |
| 198 | + |
| 199 | + let logical = ctx |
| 200 | + .state() |
| 201 | + .create_logical_plan( |
| 202 | + "select g, count(distinct trace_id) as n from tv group by g", |
| 203 | + ) |
| 204 | + .await?; |
| 205 | + let plan = ctx.state().create_physical_plan(&logical).await?; |
| 206 | + let batches = collect(Arc::clone(&plan), ctx.task_ctx()).await?; |
| 207 | + |
| 208 | + // Count the spills of the `RepartitionExec`s. The caller uses this to |
| 209 | + // make sure the query still goes through the spill path: if a change |
| 210 | + // stops it spilling, this test keeps passing but no longer covers the |
| 211 | + // deadlock, and that must fail loudly instead. |
| 212 | + let mut spills = 0; |
| 213 | + plan.transform_down(|node| { |
| 214 | + if node.is::<RepartitionExec>() { |
| 215 | + spills += node.metrics().and_then(|m| m.spill_count()).unwrap_or(0); |
| 216 | + } |
| 217 | + Ok(Transformed::no(node)) |
| 218 | + })?; |
| 219 | + |
| 220 | + Ok((batches.iter().map(|b| b.num_rows()).sum(), spills)) |
| 221 | + } |
| 222 | + |
| 223 | + #[tokio::test(flavor = "multi_thread", worker_threads = 8)] |
| 224 | + async fn repartition_spill_pool_does_not_deadlock() { |
| 225 | + let mut attempts_that_spilled = 0; |
| 226 | + for attempt in 0..ATTEMPTS { |
| 227 | + match tokio::time::timeout(PER_ATTEMPT_TIMEOUT, run_once()).await { |
| 228 | + Err(elapsed) => panic!( |
| 229 | + "attempt {attempt}: grouped COUNT(DISTINCT Utf8View) never completed \ |
| 230 | + within {PER_ATTEMPT_TIMEOUT:?} ({elapsed}); RepartitionExec spill \ |
| 231 | + pool deadlock" |
| 232 | + ), |
| 233 | + Ok(Ok((rows, spills))) => { |
| 234 | + assert_eq!(rows, 13, "attempt {attempt}: wrong row count"); |
| 235 | + if spills > 0 { |
| 236 | + attempts_that_spilled += 1; |
| 237 | + } |
| 238 | + } |
| 239 | + // The budget is deliberately far too small for the query, and the |
| 240 | + // greedy pool hands memory out first come first served, so once in |
| 241 | + // a few thousand attempts an aggregate's allocation is refused |
| 242 | + // while the repartition reservations hold the pool. That is a |
| 243 | + // legitimate outcome of the limit, not the hang this test guards |
| 244 | + // against. |
| 245 | + Ok(Err(e)) |
| 246 | + if matches!( |
| 247 | + e.find_root(), |
| 248 | + datafusion::error::DataFusionError::ResourcesExhausted(_) |
| 249 | + ) => {} |
| 250 | + Ok(Err(e)) => panic!("attempt {attempt}: query failed: {e}"), |
| 251 | + } |
| 252 | + } |
| 253 | + |
| 254 | + // The memory limit must be tight enough that `RepartitionExec` spills, |
| 255 | + // because the deadlock is in its spill pool. Timing decides how many |
| 256 | + // attempts spill, but if none of them do, the limit no longer forces |
| 257 | + // the spill path and this test has stopped testing anything. |
| 258 | + assert!( |
| 259 | + attempts_that_spilled > 0, |
| 260 | + "no attempt reached the RepartitionExec spill path in {ATTEMPTS} \ |
| 261 | + attempts; the memory limit no longer forces a spill" |
| 262 | + ); |
| 263 | + } |
| 264 | +} |
0 commit comments