|
16 | 16 | // under the License. |
17 | 17 |
|
18 | 18 | use std::sync::Arc; |
| 19 | +use std::time::Duration; |
19 | 20 |
|
20 | 21 | use arrow::array::{ArrayRef, Int32Array, RecordBatch}; |
21 | 22 | use datafusion::{ |
22 | 23 | assert_batches_sorted_eq, |
| 24 | + execution::session_state::SessionStateBuilder, |
23 | 25 | prelude::{SessionConfig, SessionContext}, |
24 | 26 | }; |
25 | 27 | use datafusion_catalog::MemTable; |
26 | 28 | use datafusion_common::tree_node::{Transformed, TreeNode}; |
| 29 | +use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; |
| 30 | +use datafusion_execution::memory_pool::{GreedyMemoryPool, MemoryPool}; |
27 | 31 | use datafusion_execution::runtime_env::RuntimeEnvBuilder; |
28 | 32 | use datafusion_physical_plan::{ExecutionPlanProperties, repartition::RepartitionExec}; |
29 | 33 | use futures::TryStreamExt; |
@@ -117,3 +121,111 @@ async fn test_repartition_memory_limit() { |
117 | 121 | ]; |
118 | 122 | assert_batches_sorted_eq!(expected, &all_batches); |
119 | 123 | } |
| 124 | + |
| 125 | +/// Regression test for <https://github.com/apache/datafusion/issues/24883>. |
| 126 | +/// |
| 127 | +/// In non-preserve-order mode every input task of a `RepartitionExec` shares one |
| 128 | +/// spill pool per output partition. When two input tasks spilled concurrently the |
| 129 | +/// pool could end up with two open spill files while the reader only ever drained |
| 130 | +/// the head file: it parked on a drained-but-unfinished head file even though the |
| 131 | +/// batch it was waiting for had been written to the second file. Once every |
| 132 | +/// distributor channel was non-empty the gate closed, both input tasks parked in |
| 133 | +/// `send`, no sink was dropped, and nothing could wake anybody. |
| 134 | +mod spill_pool_deadlock { |
| 135 | + use super::*; |
| 136 | + |
| 137 | + /// Memory limit that puts `RepartitionExec` into its spilling path for this |
| 138 | + /// query without failing the aggregation outright. |
| 139 | + const MEMORY_LIMIT: usize = 4 * 1024 * 1024; |
| 140 | + |
| 141 | + /// Number of attempts. On an unfixed tree the first attempt deadlocks; the |
| 142 | + /// budget is only there so a fix cannot pass by luck. |
| 143 | + const ATTEMPTS: usize = 12; |
| 144 | + |
| 145 | + /// Generous per-attempt budget: a healthy run of this query takes well under |
| 146 | + /// a second. |
| 147 | + const PER_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(20); |
| 148 | + |
| 149 | + async fn run_once() -> datafusion::error::Result<usize> { |
| 150 | + let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(MEMORY_LIMIT)); |
| 151 | + let runtime = RuntimeEnvBuilder::new() |
| 152 | + // Spilling must be possible: with the disk manager disabled the query |
| 153 | + // fails with a resource error instead of deadlocking. |
| 154 | + .with_disk_manager_builder( |
| 155 | + DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory), |
| 156 | + ) |
| 157 | + .with_memory_pool(pool) |
| 158 | + .build_arc()?; |
| 159 | + |
| 160 | + let config = SessionConfig::new() |
| 161 | + // Two input partitions feeding one hash RepartitionExec is the |
| 162 | + // smallest configuration with two concurrent writers per spill pool. |
| 163 | + .with_target_partitions(2) |
| 164 | + // Small batches so many small batches reach the repartition spill |
| 165 | + // path while the spill file stays far below |
| 166 | + // `max_spill_file_size_bytes` and so never rotates. |
| 167 | + .with_batch_size(64); |
| 168 | + |
| 169 | + let state = SessionStateBuilder::new() |
| 170 | + .with_config(config) |
| 171 | + .with_runtime_env(runtime) |
| 172 | + .with_default_features() |
| 173 | + .build(); |
| 174 | + let ctx = SessionContext::new_with_state(state); |
| 175 | + |
| 176 | + ctx.sql( |
| 177 | + "create table trace_events as |
| 178 | + select v % 13 as g, |
| 179 | + case when v % 29 = 0 then null |
| 180 | + else md5(cast(v % 337 as varchar)) end as trace_id |
| 181 | + from generate_series(1, 40000) as t(v)", |
| 182 | + ) |
| 183 | + .await? |
| 184 | + .collect() |
| 185 | + .await?; |
| 186 | + |
| 187 | + ctx.sql( |
| 188 | + "create view tv as |
| 189 | + select g, arrow_cast(trace_id, 'Utf8View') as trace_id from trace_events", |
| 190 | + ) |
| 191 | + .await? |
| 192 | + .collect() |
| 193 | + .await?; |
| 194 | + |
| 195 | + let batches = ctx |
| 196 | + .sql("select g, count(distinct trace_id) as n from tv group by g") |
| 197 | + .await? |
| 198 | + .collect() |
| 199 | + .await?; |
| 200 | + |
| 201 | + Ok(batches.iter().map(|b| b.num_rows()).sum()) |
| 202 | + } |
| 203 | + |
| 204 | + #[tokio::test(flavor = "multi_thread", worker_threads = 8)] |
| 205 | + async fn repartition_spill_pool_does_not_deadlock() { |
| 206 | + for attempt in 0..ATTEMPTS { |
| 207 | + match tokio::time::timeout(PER_ATTEMPT_TIMEOUT, run_once()).await { |
| 208 | + Err(elapsed) => panic!( |
| 209 | + "attempt {attempt}: grouped COUNT(DISTINCT Utf8View) never completed \ |
| 210 | + within {PER_ATTEMPT_TIMEOUT:?} ({elapsed}); RepartitionExec spill \ |
| 211 | + pool deadlock" |
| 212 | + ), |
| 213 | + Ok(Ok(rows)) => { |
| 214 | + assert_eq!(rows, 13, "attempt {attempt}: wrong row count") |
| 215 | + } |
| 216 | + // The budget is deliberately far too small for the query, and the |
| 217 | + // greedy pool hands memory out first come first served, so once in |
| 218 | + // a few thousand attempts an aggregate's allocation is refused |
| 219 | + // while the repartition reservations hold the pool. That is a |
| 220 | + // legitimate outcome of the limit, not the hang this test guards |
| 221 | + // against. |
| 222 | + Ok(Err(e)) |
| 223 | + if matches!( |
| 224 | + e.find_root(), |
| 225 | + datafusion::error::DataFusionError::ResourcesExhausted(_) |
| 226 | + ) => {} |
| 227 | + Ok(Err(e)) => panic!("attempt {attempt}: query failed: {e}"), |
| 228 | + } |
| 229 | + } |
| 230 | + } |
| 231 | +} |
0 commit comments