|
| 1 | +//! #1985: a hash exchange that spills under a tight memory pool finishes. |
| 2 | +//! |
| 3 | +//! DataFusion 54.1.0's `RepartitionExec` could deadlock when its spill pool held |
| 4 | +//! two open files: the reader waited on a drained file whose writer had moved on, |
| 5 | +//! while the batch it was owed sat in a newer file, and nothing woke it |
| 6 | +//! (apache/datafusion#24883). The workspace resolves `datafusion-physical-plan` |
| 7 | +//! from `vendor/datafusion-physical-plan`, which carries the upstream fix |
| 8 | +//! (apache/datafusion#24891). This runs the upstream reproduction through the |
| 9 | +//! session's own query memory pool and fails, instead of hanging, if the |
| 10 | +//! exchange stops delivering. |
| 11 | +
|
| 12 | +use super::*; |
| 13 | +use graphforge_ir::RuntimeCatalog; |
| 14 | +use std::path::PathBuf; |
| 15 | +use std::time::Duration; |
| 16 | +use tempfile::TempDir; |
| 17 | + |
| 18 | +/// Memory pool budget of the reproduction. |
| 19 | +const POOL_BYTES: u64 = 2 * 1024 * 1024; |
| 20 | +/// Rows of the reproduction table. |
| 21 | +const ROWS: i64 = 40_000; |
| 22 | +/// Distinct `g` values. |
| 23 | +const GROUPS: usize = 13; |
| 24 | +/// Distinct `trace_id` values in every group: `v % 13` and `v % 337` are |
| 25 | +/// independent because 13 and 337 are coprime, and 40 000 rows cover every pair |
| 26 | +/// that 13 * 337 = 4 381 admits. |
| 27 | +const TRACES_PER_GROUP: i64 = 337; |
| 28 | +/// Times the aggregate runs. The deadlock needs a particular interleaving of the |
| 29 | +/// exchange's writers and its reader, so one run proves little. |
| 30 | +const RUNS: usize = 20; |
| 31 | +/// A run takes well under a second; this bounds a hang, it is not a tuning knob. |
| 32 | +const RUN_DEADLINE: Duration = Duration::from_secs(60); |
| 33 | + |
| 34 | +fn session(spill: &TempDir) -> (ExecutionSession, TempDir) { |
| 35 | + let dir = TempDir::new().unwrap(); |
| 36 | + let catalog = GraphCatalog::open(dir.path(), None, &RuntimeCatalog::new()).unwrap(); |
| 37 | + let resources = SessionResourceConfig { |
| 38 | + target_partitions: 2, |
| 39 | + batch_size: 64, |
| 40 | + memory_budget_bytes: POOL_BYTES, |
| 41 | + spill_enabled: true, |
| 42 | + spill_directory: Some(spill.path().to_path_buf()), |
| 43 | + ..SessionResourceConfig::default() |
| 44 | + }; |
| 45 | + let session = ExecutionSession::build( |
| 46 | + catalog, |
| 47 | + None, |
| 48 | + PathBuf::new(), |
| 49 | + OntologyMode::Exploratory, |
| 50 | + None, |
| 51 | + OrdinalIdentityConfig::default(), |
| 52 | + &resources, |
| 53 | + ) |
| 54 | + .unwrap(); |
| 55 | + (session, dir) |
| 56 | +} |
| 57 | + |
| 58 | +/// Spill operations performed by every `RepartitionExec` of `plan`. |
| 59 | +fn exchange_spills(plan: &Arc<dyn ExecutionPlan>) -> usize { |
| 60 | + let own = if plan.name() == "RepartitionExec" { |
| 61 | + plan.metrics() |
| 62 | + .and_then(|metrics| metrics.spill_count()) |
| 63 | + .unwrap_or(0) |
| 64 | + } else { |
| 65 | + 0 |
| 66 | + }; |
| 67 | + own + plan |
| 68 | + .children() |
| 69 | + .into_iter() |
| 70 | + .map(exchange_spills) |
| 71 | + .sum::<usize>() |
| 72 | +} |
| 73 | + |
| 74 | +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] |
| 75 | +async fn a_hash_exchange_spilling_under_a_tight_pool_delivers_every_group() { |
| 76 | + let spill = TempDir::new().unwrap(); |
| 77 | + let (session, _dir) = session(&spill); |
| 78 | + session |
| 79 | + .ctx |
| 80 | + .sql(&format!( |
| 81 | + "CREATE TABLE t AS SELECT v % {GROUPS} AS g, \ |
| 82 | + md5(CAST(v % {TRACES_PER_GROUP} AS VARCHAR)) AS trace_id \ |
| 83 | + FROM generate_series(1, {ROWS}) AS s(v)" |
| 84 | + )) |
| 85 | + .await |
| 86 | + .unwrap() |
| 87 | + .collect() |
| 88 | + .await |
| 89 | + .expect("the table must load within the pool"); |
| 90 | + |
| 91 | + let mut exchange_spills_seen = 0; |
| 92 | + for run in 0..RUNS { |
| 93 | + let plan = session |
| 94 | + .ctx |
| 95 | + .sql("SELECT g, count(DISTINCT trace_id) AS traces FROM t GROUP BY g") |
| 96 | + .await |
| 97 | + .unwrap() |
| 98 | + .create_physical_plan() |
| 99 | + .await |
| 100 | + .unwrap(); |
| 101 | + let batches = tokio::time::timeout( |
| 102 | + RUN_DEADLINE, |
| 103 | + datafusion::physical_plan::collect(Arc::clone(&plan), session.ctx.task_ctx()), |
| 104 | + ) |
| 105 | + .await |
| 106 | + .unwrap_or_else(|_| { |
| 107 | + panic!("run {run}: the aggregate did not finish within {RUN_DEADLINE:?}: a spilling exchange deadlocked") |
| 108 | + }) |
| 109 | + .expect("a spilling exchange must deliver, not exhaust the pool"); |
| 110 | + |
| 111 | + let mut rows = batches |
| 112 | + .iter() |
| 113 | + .flat_map(|batch| { |
| 114 | + let groups = batch |
| 115 | + .column(0) |
| 116 | + .as_any() |
| 117 | + .downcast_ref::<arrow::array::Int64Array>() |
| 118 | + .expect("g is an Int64"); |
| 119 | + let traces = batch |
| 120 | + .column(1) |
| 121 | + .as_any() |
| 122 | + .downcast_ref::<arrow::array::Int64Array>() |
| 123 | + .expect("count is an Int64"); |
| 124 | + (0..batch.num_rows()) |
| 125 | + .map(|row| (groups.value(row), traces.value(row))) |
| 126 | + .collect::<Vec<_>>() |
| 127 | + }) |
| 128 | + .collect::<Vec<_>>(); |
| 129 | + rows.sort_unstable(); |
| 130 | + let expected = (0..GROUPS as i64) |
| 131 | + .map(|group| (group, TRACES_PER_GROUP)) |
| 132 | + .collect::<Vec<_>>(); |
| 133 | + assert_eq!(rows, expected, "run {run}: every group counts every trace"); |
| 134 | + exchange_spills_seen += exchange_spills(&plan); |
| 135 | + } |
| 136 | + assert!( |
| 137 | + exchange_spills_seen > 0, |
| 138 | + "the pool must be tight enough that an exchange spills, or this proves nothing" |
| 139 | + ); |
| 140 | +} |
0 commit comments