Skip to content

Commit 8cf7439

Browse files
DecisionNerdclaude
andcommitted
test(exec): regression test that a spilling hash exchange does not deadlock (Part of #1985)
Runs the upstream reproduction of apache/datafusion#24883 through the session's own query memory pool: target_partitions=2, batch_size=64, a 2 MiB pool with the disk manager enabled, and a multi-thread runtime. A 40 000 row table is aggregated with count(DISTINCT) over 13 groups, 20 times, under a 60 s deadline so a hang fails instead of stalling the suite. Every group must count all 337 traces and at least one exchange must have spilled. Mutation proof (60 runs each, 4 concurrent, test binaries differ): stock datafusion-physical-plan 54.1.0 ([patch] removed): 0 pass, 60 fail with the deadlock panic. vendored crate with the #24891 port: 100 pass, 0 fail. Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
1 parent 9b382fd commit 8cf7439

2 files changed

Lines changed: 143 additions & 0 deletions

File tree

‎crates/graphforge-exec/src/session.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1752,3 +1752,6 @@ mod tests;
17521752

17531753
#[cfg(test)]
17541754
mod query_memory_tests;
1755+
1756+
#[cfg(test)]
1757+
mod spill_pool_regression_tests;
Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
//! #1985: a hash exchange that spills under a tight memory pool finishes.
2+
//!
3+
//! DataFusion 54.1.0's `RepartitionExec` could deadlock when its spill pool held
4+
//! two open files: the reader waited on a drained file whose writer had moved on,
5+
//! while the batch it was owed sat in a newer file, and nothing woke it
6+
//! (apache/datafusion#24883). The workspace resolves `datafusion-physical-plan`
7+
//! from `vendor/datafusion-physical-plan`, which carries the upstream fix
8+
//! (apache/datafusion#24891). This runs the upstream reproduction through the
9+
//! session's own query memory pool and fails, instead of hanging, if the
10+
//! exchange stops delivering.
11+
12+
use super::*;
13+
use graphforge_ir::RuntimeCatalog;
14+
use std::path::PathBuf;
15+
use std::time::Duration;
16+
use tempfile::TempDir;
17+
18+
/// Memory pool budget of the reproduction.
19+
const POOL_BYTES: u64 = 2 * 1024 * 1024;
20+
/// Rows of the reproduction table.
21+
const ROWS: i64 = 40_000;
22+
/// Distinct `g` values.
23+
const GROUPS: usize = 13;
24+
/// Distinct `trace_id` values in every group: `v % 13` and `v % 337` are
25+
/// independent because 13 and 337 are coprime, and 40 000 rows cover every pair
26+
/// that 13 * 337 = 4 381 admits.
27+
const TRACES_PER_GROUP: i64 = 337;
28+
/// Times the aggregate runs. The deadlock needs a particular interleaving of the
29+
/// exchange's writers and its reader, so one run proves little.
30+
const RUNS: usize = 20;
31+
/// A run takes well under a second; this bounds a hang, it is not a tuning knob.
32+
const RUN_DEADLINE: Duration = Duration::from_secs(60);
33+
34+
fn session(spill: &TempDir) -> (ExecutionSession, TempDir) {
35+
let dir = TempDir::new().unwrap();
36+
let catalog = GraphCatalog::open(dir.path(), None, &RuntimeCatalog::new()).unwrap();
37+
let resources = SessionResourceConfig {
38+
target_partitions: 2,
39+
batch_size: 64,
40+
memory_budget_bytes: POOL_BYTES,
41+
spill_enabled: true,
42+
spill_directory: Some(spill.path().to_path_buf()),
43+
..SessionResourceConfig::default()
44+
};
45+
let session = ExecutionSession::build(
46+
catalog,
47+
None,
48+
PathBuf::new(),
49+
OntologyMode::Exploratory,
50+
None,
51+
OrdinalIdentityConfig::default(),
52+
&resources,
53+
)
54+
.unwrap();
55+
(session, dir)
56+
}
57+
58+
/// Spill operations performed by every `RepartitionExec` of `plan`.
59+
fn exchange_spills(plan: &Arc<dyn ExecutionPlan>) -> usize {
60+
let own = if plan.name() == "RepartitionExec" {
61+
plan.metrics()
62+
.and_then(|metrics| metrics.spill_count())
63+
.unwrap_or(0)
64+
} else {
65+
0
66+
};
67+
own + plan
68+
.children()
69+
.into_iter()
70+
.map(exchange_spills)
71+
.sum::<usize>()
72+
}
73+
74+
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
75+
async fn a_hash_exchange_spilling_under_a_tight_pool_delivers_every_group() {
76+
let spill = TempDir::new().unwrap();
77+
let (session, _dir) = session(&spill);
78+
session
79+
.ctx
80+
.sql(&format!(
81+
"CREATE TABLE t AS SELECT v % {GROUPS} AS g, \
82+
md5(CAST(v % {TRACES_PER_GROUP} AS VARCHAR)) AS trace_id \
83+
FROM generate_series(1, {ROWS}) AS s(v)"
84+
))
85+
.await
86+
.unwrap()
87+
.collect()
88+
.await
89+
.expect("the table must load within the pool");
90+
91+
let mut exchange_spills_seen = 0;
92+
for run in 0..RUNS {
93+
let plan = session
94+
.ctx
95+
.sql("SELECT g, count(DISTINCT trace_id) AS traces FROM t GROUP BY g")
96+
.await
97+
.unwrap()
98+
.create_physical_plan()
99+
.await
100+
.unwrap();
101+
let batches = tokio::time::timeout(
102+
RUN_DEADLINE,
103+
datafusion::physical_plan::collect(Arc::clone(&plan), session.ctx.task_ctx()),
104+
)
105+
.await
106+
.unwrap_or_else(|_| {
107+
panic!("run {run}: the aggregate did not finish within {RUN_DEADLINE:?}: a spilling exchange deadlocked")
108+
})
109+
.expect("a spilling exchange must deliver, not exhaust the pool");
110+
111+
let mut rows = batches
112+
.iter()
113+
.flat_map(|batch| {
114+
let groups = batch
115+
.column(0)
116+
.as_any()
117+
.downcast_ref::<arrow::array::Int64Array>()
118+
.expect("g is an Int64");
119+
let traces = batch
120+
.column(1)
121+
.as_any()
122+
.downcast_ref::<arrow::array::Int64Array>()
123+
.expect("count is an Int64");
124+
(0..batch.num_rows())
125+
.map(|row| (groups.value(row), traces.value(row)))
126+
.collect::<Vec<_>>()
127+
})
128+
.collect::<Vec<_>>();
129+
rows.sort_unstable();
130+
let expected = (0..GROUPS as i64)
131+
.map(|group| (group, TRACES_PER_GROUP))
132+
.collect::<Vec<_>>();
133+
assert_eq!(rows, expected, "run {run}: every group counts every trace");
134+
exchange_spills_seen += exchange_spills(&plan);
135+
}
136+
assert!(
137+
exchange_spills_seen > 0,
138+
"the pool must be tight enough that an exchange spills, or this proves nothing"
139+
);
140+
}

0 commit comments

Comments
 (0)