Skip to content

Commit 8348047

Browse files
authored
chore: Remove memory limits from property tests for now, and add spilling integration test. (#7439)
## What Lift `work_mem` constraints in the `qgen` property tests and cover DataFusion disk spilling in a dedicated parameterized `rstest` suite (`tests/tests/spill.rs`). ## Why Until `1c8c0bc9bf` (`#7108`), MPP spilling was covered by a `pg_regress` test; `1c8c0bc9bf` replaced that with a single-shape `#[pg_test]` and attempted to cover memory pressure more broadly via shape-dependent `work_mem` bounds in the `qgen` property tests. In practice, upstream DataFusion does not yet support spilling across enough operators (notably `HashJoinExec`: apache/datafusion#24768) for arbitrary generated query shapes to run under tight `work_mem` limits. ## How * Set `work_mem` to `1GB` unconditionally in `PgGucs::set_with` (`tests/src/fixtures/querygen/mod.rs`), and drop the `work_mem` overrides and `sketch_keys` workaround in `tests/tests/qgen.rs` and `tests/src/fixtures/querygen/pdbagggen.rs`. * Add `datafusion_spill_to_disk` in `tests/tests/spill.rs` parameterized over `(Serial, Mpp)` and the spillable `AggregateScan` plan shapes.
1 parent e6c9a44 commit 8348047

6 files changed

Lines changed: 565 additions & 95 deletions

File tree

‎tests/src/fixtures/db.rs‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,18 +40,34 @@ thread_local! {
4040

4141
struct TestDatabases(RefCell<Vec<String>>);
4242

43+
fn drop_test_databases(db_names: Vec<String>) {
44+
if db_names.is_empty() {
45+
return;
46+
}
47+
// Wait for the drops, so that the process cannot exit before they finish.
48+
block_on(async_std::task::spawn(async move {
49+
for db_name in db_names {
50+
if let Err(err) = Postgres::cleanup_test(&db_name).await {
51+
// A panic in a destructor during unwind or TLS teardown aborts the process.
52+
eprintln!("could not drop test database {db_name}: {err}");
53+
}
54+
}
55+
}));
56+
}
57+
58+
/// Drops all temporary databases registered on the current test thread immediately rather than
59+
/// deferring to thread exit, so serialized tests do not overlap `DROP DATABASE` checkpoints with
60+
/// the next test case.
61+
pub(crate) fn cleanup_current_thread_databases() {
62+
let db_names = TEST_DATABASES
63+
.try_with(|databases| databases.0.take())
64+
.unwrap_or_default();
65+
drop_test_databases(db_names);
66+
}
67+
4368
impl Drop for TestDatabases {
4469
fn drop(&mut self) {
45-
let db_names = self.0.take();
46-
// Wait for the drops, so that the process cannot exit before they finish.
47-
block_on(async_std::task::spawn(async move {
48-
for db_name in db_names {
49-
if let Err(err) = Postgres::cleanup_test(&db_name).await {
50-
// A panic in a thread-local destructor aborts the process.
51-
eprintln!("could not drop test database {db_name}: {err}");
52-
}
53-
}
54-
}));
70+
drop_test_databases(self.0.take());
5571
}
5672
}
5773

‎tests/src/fixtures/mod.rs‎

Lines changed: 63 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,9 @@ pub mod utils;
2828

2929
use async_std::task::block_on;
3030
use rstest::*;
31-
use sqlx::{self, PgConnection};
32-
use std::sync::Once;
31+
use sqlx::{self, Connection, PgConnection};
32+
use std::ops::{Deref, DerefMut};
33+
use std::sync::{Mutex, MutexGuard, Once, PoisonError};
3334

3435
pub use crate::fixtures::backend::*;
3536
pub use crate::fixtures::db::*;
@@ -101,3 +102,63 @@ pub fn conn(database: Db) -> PgConnection {
101102
conn
102103
})
103104
}
105+
106+
/// Guard that serializes test cases within a single integration test binary and drops the
107+
/// current thread's temporary databases before releasing the mutex.
108+
///
109+
/// Bind this before any [`Db`] or [`PgConnection`] parameters (or use [`serial_conn`] for
110+
/// single-connection `#[rstest]` tests) so open connections close before `Drop` cleans up their
111+
/// databases and unlocks the mutex. Poisoning is ignored so one panicked case does not cascade
112+
/// to subsequent cases.
113+
pub struct SerialGuard(#[allow(dead_code)] MutexGuard<'static, ()>);
114+
115+
impl Drop for SerialGuard {
116+
fn drop(&mut self) {
117+
cleanup_current_thread_databases();
118+
}
119+
}
120+
121+
#[fixture]
122+
pub fn serial() -> SerialGuard {
123+
static SERIAL: Mutex<()> = Mutex::new(());
124+
SerialGuard(SERIAL.lock().unwrap_or_else(PoisonError::into_inner))
125+
}
126+
127+
/// A [`PgConnection`] fixture that serializes database creation, query execution, connection
128+
/// close, and `DROP DATABASE` teardown across cases in the current test binary.
129+
pub struct SerialConn {
130+
conn: Option<PgConnection>,
131+
_serial: SerialGuard,
132+
}
133+
134+
impl Deref for SerialConn {
135+
type Target = PgConnection;
136+
137+
fn deref(&self) -> &Self::Target {
138+
self.conn.as_ref().expect("SerialConn is open until Drop")
139+
}
140+
}
141+
142+
impl DerefMut for SerialConn {
143+
fn deref_mut(&mut self) -> &mut Self::Target {
144+
self.conn.as_mut().expect("SerialConn is open until Drop")
145+
}
146+
}
147+
148+
impl Drop for SerialConn {
149+
fn drop(&mut self) {
150+
if let Some(conn) = self.conn.take() {
151+
let _ = block_on(conn.close());
152+
}
153+
}
154+
}
155+
156+
#[fixture]
157+
pub fn serial_conn() -> SerialConn {
158+
let serial = serial();
159+
let conn = conn(database());
160+
SerialConn {
161+
conn: Some(conn),
162+
_serial: serial,
163+
}
164+
}

‎tests/src/fixtures/querygen/mod.rs‎

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -837,21 +837,13 @@ impl PgGucs {
837837
writeln!(gucs, "{verb} paradedb.min_rows_per_worker TO DEFAULT;").unwrap();
838838
writeln!(gucs, "{verb} paradedb.mpp_min_rows TO DEFAULT;").unwrap();
839839
}
840-
// In MPP distributed execution (with `paradedb.spill_to_disk` off by default),
841-
// `RepartitionExec` at the top of a shuffle producer stage creates `W * T` output partition
842-
// channels (`max_parallel_workers_per_gather * consumer_tasks`, up to 16 in qgen). Each
843-
// output partition's `8192`-row `LimitedBatchCoalescer` pre-allocates `16,384` slots per
844-
// column and does not shrink `Buffer::capacity()` on residual batches at EOF (~600 KB per
845-
// partition for multi-column grouping sets), which can exceed the 4MB default `work_mem`
846-
// across 8-16 partitions. Single-table MPP queries use a 16MB bound, while join-eligible
847-
// MPP queries use 32MB because DataFusion's `HashJoinExec` and `NestedLoopJoinExec` do not
848-
// spill to disk.
849-
if *parallel_workers {
850-
let limit = if *join_custom_scan { "32MB" } else { "16MB" };
851-
writeln!(gucs, "{verb} work_mem TO '{limit}';").unwrap();
852-
} else {
853-
writeln!(gucs, "{verb} work_mem TO DEFAULT;").unwrap();
854-
}
840+
// Lift `work_mem` so generated query shapes (including keyless joins with `pdb.agg()`
841+
// `cardinality` sketches and MPP shuffles) are not memory-constrained; spilling is
842+
// covered by dedicated parameterized tests instead.
843+
// TODO(https://github.com/apache/datafusion/issues/24768): Re-evaluate testing spilling
844+
// across generated query shapes once upstream DataFusion operators such as `HashJoinExec`
845+
// support spilling to disk.
846+
writeln!(gucs, "{verb} work_mem TO '1GB';").unwrap();
855847
writeln!(
856848
gucs,
857849
"{verb} statement_timeout TO {};",

‎tests/src/fixtures/querygen/pdbagggen.rs‎

Lines changed: 2 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -482,10 +482,6 @@ struct SpecShape {
482482
numeric_metrics: bool,
483483
/// A `size` may cut any level, not only a lone top level under no group.
484484
size_anywhere: bool,
485-
/// Bucket keys a spec with a `cardinality` metric may have. Every bucket
486-
/// carries its own sketch, and the DataFusion aggregate holds them all in
487-
/// `work_mem` with no spill.
488-
sketch_keys: usize,
489485
/// Array columns may be used as terms fields.
490486
allow_arrays: bool,
491487
}
@@ -516,9 +512,7 @@ impl PdbAggJoinWheres {
516512

517513
/// A join over a prefix of `tables` and a `pdb.agg()` whose fields belong to
518514
/// those tables. `key_columns` are the join keys; `terms` keys come from the
519-
/// text and integer columns, metrics from the integer and NUMERIC ones. A join
520-
/// with a keyless step yields tens of thousands of buckets under several keys,
521-
/// so `cardinality` keeps to one key there.
515+
/// text and integer columns, metrics from the integer and NUMERIC ones.
522516
///
523517
/// Generates `Inner` and `Left` joins. To prevent the PostgreSQL planner from
524518
/// eliminating outer-joined tables that aren't referenced elsewhere, a WHERE
@@ -544,11 +538,6 @@ pub fn arb_pdb_agg_join(
544538
allow_outer_aggs: true,
545539
numeric_metrics: true,
546540
size_anywhere: false,
547-
sketch_keys: if join.has_keyless_step() {
548-
1
549-
} else {
550-
usize::MAX
551-
},
552541
allow_arrays: true,
553542
};
554543
let agg = arb_pdb_agg(joined.clone(), join.unnested_fields(), shape);
@@ -582,19 +571,16 @@ pub fn arb_pdb_agg_single_table() -> impl Strategy<Value = PdbAggExpr> {
582571
allow_outer_aggs: false,
583572
numeric_metrics: false,
584573
size_anywhere: true,
585-
sketch_keys: usize::MAX,
586574
allow_arrays: false,
587575
},
588576
)
589577
}
590578

591579
/// A `pdb.agg()` for `pdb.agg(...) OVER ()` beside the rows of a join over
592580
/// `tables`. A window has no SQL group or aggregates beside it.
593-
/// `has_keyless_step` bounds `cardinality` and `unnested` names the arrays the
594-
/// join unnests itself, both as in [`arb_pdb_agg_join`].
581+
/// `unnested` names the arrays the join unnests itself, as in [`arb_pdb_agg_join`].
595582
pub fn arb_pdb_agg_window(
596583
tables: Vec<String>,
597-
has_keyless_step: bool,
598584
unnested: Vec<(String, String)>,
599585
) -> impl Strategy<Value = PdbAggExpr> {
600586
arb_pdb_agg(
@@ -605,7 +591,6 @@ pub fn arb_pdb_agg_window(
605591
allow_outer_aggs: false,
606592
numeric_metrics: true,
607593
size_anywhere: false,
608-
sketch_keys: if has_keyless_step { 1 } else { usize::MAX },
609594
allow_arrays: true,
610595
},
611596
)
@@ -795,13 +780,6 @@ fn arb_pdb_agg(
795780
});
796781
}
797782
}
798-
if usize::from(outer_group.is_some()) + terms.len() > shape.sketch_keys {
799-
for metric in &mut metrics {
800-
if metric.kind == MetricKind::Cardinality {
801-
metric.kind = MetricKind::ValueCount;
802-
}
803-
}
804-
}
805783
let has_bucket_outer_group = outer_group.is_some() && !combine_across_groups;
806784
let size = size.filter(|&(level, _)| {
807785
level < terms.len()
@@ -828,44 +806,6 @@ mod tests {
828806
use proptest::strategy::{Strategy, ValueTree};
829807
use proptest::test_runner::TestRunner;
830808

831-
#[test]
832-
fn cardinality_keeps_to_one_key_over_a_keyless_join() {
833-
let mut runner = TestRunner::default();
834-
let tables = vec![
835-
"users".to_string(),
836-
"products".to_string(),
837-
"orders".to_string(),
838-
];
839-
let key_columns = vec![
840-
Column::new("id", "BIGINT", "1"),
841-
Column::new("age", "INTEGER", "20"),
842-
];
843-
let where_columns = vec![
844-
Column::new("name", "TEXT", "'bob'").whereable(true),
845-
Column::new("color", "VARCHAR", "'blue'").whereable(true),
846-
];
847-
let strategy = arb_pdb_agg_join(tables, &key_columns, &where_columns);
848-
849-
let mut saw_sketch_over_keyless = false;
850-
let mut saw_sketch_under_keys = false;
851-
for _ in 0..500 {
852-
let (join, agg, _wheres) = strategy.new_tree(&mut runner).unwrap().current();
853-
let keys = usize::from(agg.outer_group.is_some()) + agg.terms.len();
854-
let sketch = agg
855-
.metrics
856-
.iter()
857-
.any(|m| m.kind == MetricKind::Cardinality);
858-
if join.has_keyless_step() {
859-
assert!(!(sketch && keys > 1), "{join:?} {agg:?}");
860-
saw_sketch_over_keyless |= sketch;
861-
} else {
862-
saw_sketch_under_keys |= sketch && keys > 1;
863-
}
864-
}
865-
assert!(saw_sketch_over_keyless);
866-
assert!(saw_sketch_under_keys);
867-
}
868-
869809
#[test]
870810
fn test_arb_pdb_agg_join_generates_left_joins_and_multi_table_wheres() {
871811
let mut runner = TestRunner::default();

‎tests/tests/qgen.rs‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,6 @@ async fn generated_joins_small(database: Db, #[case] churn_on: bool) {
331331
.collect::<Vec<_>>();
332332
let pdb_agg = proptest::option::of(arb_pdb_agg_window(
333333
used_tables.clone(),
334-
join.has_keyless_step(),
335334
join.unnested_fields(),
336335
));
337336
(
@@ -488,7 +487,6 @@ async fn generated_joins_small(database: Db, #[case] churn_on: bool) {
488487
&pool,
489488
&setup_sql,
490489
|query, side, conn| {
491-
"SET work_mem TO '16MB';".execute_result(conn)?;
492490
let rows = query.fetch_dynamic_result(conn)?;
493491
// The document on the candidate's rows against the buckets of the
494492
// PostgreSQL `GROUP BY`. With no row there is no document to compare.
@@ -1532,7 +1530,6 @@ async fn generated_pdb_agg_join(database: Db) {
15321530
&pool,
15331531
&setup_sql,
15341532
|query, side, conn| {
1535-
"SET work_mem TO '64MB';".execute_result(conn)?;
15361533
let rows = query.fetch_dynamic_result(conn)?;
15371534
let mut outer = agg.outer_rows(rows, side.is_candidate())?;
15381535
if agg.group_limit.is_none() {
@@ -1558,9 +1555,6 @@ async fn generated_pdb_agg_join(database: Db) {
15581555
&pool,
15591556
&setup_sql,
15601557
|query, side, conn| {
1561-
// A keyless join under three bucket keys makes tens of thousands of
1562-
// buckets, and the DataFusion aggregate cannot spill past `work_mem`.
1563-
"SET work_mem TO '64MB';".execute_result(conn)?;
15641558
let rows = query.fetch_dynamic_result(conn)?;
15651559
let mut buckets = if agg.combine_across_groups {
15661560
if rows.is_empty() {

0 commit comments

Comments
 (0)