Skip to content

Commit c3ea1ce

Browse files
Simplify the clustered test helpers and drop the partial clustering tests
1 parent baa8fe5 commit c3ea1ce

4 files changed

Lines changed: 24 additions & 64 deletions

File tree

‎datafusion/physical-plan/src/aggregates/clustered_final_stream.rs‎

Lines changed: 4 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -474,7 +474,7 @@ mod tests {
474474
use crate::aggregates::clustered_test_util::{
475475
assert_completed_groups_emitted_under_limit,
476476
assert_resources_exhausted_under_limit, full_batches, held_memory,
477-
partial_batches, single_key_batches, sum_over_sorted_input,
477+
single_key_batches, sum_over_sorted_input,
478478
};
479479
use crate::common::collect;
480480
use crate::test::TestMemoryExec;
@@ -751,31 +751,18 @@ mod tests {
751751

752752
#[tokio::test]
753753
async fn fully_clustered_completed_groups_under_memory_pressure() -> Result<()> {
754-
let (plan, mode) =
755-
sum_over_sorted_input(AggregateMode::Final, &["k"], full_batches())?;
756-
assert_eq!(mode, GroupClusteringMode::Full);
754+
let plan = sum_over_sorted_input(AggregateMode::Final, &["k"], full_batches())?;
755+
assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);
757756
assert_completed_groups_emitted_under_limit(&plan).await
758757
}
759758

760759
/// A single key never completes a group before the input ends. The limit
761760
/// fits only the initial table.
762761
#[tokio::test]
763762
async fn memory_pressure_without_completed_groups() -> Result<()> {
764-
let (plan, _) =
763+
let plan =
765764
sum_over_sorted_input(AggregateMode::Final, &["k"], single_key_batches())?;
766765
let limit = held_memory(&plan).await?;
767766
assert_resources_exhausted_under_limit(&plan, limit).await
768767
}
769-
770-
/// Without temporary files, partial group clustering takes the same path as
771-
/// full clustering. Its group values keep their capacity after emitting, so
772-
/// one byte below the unlimited footprint still fails.
773-
#[tokio::test]
774-
async fn partially_clustered_memory_pressure_without_spilling() -> Result<()> {
775-
let (plan, mode) =
776-
sum_over_sorted_input(AggregateMode::Final, &["a", "b"], partial_batches())?;
777-
assert_eq!(mode, GroupClusteringMode::Partial(vec![0]));
778-
let limit = held_memory(&plan).await? - 1;
779-
assert_resources_exhausted_under_limit(&plan, limit).await
780-
}
781768
}

‎datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -429,9 +429,8 @@ mod tests {
429429

430430
#[tokio::test]
431431
async fn fully_clustered_completed_groups_under_memory_pressure() -> Result<()> {
432-
let (plan, mode) =
433-
sum_over_sorted_input(AggregateMode::Partial, &["k"], full_batches())?;
434-
assert_eq!(mode, GroupClusteringMode::Full);
432+
let plan = sum_over_sorted_input(AggregateMode::Partial, &["k"], full_batches())?;
433+
assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);
435434
assert_completed_groups_emitted_under_limit(&plan).await
436435
}
437436
}

‎datafusion/physical-plan/src/aggregates/clustered_single_stream.rs‎

Lines changed: 4 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -772,7 +772,7 @@ mod tests {
772772
use crate::aggregates::clustered_test_util::{
773773
assert_completed_groups_emitted_under_limit,
774774
assert_resources_exhausted_under_limit, full_batches, held_memory,
775-
partial_batches, single_key_batches, sum_over_sorted_input,
775+
single_key_batches, sum_over_sorted_input,
776776
};
777777
use crate::collect;
778778
use crate::test::TestMemoryExec;
@@ -866,31 +866,18 @@ mod tests {
866866

867867
#[tokio::test]
868868
async fn fully_clustered_completed_groups_under_memory_pressure() -> Result<()> {
869-
let (plan, mode) =
870-
sum_over_sorted_input(AggregateMode::Single, &["k"], full_batches())?;
871-
assert_eq!(mode, GroupClusteringMode::Full);
869+
let plan = sum_over_sorted_input(AggregateMode::Single, &["k"], full_batches())?;
870+
assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);
872871
assert_completed_groups_emitted_under_limit(&plan).await
873872
}
874873

875874
/// A single key never completes a group before the input ends. The limit
876875
/// fits only the initial table.
877876
#[tokio::test]
878877
async fn memory_pressure_without_completed_groups() -> Result<()> {
879-
let (plan, _) =
878+
let plan =
880879
sum_over_sorted_input(AggregateMode::Single, &["k"], single_key_batches())?;
881880
let limit = held_memory(&plan).await?;
882881
assert_resources_exhausted_under_limit(&plan, limit).await
883882
}
884-
885-
/// Without temporary files, partial group clustering takes the same path as
886-
/// full clustering. Its group values keep their capacity after emitting, so
887-
/// one byte below the unlimited footprint still fails.
888-
#[tokio::test]
889-
async fn partially_clustered_memory_pressure_without_spilling() -> Result<()> {
890-
let (plan, mode) =
891-
sum_over_sorted_input(AggregateMode::Single, &["a", "b"], partial_batches())?;
892-
assert_eq!(mode, GroupClusteringMode::Partial(vec![0]));
893-
let limit = held_memory(&plan).await? - 1;
894-
assert_resources_exhausted_under_limit(&plan, limit).await
895-
}
896883
}

‎datafusion/physical-plan/src/aggregates/clustered_test_util.rs‎

Lines changed: 14 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
2020
use std::sync::Arc;
2121

22-
use arrow::array::Int64Array;
22+
use arrow::array::{ArrayRef, Int64Array};
2323
use arrow::datatypes::{DataType, Field, Schema};
2424
use arrow::record_batch::RecordBatch;
2525
use datafusion_common::test_util::batches_to_string;
@@ -38,7 +38,7 @@ use datafusion_physical_expr::expressions::col;
3838
use datafusion_physical_expr_common::sort_expr::LexOrdering;
3939
use futures::StreamExt;
4040

41-
use super::{AggregateExec, AggregateMode, GroupClusteringMode, PhysicalGroupBy};
41+
use super::{AggregateExec, AggregateMode, PhysicalGroupBy};
4242
use crate::ExecutionPlan;
4343
use crate::test::TestMemoryExec;
4444

@@ -48,16 +48,6 @@ pub(super) fn full_batches() -> Vec<Vec<Vec<i64>>> {
4848
vec![vec![(0..=256).collect()], vec![(256..=512).collect()]]
4949
}
5050

51-
/// Key columns `(a, b)` of batches sorted on `a` only, with 256 groups for each
52-
/// `a` spread over two batches. The groups of one `a` complete together when the
53-
/// next `a` starts.
54-
pub(super) fn partial_batches() -> Vec<Vec<Vec<i64>>> {
55-
[0, 0, 1, 1, 2, 2]
56-
.into_iter()
57-
.map(|a| vec![vec![a; 256], (0..256).collect()])
58-
.collect()
59-
}
60-
6151
/// Key column of two batches that hold the same single key, so no group
6252
/// completes before the input ends.
6353
pub(super) fn single_key_batches() -> Vec<Vec<Vec<i64>>> {
@@ -75,7 +65,7 @@ pub(super) fn sum_over_sorted_input(
7565
mode: AggregateMode,
7666
keys: &[&str],
7767
batches: Vec<Vec<Vec<i64>>>,
78-
) -> Result<(Arc<dyn ExecutionPlan>, GroupClusteringMode)> {
68+
) -> Result<AggregateExec> {
7969
let fields = keys
8070
.iter()
8171
.chain(&["v"])
@@ -112,12 +102,11 @@ pub(super) fn sum_over_sorted_input(
112102
let batches = batches
113103
.into_iter()
114104
.map(|columns| {
115-
let v = columns.last().cloned().unwrap_or_default();
116-
let columns = columns
105+
let mut columns = columns
117106
.into_iter()
118-
.chain([v])
119-
.map(|values| Arc::new(Int64Array::from(values)) as _)
120-
.collect();
107+
.map(|values| Arc::new(Int64Array::from(values)) as ArrayRef)
108+
.collect::<Vec<_>>();
109+
columns.push(Arc::clone(columns.last().unwrap()));
121110
RecordBatch::try_new(Arc::clone(&input_schema), columns)
122111
})
123112
.collect::<Result<Vec<_>, _>>()?;
@@ -126,23 +115,21 @@ pub(super) fn sum_over_sorted_input(
126115
.unwrap();
127116
let input = TestMemoryExec::try_new(&[batches], Arc::clone(&input_schema), None)?
128117
.try_with_sort_information(vec![ordering])?;
129-
let aggregate = AggregateExec::try_new(
118+
AggregateExec::try_new(
130119
mode,
131120
group_by,
132121
aggregate_exprs,
133122
vec![None],
134123
Arc::new(TestMemoryExec::update_cache(&Arc::new(input))),
135124
schema,
136-
)?;
137-
let group_clustering_mode = aggregate.group_clustering_mode().clone();
138-
Ok((Arc::new(aggregate), group_clustering_mode))
125+
)
139126
}
140127

141128
/// Executes partition 0 of `plan` on the clustered aggregate path, with `pool`
142129
/// and temporary files disabled. Returns the output and the most memory the
143130
/// plan held when it handed off a batch, including its initial reservation.
144-
pub(super) async fn collect_with_held_memory(
145-
plan: &Arc<dyn ExecutionPlan>,
131+
async fn collect_with_held_memory(
132+
plan: &AggregateExec,
146133
pool: Arc<dyn MemoryPool>,
147134
) -> Result<(Vec<RecordBatch>, usize)> {
148135
let context = Arc::new(
@@ -174,7 +161,7 @@ pub(super) async fn collect_with_held_memory(
174161
}
175162

176163
/// Returns the memory `plan` holds when handing off output without a limit.
177-
pub(super) async fn held_memory(plan: &Arc<dyn ExecutionPlan>) -> Result<usize> {
164+
pub(super) async fn held_memory(plan: &AggregateExec) -> Result<usize> {
178165
let (_, held) =
179166
collect_with_held_memory(plan, Arc::new(UnboundedMemoryPool::default())).await?;
180167
Ok(held)
@@ -184,7 +171,7 @@ pub(super) async fn held_memory(plan: &Arc<dyn ExecutionPlan>) -> Result<usize>
184171
/// with only the active groups, but not the table before completed groups are
185172
/// emitted. The result must not change.
186173
pub(super) async fn assert_completed_groups_emitted_under_limit(
187-
plan: &Arc<dyn ExecutionPlan>,
174+
plan: &AggregateExec,
188175
) -> Result<()> {
189176
let (expected, held) =
190177
collect_with_held_memory(plan, Arc::new(UnboundedMemoryPool::default())).await?;
@@ -197,7 +184,7 @@ pub(super) async fn assert_completed_groups_emitted_under_limit(
197184
/// Runs `plan` under `limit`, where emitting completed groups cannot bring the
198185
/// table under the limit, so the refused reservation must be returned.
199186
pub(super) async fn assert_resources_exhausted_under_limit(
200-
plan: &Arc<dyn ExecutionPlan>,
187+
plan: &AggregateExec,
201188
limit: usize,
202189
) -> Result<()> {
203190
let result =

0 commit comments

Comments
 (0)