Skip to content

Commit 426b351

Browse files
Add config-matrix tests in enforce_distribution.rs for range-satisfaction settings (#23627)
## Which issue does this PR close? - Closes #23572 - Part of #22395 ## Rationale for this change Range partitioning reuse is controlled by shared optimizer settings, but the same configuration combinations were repeated across operator-specific SQL logic tests. This PR moves that shared coverage into one table-driven Rust test while retaining the operator-specific cases that test separate planning behavior. ## What changes are included in this PR? - Extends `RequirementsTestExec` with a configurable input distribution. - Adds a 12-row configuration matrix covering 36 combinations of key compatibility, subset threshold, preserve-file setting, and target partition count. - Requires reuse cases to contain no repartition and Hash cases to contain exactly one Hash repartition. - Removes three redundant aggregate SLT cases and renumbers the remaining tests. ## Are these changes tested? Yes. The 36-cell matrix and `range_partitioning.slt` pass. ## Are there any user-facing changes? No. These changes only affect tests and test utilities. --------- Co-authored-by: blinding-pixels <[email protected]>
1 parent 2dd1a14 commit 426b351

3 files changed

Lines changed: 157 additions & 135 deletions

File tree

‎datafusion/core/tests/physical_optimizer/enforce_distribution.rs‎

Lines changed: 91 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,10 @@ use std::ops::Deref;
2020
use std::sync::Arc;
2121

2222
use crate::physical_optimizer::test_utils::{
23-
bounded_window_exec_with_can_repartition, check_integrity, coalesce_partitions_exec,
24-
parquet_exec_with_sort, parquet_exec_with_stats, repartition_exec, schema, sort_exec,
25-
sort_exec_with_preserve_partitioning, sort_merge_join_exec,
26-
sort_preserving_merge_exec, union_exec,
23+
RequirementsTestExec, bounded_window_exec_with_can_repartition, check_integrity,
24+
coalesce_partitions_exec, parquet_exec_with_sort, parquet_exec_with_stats,
25+
repartition_exec, schema, sort_exec, sort_exec_with_preserve_partitioning,
26+
sort_merge_join_exec, sort_preserving_merge_exec, union_exec,
2727
};
2828

2929
use arrow::array::{RecordBatch, UInt8Array, UInt64Array};
@@ -747,6 +747,93 @@ impl TestConfig {
747747
}
748748
}
749749

750+
#[derive(Debug, Clone, Copy)]
751+
enum ExpectedPlan {
752+
Reuse,
753+
Hash,
754+
}
755+
756+
#[test]
757+
fn range_satisfaction_config_matrix() -> Result<()> {
758+
const INPUT_PARTITIONS: usize = 4;
759+
const MET: usize = INPUT_PARTITIONS;
760+
const NOT_MET: usize = INPUT_PARTITIONS + 1;
761+
const DISABLED: usize = 0;
762+
const EQUAL: usize = INPUT_PARTITIONS;
763+
const GREATER: usize = INPUT_PARTITIONS + 1;
764+
use ExpectedPlan::{Hash, Reuse};
765+
766+
let config_cases = [
767+
// subset preserve target exact subset incompatible
768+
(NOT_MET, DISABLED, EQUAL, [Reuse, Hash, Hash]),
769+
(NOT_MET, DISABLED, GREATER, [Hash, Hash, Hash]),
770+
(NOT_MET, NOT_MET, EQUAL, [Reuse, Hash, Hash]),
771+
(NOT_MET, NOT_MET, GREATER, [Hash, Hash, Hash]),
772+
(NOT_MET, MET, EQUAL, [Reuse, Hash, Hash]),
773+
(NOT_MET, MET, GREATER, [Reuse, Reuse, Hash]),
774+
(MET, DISABLED, EQUAL, [Reuse, Reuse, Hash]),
775+
(MET, DISABLED, GREATER, [Reuse, Reuse, Hash]),
776+
(MET, NOT_MET, EQUAL, [Reuse, Reuse, Hash]),
777+
(MET, NOT_MET, GREATER, [Reuse, Reuse, Hash]),
778+
(MET, MET, EQUAL, [Reuse, Reuse, Hash]),
779+
(MET, MET, GREATER, [Reuse, Reuse, Hash]),
780+
];
781+
for (subset_threshold, preserve_file_partitions, target_partitions, expected) in
782+
config_cases
783+
{
784+
let key_cases = [
785+
("exact", vec![col("a", &schema())?], expected[0]),
786+
(
787+
"subset",
788+
vec![col("a", &schema())?, col("b", &schema())?],
789+
expected[1],
790+
),
791+
("incompatible", vec![col("b", &schema())?], expected[2]),
792+
];
793+
for (key_match, partition_keys, expected_plan) in key_cases {
794+
let input = parquet_exec_with_output_partitioning(range_partitioning(
795+
"a",
796+
[10, 20, 30],
797+
SortOptions::default(),
798+
)?);
799+
let requirement = RequirementsTestExec::new(input)
800+
.with_required_input_distribution(Distribution::KeyPartitioned(
801+
partition_keys,
802+
))
803+
.into_arc();
804+
805+
let mut config =
806+
TestConfig::default().with_query_execution_partitions(target_partitions);
807+
config.config.optimizer.subset_repartition_threshold = subset_threshold;
808+
config.config.optimizer.preserve_file_partitions = preserve_file_partitions;
809+
810+
let plan = config.to_plan(requirement, &DISTRIB_DISTRIB_SORT);
811+
let plan = displayable(plan.as_ref()).indent(true).to_string();
812+
let repartitions = plan
813+
.lines()
814+
.filter(|line| line.contains("RepartitionExec:"))
815+
.collect::<Vec<_>>();
816+
817+
let matches_expected = match expected_plan {
818+
Reuse => repartitions.is_empty(),
819+
Hash => matches!(
820+
repartitions.as_slice(),
821+
[repartition] if repartition.contains("partitioning=Hash")
822+
),
823+
};
824+
assert!(
825+
matches_expected,
826+
"unexpected optimized plan for key_match={key_match}, \
827+
subset_threshold={subset_threshold}, \
828+
preserve_file_partitions={preserve_file_partitions}, \
829+
target_partitions={target_partitions}:\n{plan}"
830+
);
831+
}
832+
}
833+
834+
Ok(())
835+
}
836+
750837
#[test]
751838
fn range_aggregate_reuses_range_partitioning() -> Result<()> {
752839
let input = parquet_exec_with_output_partitioning(range_partitioning(

‎datafusion/core/tests/physical_optimizer/test_utils.rs‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,10 @@ use datafusion_execution::object_store::ObjectStoreUrl;
4040
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
4141
use datafusion_expr::{WindowFrame, WindowFunctionDefinition};
4242
use datafusion_functions_aggregate::count::count_udaf;
43-
use datafusion_physical_expr::EquivalenceProperties;
4443
use datafusion_physical_expr::aggregate::{AggregateExprBuilder, AggregateFunctionExpr};
4544
use datafusion_physical_expr::expressions::{self, col};
4645
use datafusion_physical_expr::projection::ProjectionExprs;
46+
use datafusion_physical_expr::{Distribution, EquivalenceProperties};
4747
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
4848
use datafusion_physical_expr_common::sort_expr::{
4949
LexOrdering, OrderingRequirements, PhysicalSortExpr,
@@ -68,8 +68,9 @@ use datafusion_physical_plan::tree_node::PlanContext;
6868
use datafusion_physical_plan::union::UnionExec;
6969
use datafusion_physical_plan::windows::{BoundedWindowAggExec, create_window_expr};
7070
use datafusion_physical_plan::{
71-
DisplayAs, DisplayFormatType, ExecutionPlan, InputOrderMode, Partitioning,
72-
PlanProperties, SortOrderPushdownResult, StatisticsArgs, displayable,
71+
DisplayAs, DisplayFormatType, ExecutionPlan, InputDistributionRequirements,
72+
InputOrderMode, Partitioning, PlanProperties, SortOrderPushdownResult,
73+
StatisticsArgs, displayable,
7374
};
7475

7576
/// Create a non sorted parquet exec
@@ -435,6 +436,7 @@ pub fn projection_exec(
435436
#[derive(Debug)]
436437
pub struct RequirementsTestExec {
437438
required_input_ordering: Option<LexOrdering>,
439+
required_input_distribution: Distribution,
438440
maintains_input_order: bool,
439441
input: Arc<dyn ExecutionPlan>,
440442
}
@@ -443,6 +445,7 @@ impl RequirementsTestExec {
443445
pub fn new(input: Arc<dyn ExecutionPlan>) -> Self {
444446
Self {
445447
required_input_ordering: None,
448+
required_input_distribution: Distribution::UnspecifiedDistribution,
446449
maintains_input_order: true,
447450
input,
448451
}
@@ -457,6 +460,15 @@ impl RequirementsTestExec {
457460
self
458461
}
459462

463+
/// sets the required input distribution
464+
pub fn with_required_input_distribution(
465+
mut self,
466+
required_input_distribution: Distribution,
467+
) -> Self {
468+
self.required_input_distribution = required_input_distribution;
469+
self
470+
}
471+
460472
/// set the maintains_input_order flag
461473
pub fn with_maintains_input_order(mut self, maintains_input_order: bool) -> Self {
462474
self.maintains_input_order = maintains_input_order;
@@ -500,6 +512,10 @@ impl ExecutionPlan for RequirementsTestExec {
500512
]
501513
}
502514

515+
fn input_distribution_requirements(&self) -> InputDistributionRequirements {
516+
InputDistributionRequirements::new(vec![self.required_input_distribution.clone()])
517+
}
518+
503519
fn maintains_input_order(&self) -> Vec<bool> {
504520
vec![self.maintains_input_order]
505521
}
@@ -515,6 +531,7 @@ impl ExecutionPlan for RequirementsTestExec {
515531
assert_eq!(children.len(), 1);
516532
Ok(RequirementsTestExec::new(Arc::clone(&children[0]))
517533
.with_required_input_ordering(self.required_input_ordering.clone())
534+
.with_required_input_distribution(self.required_input_distribution.clone())
518535
.with_maintains_input_order(self.maintains_input_order)
519536
.into_arc())
520537
}

0 commit comments

Comments
 (0)