Skip to content

Commit 2bf86aa

Browse files
fix: keep required sorts above custom execution plans
Keep sorts above operators without explicit sort-pushdown handling. Preserve known built-in input mappings and make custom-node support opt-in through the existing try_pushdown_sort API. Cover correct query results and sort placement for a value-transforming custom projection, built-in input mappings, and explicit custom support. Closes apache#26656.
1 parent 350ca76 commit 2bf86aa

8 files changed

Lines changed: 288 additions & 202 deletions

File tree

‎datafusion/core/tests/physical_optimizer/enforce_sorting.rs‎

Lines changed: 3 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2604,9 +2604,9 @@ async fn test_push_with_required_input_ordering_prohibited() -> Result<()> {
26042604
Ok(())
26052605
}
26062606

2607-
// test when the required input ordering is satisfied so could push through
2607+
// Compatible input requirements do not opt a custom node into sort pushdown.
26082608
#[tokio::test]
2609-
async fn test_push_with_required_input_ordering_allowed() -> Result<()> {
2609+
async fn test_custom_node_keeps_sort_with_compatible_input_ordering() -> Result<()> {
26102610
let schema = create_test_schema3()?;
26112611
let ordering_a: LexOrdering = [sort_expr("a", &schema)].into();
26122612
let ordering_ab = [sort_expr("a", &schema), sort_expr("b", &schema)].into();
@@ -2618,29 +2618,15 @@ async fn test_push_with_required_input_ordering_allowed() -> Result<()> {
26182618
.into_arc();
26192619
let plan = sort_exec(ordering_ab, plan);
26202620

2621-
/*
2622-
let expected_input = [
2623-
"SortExec: expr=[a@0 ASC, b@1 ASC], preserve_partitioning=[false]", // <-- can push this down (as it is compatible with the required input ordering)
2624-
" RequiredInputOrderingExec", // <-- this requires input sorted by a, and preserves the input order
2625-
" SortExec: expr=[a@0 ASC], preserve_partitioning=[false]",
2626-
" DataSourceExec: partitions=1, partition_sizes=[0]",
2627-
];
2628-
*/
26292621
let test = EnforceSortingTest::new(plan.clone()).with_repartition_sorts(true);
26302622

26312623
assert_snapshot!(test.run(), @r"
2632-
Input Plan:
2624+
Input / Optimized Plan:
26332625
SortExec: expr=[a@0 ASC, b@1 ASC], preserve_partitioning=[false]
26342626
RequiredInputOrderingExec
26352627
SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
26362628
DataSourceExec: partitions=1, partition_sizes=[0]
2637-
2638-
Optimized Plan:
2639-
RequiredInputOrderingExec
2640-
SortExec: expr=[a@0 ASC, b@1 ASC], preserve_partitioning=[false]
2641-
DataSourceExec: partitions=1, partition_sizes=[0]
26422629
");
2643-
// Should be able to push down
26442630
Ok(())
26452631
}
26462632

‎datafusion/core/tests/physical_optimizer/pushdown_sort.rs‎

Lines changed: 36 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,10 @@ use datafusion_physical_expr::expressions;
3434
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
3535
use datafusion_physical_expr_common::sort_expr::LexOrdering;
3636
use datafusion_physical_optimizer::PhysicalOptimizerRule;
37+
use datafusion_physical_optimizer::ensure_requirements::EnsureRequirements;
3738
use datafusion_physical_optimizer::pushdown_sort::PushdownSort;
38-
use datafusion_physical_plan::collect;
39+
use datafusion_physical_plan::sorts::sort::SortExec;
40+
use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties, collect};
3941
use std::sync::Arc;
4042

4143
use crate::physical_optimizer::test_utils::{
@@ -1111,30 +1113,47 @@ fn test_sort_pushdown_with_test_scan_arbitrary_ordering() {
11111113
// ============================================================================
11121114

11131115
#[test]
1114-
fn test_sort_pushdown_exact_no_fetch_no_limit() {
1115-
// When a source returns Exact (without fetch), the SortExec should be
1116-
// removed entirely with no GlobalLimitExec wrapper.
1116+
fn test_custom_plan_can_opt_into_sort_pushdown() -> Result<()> {
11171117
let schema = schema();
11181118
let a = sort_expr("a", &schema);
11191119
let b = sort_expr("b", &schema);
11201120
let source =
11211121
Arc::new(TestScan::new(schema.clone(), vec![]).with_exact_pushdown(true));
11221122

11231123
let ordering = LexOrdering::new(vec![a, b.reverse()]).unwrap();
1124-
let plan = sort_exec(ordering, source);
1124+
let plan = sort_exec(ordering.clone(), source);
1125+
let mut config = ConfigOptions::new();
1126+
config.execution.target_partitions = 1;
1127+
let plan = EnsureRequirements::new().optimize(plan, &config)?;
1128+
assert!(plan.is::<SortExec>());
11251129

1126-
insta::assert_snapshot!(
1127-
OptimizationTest::new(plan, PushdownSort::new(), true),
1128-
@r"
1129-
OptimizationTest:
1130-
input:
1131-
- SortExec: expr=[a@0 ASC, b@1 DESC NULLS LAST], preserve_partitioning=[false]
1132-
- TestScan
1133-
output:
1134-
Ok:
1135-
- TestScan: requested_ordering=[a@0 ASC, b@1 DESC NULLS LAST]
1136-
"
1137-
);
1130+
let optimized = PushdownSort::new().optimize(plan, &config)?;
1131+
assert!(optimized.is::<TestScan>());
1132+
assert_eq!(optimized.output_ordering(), Some(&ordering));
1133+
Ok(())
1134+
}
1135+
1136+
#[test]
1137+
fn test_custom_plan_with_fetch_keeps_outer_sort() -> Result<()> {
1138+
let schema = schema();
1139+
let a = sort_expr("a", &schema);
1140+
let source = TestScan::new(schema.clone(), vec![[a.clone()].into()])
1141+
.with_supports_fetch(true)
1142+
.with_fetch(Some(10))
1143+
.unwrap();
1144+
let ordering = LexOrdering::new(vec![a, sort_expr("b", &schema)]).unwrap();
1145+
let plan = sort_exec(ordering.clone(), source);
1146+
1147+
let mut config = ConfigOptions::new();
1148+
config.execution.target_partitions = 1;
1149+
let optimized = EnsureRequirements::new().optimize(plan, &config)?;
1150+
let sort = optimized
1151+
.downcast_ref::<SortExec>()
1152+
.expect("a fetched custom plan must retain the requested outer sort");
1153+
assert_eq!(sort.expr(), &ordering);
1154+
assert!(sort.input().is::<TestScan>());
1155+
assert_eq!(sort.input().fetch(), Some(10));
1156+
Ok(())
11381157
}
11391158

11401159
#[test]

‎datafusion/core/tests/physical_optimizer/test_utils.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1074,6 +1074,10 @@ impl ExecutionPlan for TestScan {
10741074
Ok(Arc::new(Statistics::new_unknown(&self.schema)))
10751075
}
10761076

1077+
fn supports_limit_pushdown(&self) -> bool {
1078+
self.supports_fetch
1079+
}
1080+
10771081
fn with_fetch(&self, fetch: Option<usize>) -> Option<Arc<dyn ExecutionPlan>> {
10781082
if self.supports_fetch {
10791083
let mut new_scan = self.clone();

0 commit comments

Comments
 (0)