Skip to content

Commit a95754f

Browse files
fix: keep required sorts above custom execution plans
Keep sorts above operators without explicit sort-pushdown handling. Preserve known built-in input mappings and make custom-node support opt-in through the existing try_pushdown_sort API. Cover correct query results and sort placement for a value-transforming custom projection, built-in input mappings, and explicit custom support. Closes apache#26656.
1 parent e8940ac commit a95754f

8 files changed

Lines changed: 288 additions & 202 deletions

File tree

‎datafusion/core/tests/physical_optimizer/enforce_sorting.rs‎

Lines changed: 3 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2735,9 +2735,9 @@ async fn test_push_with_required_input_ordering_prohibited() -> Result<()> {
27352735
Ok(())
27362736
}
27372737

2738-
// test when the required input ordering is satisfied so could push through
2738+
// Compatible input requirements do not opt a custom node into sort pushdown.
27392739
#[tokio::test]
2740-
async fn test_push_with_required_input_ordering_allowed() -> Result<()> {
2740+
async fn test_custom_node_keeps_sort_with_compatible_input_ordering() -> Result<()> {
27412741
let schema = create_test_schema3()?;
27422742
let ordering_a: LexOrdering = [sort_expr("a", &schema)].into();
27432743
let ordering_ab = [sort_expr("a", &schema), sort_expr("b", &schema)].into();
@@ -2749,29 +2749,15 @@ async fn test_push_with_required_input_ordering_allowed() -> Result<()> {
27492749
.into_arc();
27502750
let plan = sort_exec(ordering_ab, plan);
27512751

2752-
/*
2753-
let expected_input = [
2754-
"SortExec: expr=[a@0 ASC, b@1 ASC], preserve_partitioning=[false]", // <-- can push this down (as it is compatible with the required input ordering)
2755-
" RequiredInputOrderingExec", // <-- this requires input sorted by a, and preserves the input order
2756-
" SortExec: expr=[a@0 ASC], preserve_partitioning=[false]",
2757-
" DataSourceExec: partitions=1, partition_sizes=[0]",
2758-
];
2759-
*/
27602752
let test = EnforceSortingTest::new(plan.clone()).with_repartition_sorts(true);
27612753

27622754
assert_snapshot!(test.run(), @r"
2763-
Input Plan:
2755+
Input / Optimized Plan:
27642756
SortExec: expr=[a@0 ASC, b@1 ASC], preserve_partitioning=[false]
27652757
RequiredInputOrderingExec
27662758
SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
27672759
DataSourceExec: partitions=1, partition_sizes=[0]
2768-
2769-
Optimized Plan:
2770-
RequiredInputOrderingExec
2771-
SortExec: expr=[a@0 ASC, b@1 ASC], preserve_partitioning=[false]
2772-
DataSourceExec: partitions=1, partition_sizes=[0]
27732760
");
2774-
// Should be able to push down
27752761
Ok(())
27762762
}
27772763

‎datafusion/core/tests/physical_optimizer/pushdown_sort.rs‎

Lines changed: 36 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@ use datafusion_physical_expr::expressions;
2121
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
2222
use datafusion_physical_expr_common::sort_expr::LexOrdering;
2323
use datafusion_physical_optimizer::PhysicalOptimizerRule;
24+
use datafusion_physical_optimizer::ensure_requirements::EnsureRequirements;
2425
use datafusion_physical_optimizer::pushdown_sort::PushdownSort;
25-
use datafusion_physical_plan::collect;
26+
use datafusion_physical_plan::sorts::sort::SortExec;
27+
use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties, collect};
2628
use std::sync::Arc;
2729

2830
use crate::physical_optimizer::test_utils::{
@@ -1098,30 +1100,47 @@ fn test_sort_pushdown_with_test_scan_arbitrary_ordering() {
10981100
// ============================================================================
10991101

11001102
#[test]
1101-
fn test_sort_pushdown_exact_no_fetch_no_limit() {
1102-
// When a source returns Exact (without fetch), the SortExec should be
1103-
// removed entirely with no GlobalLimitExec wrapper.
1103+
fn test_custom_plan_can_opt_into_sort_pushdown() -> Result<()> {
11041104
let schema = schema();
11051105
let a = sort_expr("a", &schema);
11061106
let b = sort_expr("b", &schema);
11071107
let source =
11081108
Arc::new(TestScan::new(schema.clone(), vec![]).with_exact_pushdown(true));
11091109

11101110
let ordering = LexOrdering::new(vec![a, b.reverse()]).unwrap();
1111-
let plan = sort_exec(ordering, source);
1111+
let plan = sort_exec(ordering.clone(), source);
1112+
let mut config = ConfigOptions::new();
1113+
config.execution.target_partitions = 1;
1114+
let plan = EnsureRequirements::new().optimize(plan, &config)?;
1115+
assert!(plan.is::<SortExec>());
11121116

1113-
insta::assert_snapshot!(
1114-
OptimizationTest::new(plan, PushdownSort::new(), true),
1115-
@r"
1116-
OptimizationTest:
1117-
input:
1118-
- SortExec: expr=[a@0 ASC, b@1 DESC NULLS LAST], preserve_partitioning=[false]
1119-
- TestScan
1120-
output:
1121-
Ok:
1122-
- TestScan: requested_ordering=[a@0 ASC, b@1 DESC NULLS LAST]
1123-
"
1124-
);
1117+
let optimized = PushdownSort::new().optimize(plan, &config)?;
1118+
assert!(optimized.is::<TestScan>());
1119+
assert_eq!(optimized.output_ordering(), Some(&ordering));
1120+
Ok(())
1121+
}
1122+
1123+
#[test]
1124+
fn test_custom_plan_with_fetch_keeps_outer_sort() -> Result<()> {
1125+
let schema = schema();
1126+
let a = sort_expr("a", &schema);
1127+
let source = TestScan::new(schema.clone(), vec![[a.clone()].into()])
1128+
.with_supports_fetch(true)
1129+
.with_fetch(Some(10))
1130+
.unwrap();
1131+
let ordering = LexOrdering::new(vec![a, sort_expr("b", &schema)]).unwrap();
1132+
let plan = sort_exec(ordering.clone(), source);
1133+
1134+
let mut config = ConfigOptions::new();
1135+
config.execution.target_partitions = 1;
1136+
let optimized = EnsureRequirements::new().optimize(plan, &config)?;
1137+
let sort = optimized
1138+
.downcast_ref::<SortExec>()
1139+
.expect("a fetched custom plan must retain the requested outer sort");
1140+
assert_eq!(sort.expr(), &ordering);
1141+
assert!(sort.input().is::<TestScan>());
1142+
assert_eq!(sort.input().fetch(), Some(10));
1143+
Ok(())
11251144
}
11261145

11271146
#[test]

‎datafusion/core/tests/physical_optimizer/test_utils.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1061,6 +1061,10 @@ impl ExecutionPlan for TestScan {
10611061
Ok(Arc::new(Statistics::new_unknown(&self.schema)))
10621062
}
10631063

1064+
fn supports_limit_pushdown(&self) -> bool {
1065+
self.supports_fetch
1066+
}
1067+
10641068
fn with_fetch(&self, fetch: Option<usize>) -> Option<Arc<dyn ExecutionPlan>> {
10651069
if self.supports_fetch {
10661070
let mut new_scan = self.clone();

0 commit comments

Comments
 (0)