Skip to content

Commit 1038d35

Browse files
authored
Fix panic on RightMark hash joins when propagating ordering (#24718)
## Which issue does this PR close? - Closes #24717 (affecting DF 55) ## Rationale for this change Fixes a panic in `enforce_sorting` when given a `RightMark` hash join: `HashjoinExec::maintains_input_order` has been changed to also return `true` on `RightMark` but `enforce_sorting` has not been updated ## What changes are included in this PR? Adds support of `RightMark` joins to `enforce_sorting` they keep the input sorting.
1 parent a38bb10 commit 1038d35

2 files changed

Lines changed: 36 additions & 2 deletions

File tree

‎datafusion/core/tests/physical_optimizer/enforce_sorting.rs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,40 @@ async fn test_remove_unnecessary_sort5() -> Result<()> {
243243
Ok(())
244244
}
245245

246+
#[tokio::test]
247+
async fn test_propagate_in_right_mark_join() -> Result<()> {
248+
let left_schema = create_test_schema2()?;
249+
let right_schema = create_test_schema3()?;
250+
let left_input = memory_exec(&left_schema);
251+
let parquet_ordering = [sort_expr("a", &right_schema)].into();
252+
let right_input =
253+
parquet_exec_with_sort(right_schema.clone(), vec![parquet_ordering]);
254+
let on = vec![(
255+
Arc::new(Column::new_with_schema("col_a", &left_schema)?) as _,
256+
Arc::new(Column::new_with_schema("c", &right_schema)?) as _,
257+
)];
258+
let join = hash_join_exec(left_input, right_input, on, None, &JoinType::RightMark)?;
259+
let physical_plan = sort_exec([sort_expr("a", &join.schema())].into(), join);
260+
261+
let test = EnforceSortingTest::new(physical_plan).with_repartition_sorts(true);
262+
assert_snapshot!(test.run(), @r"
263+
Input Plan:
264+
SortExec: expr=[a@0 ASC], preserve_partitioning=[false]
265+
HashJoinExec: mode=Partitioned, join_type=RightMark, on=[(col_a@0, c@2)]
266+
DataSourceExec: partitions=1, partition_sizes=[0]
267+
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
268+
269+
Optimized Plan:
270+
SortPreservingMergeExec: [a@0 ASC]
271+
HashJoinExec: mode=Partitioned, join_type=RightMark, on=[(col_a@0, c@2)]
272+
RepartitionExec: partitioning=Hash([col_a@0], 10), input_partitions=1
273+
DataSourceExec: partitions=1, partition_sizes=[0]
274+
RepartitionExec: partitioning=Hash([c@2], 10), input_partitions=1, maintains_sort_order=true
275+
DataSourceExec: file_groups={1 group: [[x]]}, projection=[a, b, c, d, e], output_ordering=[a@0 ASC], file_type=parquet
276+
");
277+
Ok(())
278+
}
279+
246280
#[tokio::test]
247281
async fn test_hash_join_interleaved_projection_preserves_parent_sort() -> Result<()> {
248282
let left_schema = create_test_schema()?;

‎datafusion/physical-optimizer/src/ensure_requirements/enforce_sorting/sort_pushdown.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -954,7 +954,7 @@ fn handle_custom_pushdown(
954954
}
955955

956956
// For hash join we only maintain the input order for the right child
957-
// for join type: Inner, Right, RightSemi, RightAnti
957+
// for join type: Inner, Right, RightSemi, RightAnti, RightMark
958958
fn handle_hash_join(
959959
plan: &HashJoinExec,
960960
parent_required: OrderingRequirements,
@@ -1045,7 +1045,7 @@ fn build_join_column_index(plan: &HashJoinExec) -> Vec<ColumnIndex> {
10451045
.chain(map_fields(plan.right().schema(), JoinSide::Right))
10461046
.collect::<Vec<_>>()
10471047
}
1048-
JoinType::RightSemi | JoinType::RightAnti => {
1048+
JoinType::RightSemi | JoinType::RightAnti | JoinType::RightMark => {
10491049
map_fields(plan.right().schema(), JoinSide::Right)
10501050
}
10511051
_ => unreachable!("unexpected join type: {}", plan.join_type()),

0 commit comments

Comments
 (0)