Skip to content

Commit 846a867

Browse files
committed
fix: split "union" and "interleave"
Silently interleaving partitions is confusing and causes bugs. Fixes apache#5970.
1 parent 463ec30 commit 846a867

5 files changed

Lines changed: 404 additions & 81 deletions

File tree

‎datafusion/core/src/execution/context.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ use crate::{
2727
logical_expr::{PlanType, ToStringifiedPlan},
2828
optimizer::optimizer::Optimizer,
2929
physical_optimizer::{
30-
aggregate_statistics::AggregateStatistics, join_selection::JoinSelection,
31-
optimizer::PhysicalOptimizerRule,
30+
aggregate_statistics::AggregateStatistics, interleave::Interleave,
31+
join_selection::JoinSelection, optimizer::PhysicalOptimizerRule,
3232
},
3333
};
3434
use datafusion_expr::{
@@ -1330,6 +1330,8 @@ impl SessionState {
13301330
// The EnforceDistribution rule is for adding essential repartition to satisfy the required
13311331
// distribution. Please make sure that the whole plan tree is determined before this rule.
13321332
Arc::new(EnforceDistribution::new()),
1333+
// Try to interleave before enforcing sorting.
1334+
Arc::new(Interleave::new()),
13331335
// The EnforceSorting rule is for adding essential local sorting to satisfy the required
13341336
// ordering. Please make sure that the whole plan tree is determined before this rule.
13351337
// Note that one should always run this rule after running the EnforceDistribution rule
Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Rule to replace [`UnionExec`] with [`InterleaveExec`].
19+
20+
use std::sync::Arc;
21+
22+
use datafusion_common::{
23+
config::ConfigOptions,
24+
tree_node::{Transformed, TreeNode},
25+
Result,
26+
};
27+
28+
use crate::physical_plan::{
29+
union::{can_interleave, InterleaveExec, UnionExec},
30+
ExecutionPlan,
31+
};
32+
33+
use super::PhysicalOptimizerRule;
34+
35+
/// Rule to replace [`UnionExec`] with [`InterleaveExec`].
36+
#[derive(Default)]
37+
pub struct Interleave {}
38+
39+
impl Interleave {
40+
#[allow(missing_docs)]
41+
pub fn new() -> Self {
42+
Self {}
43+
}
44+
}
45+
46+
impl PhysicalOptimizerRule for Interleave {
47+
fn optimize(
48+
&self,
49+
plan: Arc<dyn ExecutionPlan>,
50+
_config: &ConfigOptions,
51+
) -> Result<Arc<dyn ExecutionPlan>> {
52+
plan.transform_up(&|plan| {
53+
if let Some(union_exec) = plan.as_any().downcast_ref::<UnionExec>() {
54+
if can_interleave(&union_exec.inputs()) {
55+
let plan = InterleaveExec::try_new(union_exec.inputs().clone())?;
56+
return Ok(Transformed::Yes(Arc::new(plan)));
57+
}
58+
}
59+
Ok(Transformed::No(plan))
60+
})
61+
}
62+
63+
fn name(&self) -> &str {
64+
"interleave"
65+
}
66+
67+
fn schema_check(&self) -> bool {
68+
true
69+
}
70+
}

‎datafusion/core/src/physical_optimizer/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ pub mod coalesce_batches;
2323
pub mod combine_partial_final_agg;
2424
pub mod dist_enforcement;
2525
pub mod global_sort_selection;
26+
pub mod interleave;
2627
pub mod join_selection;
2728
pub mod optimizer;
2829
pub mod pipeline_checker;

0 commit comments

Comments
 (0)