Skip to content

Commit 4775354

Browse files
author
wiedld
committed
fix: have orderings include constants which are heterogenius across partitions
1 parent dffeaac commit 4775354

3 files changed

Lines changed: 25 additions & 7 deletions

File tree

‎datafusion/physical-expr/src/equivalence/class.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,17 @@ pub fn const_exprs_contains(
216216
.any(|const_expr| const_expr.expr.eq(expr))
217217
}
218218

219+
/// Checks whether `expr` is among in the uniform `const_exprs`.
220+
pub fn uniform_const_exprs_contains(
221+
const_exprs: &[ConstExpr],
222+
expr: &Arc<dyn PhysicalExpr>,
223+
) -> bool {
224+
const_exprs.iter().any(|const_expr| {
225+
const_expr.expr.eq(expr)
226+
&& const_expr.across_partitions() != AcrossPartitions::Heterogeneous
227+
})
228+
}
229+
219230
/// An `EquivalenceClass` is a set of [`Arc<dyn PhysicalExpr>`]s that are known
220231
/// to have the same value for all tuples in a relation. These are generated by
221232
/// equality predicates (e.g. `a = b`), typically equi-join conditions and

‎datafusion/physical-expr/src/equivalence/properties.rs‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@ use std::slice::Iter;
2222
use std::sync::Arc;
2323
use std::{fmt, mem};
2424

25-
use crate::equivalence::class::{const_exprs_contains, AcrossPartitions};
25+
use crate::equivalence::class::{
26+
const_exprs_contains, uniform_const_exprs_contains, AcrossPartitions,
27+
};
2628
use crate::equivalence::{
2729
EquivalenceClass, EquivalenceGroup, OrderingEquivalenceClass, ProjectionMapping,
2830
};
@@ -203,8 +205,9 @@ impl EquivalenceProperties {
203205
let constants = self.constants();
204206
let mut output_ordering = self.oeq_class().output_ordering().unwrap_or_default();
205207
// Prune out constant expressions
206-
output_ordering
207-
.retain(|sort_expr| !const_exprs_contains(constants, &sort_expr.expr));
208+
output_ordering.retain(|sort_expr| {
209+
!uniform_const_exprs_contains(constants, &sort_expr.expr)
210+
});
208211
(!output_ordering.is_empty()).then_some(output_ordering)
209212
}
210213

@@ -438,7 +441,7 @@ impl EquivalenceProperties {
438441
let filtered_exprs = LexOrdering::new(
439442
sort_exprs
440443
.into_iter()
441-
.filter(|expr| !self.is_expr_constant(&expr.expr))
444+
.filter(|expr| !self.is_expr_constant_across_partitions(&expr.expr))
442445
.collect(),
443446
);
444447

@@ -4091,7 +4094,9 @@ mod tests {
40914094
// Setup constant columns
40924095
let col_a = col("a", &schema)?;
40934096
let col_b = col("b", &schema)?;
4094-
eq_properties = eq_properties.with_constants([ConstExpr::from(&col_a)]);
4097+
let const_val = AcrossPartitions::Uniform(Some(ScalarValue::Int32(Some(1))));
4098+
eq_properties = eq_properties
4099+
.with_constants([ConstExpr::from(&col_a).with_across_partitions(const_val)]);
40954100

40964101
let sort_exprs = LexOrdering::new(vec![
40974102
PhysicalSortExpr {
@@ -4272,7 +4277,9 @@ mod tests {
42724277
let asc = SortOptions::default();
42734278

42744279
// Constants: c is constant
4275-
eq_properties = eq_properties.with_constants([ConstExpr::from(&col_c)]);
4280+
let const_val = AcrossPartitions::Uniform(Some(ScalarValue::Int32(Some(1))));
4281+
eq_properties = eq_properties
4282+
.with_constants([ConstExpr::from(&col_c).with_across_partitions(const_val)]);
42764283

42774284
// Equality: b = d
42784285
eq_properties.add_equal_conditions(&col_b, &col_d)?;

‎datafusion/physical-optimizer/src/utils.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ pub fn add_sort_above<T: Clone + Default>(
4545
!node
4646
.plan
4747
.equivalence_properties()
48-
.is_expr_constant(&sort_expr.expr)
48+
.is_expr_constant_across_partitions(&sort_expr.expr)
4949
});
5050
let mut new_sort = SortExec::new(sort_expr, Arc::clone(&node.plan)).with_fetch(fetch);
5151
if node.plan.output_partitioning().partition_count() > 1 {

0 commit comments

Comments
 (0)