Skip to content

Commit 9ae5916

Browse files
committed
fix: prevent pulling non-equality filters through DISTINCT in LATERAL joins (fixes apache#25808)
1 parent 4452ee4 commit 9ae5916

1 file changed

Lines changed: 4 additions & 1 deletion

File tree

‎datafusion/optimizer/src/decorrelate.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -329,8 +329,11 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
329329
{
330330
// If the aggregation is from a distinct it will not change the result for
331331
// exists/in subqueries so we can still pull up all predicates.
332+
// For LATERAL joins (where in_predicate_opt is None), pulling a non-equality
333+
// filter through a DISTINCT aggregate causes duplicate rows because the
334+
// DISTINCT output rows are duplicated for each outer row.
332335
let is_distinct = aggregate.aggr_expr.is_empty();
333-
if !is_distinct {
336+
if !is_distinct || self.in_predicate_opt.is_none() {
334337
self.can_pull_up = self.can_pull_up && self.can_pull_over_aggregation;
335338
}
336339
let mut local_correlated_cols = BTreeSet::new();

0 commit comments

Comments
 (0)