Skip to content

Commit 5eed032

Browse files
fix: only error when the pull up adds a DISTINCT key
1 parent 71e58ad commit 5eed032

2 files changed

Lines changed: 36 additions & 11 deletions

File tree

‎datafusion/optimizer/src/decorrelate.rs‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -573,17 +573,6 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
573573
self.can_pull_up = false;
574574
return Ok(Transformed::new(plan, false, TreeNodeRecursion::Stop));
575575
}
576-
// Only a DISTINCT under EXISTS/IN may skip this check. The pulled
577-
// up column becomes an extra group key, which for a DISTINCT only
578-
// adds duplicate rows, and EXISTS/IN ignore duplicates. Elsewhere,
579-
// e.g. LATERAL, the duplicates reach the result. See #25808.
580-
let is_distinct = aggregate.aggr_expr.is_empty();
581-
let is_exists_or_in =
582-
self.exists_sub_query || self.in_predicate_opt.is_some();
583-
let can_skip_check = is_distinct && is_exists_or_in;
584-
if !can_skip_check {
585-
self.can_pull_up = self.can_pull_up && self.can_pull_over_aggregation;
586-
}
587576
let mut local_correlated_cols = BTreeSet::new();
588577
collect_local_correlated_cols(
589578
&plan,
@@ -649,6 +638,20 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
649638
aggregate.input.schema(),
650639
)?
651640
};
641+
// A DISTINCT may skip this check when the pull up adds no group
642+
// key, or under EXISTS/IN. An added key only adds duplicate rows,
643+
// which EXISTS/IN ignore. Elsewhere, e.g. LATERAL, the duplicates
644+
// reach the result. See #25808.
645+
let is_distinct = aggregate.aggr_expr.is_empty();
646+
let is_exists_or_in =
647+
self.exists_sub_query || self.in_predicate_opt.is_some();
648+
let adds_group_keys = missing_exprs
649+
.iter()
650+
.any(|expr| !aggregate.group_expr.contains(expr));
651+
let can_skip_check = is_distinct && (is_exists_or_in || !adds_group_keys);
652+
if !can_skip_check {
653+
self.can_pull_up = self.can_pull_up && self.can_pull_over_aggregation;
654+
}
652655

653656
// if the original group expressions are empty, need to handle the Count bug
654657
let mut expr_result_map_for_count_bug = HashMap::new();

‎datafusion/sqllogictest/test_files/lateral_join.slt‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -866,6 +866,17 @@ SELECT t1.id, sub.t1_id
866866
FROM t1, LATERAL (SELECT DISTINCT t2.t1_id FROM t2 WHERE t2.id > t1.id) AS sub
867867
ORDER BY t1.id, sub.t1_id;
868868

869+
# The same with a GROUP BY that has no aggregates, which is planned as a
870+
# DISTINCT.
871+
statement error OuterReferenceColumn
872+
SELECT t1.id, sub.t1_id
873+
FROM t1, LATERAL (SELECT t2.t1_id FROM t2 WHERE t2.id > t1.id GROUP BY t2.t1_id) AS sub;
874+
875+
# The same with LEFT JOIN LATERAL.
876+
statement error OuterReferenceColumn
877+
SELECT t1.id, sub.t1_id
878+
FROM t1 LEFT JOIN LATERAL (SELECT DISTINCT t2.t1_id FROM t2 WHERE t2.id > t1.id) AS sub ON true;
879+
869880
# With an equality filter, every row of a group has the same `t2.t1_id`,
870881
# so adding it to the DISTINCT keys splits nothing.
871882
query II
@@ -876,6 +887,17 @@ ORDER BY t1.id, sub.t1_id;
876887
1 1
877888
2 2
878889

890+
# A non-equality filter on a column that is already a DISTINCT key adds no
891+
# key, so the pull up acts as a HAVING and stays decorrelated.
892+
query II
893+
SELECT t1.id, sub.id
894+
FROM t1, LATERAL (SELECT DISTINCT t2.id FROM t2 WHERE t2.id > t1.id * 10) AS sub
895+
ORDER BY t1.id, sub.id;
896+
----
897+
1 20
898+
1 30
899+
2 30
900+
879901
# --- Implicit LATERAL (comma without LATERAL keyword) ---
880902
# DuckDB auto-detects lateral correlation; DataFusion requires
881903
# the explicit LATERAL keyword.

0 commit comments

Comments
 (0)