Skip to content

Commit 320f268

Browse files
committed
fix(optimizer): stop pull-up at Join above scalar aggregate and address review feedback
1 parent c0e5d77 commit 320f268

2 files changed

Lines changed: 69 additions & 19 deletions

File tree

‎datafusion/optimizer/src/decorrelate.rs‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ use datafusion_common::tree_node::{
3131
use datafusion_common::utils::has_float_leaf;
3232
use datafusion_common::{
3333
Column, DFSchema, DFSchemaRef, HashMap, Result, ScalarValue, assert_or_internal_err,
34-
plan_err,
34+
internal_err, plan_err,
3535
};
3636
use datafusion_expr::expr::{Alias, GroupingSet};
3737
use datafusion_expr::expr_rewriter::strip_outer_reference;
@@ -55,10 +55,12 @@ pub struct PullUpCorrelatedExpr {
5555
/// Correlated inner columns collected across the subquery, including their
5656
/// re-qualified names after passing through [`LogicalPlan::SubqueryAlias`].
5757
pub correlated_subquery_cols: BTreeSet<Column>,
58-
/// Deprecated: use [`Self::correlated_subquery_cols`] instead.
59-
///
6058
/// Kept for SemVer compatibility; populated with the root plan's
6159
/// correlated columns when the rewrite finishes.
60+
#[deprecated(
61+
since = "56.0.0",
62+
note = "use `PullUpCorrelatedExpr::correlated_subquery_cols` instead"
63+
)]
6264
pub correlated_subquery_cols_map: HashMap<LogicalPlan, BTreeSet<Column>>,
6365
pub in_predicate_opt: Option<Expr>,
6466
/// Is this an Exists(Not Exists) SubQuery. Defaults to **FALSE**
@@ -83,7 +85,12 @@ pub struct PullUpCorrelatedExpr {
8385
///
8486
/// [the count bug]: https://github.com/apache/datafusion/issues/10553
8587
pub need_handle_count_bug: bool,
86-
/// mapping from the plan to its expressions' evaluation result on empty batch
88+
/// Kept for SemVer compatibility; holds only the root plan's expressions'
89+
/// evaluation result on an empty batch when the rewrite finishes.
90+
#[deprecated(
91+
since = "56.0.0",
92+
note = "use `PullUpCorrelatedExpr::root_count_expr_map` instead"
93+
)]
8794
pub collected_count_expr_map: HashMap<LogicalPlan, ExprResultMap>,
8895
/// The root plan's expressions' evaluation result on an empty batch.
8996
pub root_count_expr_map: Option<ExprResultMap>,
@@ -139,6 +146,7 @@ impl Default for PullUpCorrelatedExpr {
139146
}
140147

141148
impl PullUpCorrelatedExpr {
149+
#[expect(deprecated)]
142150
pub fn new() -> Self {
143151
Self {
144152
join_filters: vec![],
@@ -412,10 +420,15 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
412420
// `column_refs_above` holds only the nodes strictly above `plan` and
413421
// `frame` holds the state pulled up from `plan`'s children.
414422
self.column_refs_above.pop();
415-
let PullUpFrame {
423+
let Some(PullUpFrame {
416424
correlated_cols: local_correlated_cols,
417425
count_expr_map: input_count_expr_map,
418-
} = self.pulled_up_frames.pop().unwrap_or_default();
426+
}) = self.pulled_up_frames.pop()
427+
else {
428+
return internal_err!(
429+
"PullUpCorrelatedExpr: f_up without a matching f_down frame"
430+
);
431+
};
419432

420433
// A node that null-extends or regroups rows can put a NULL back into
421434
// a column that a correlated filter below it rejected, so the filters
@@ -753,6 +766,12 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
753766
.pass_through_pulled_up_cols(plan, input_count_expr_map.is_some())?;
754767
(transformed, local_correlated_cols, None)
755768
}
769+
LogicalPlan::Join(_) | LogicalPlan::AsOfJoin(_)
770+
if self.pulled_up_scalar_agg =>
771+
{
772+
self.can_pull_up = false;
773+
return Ok(Transformed::new(plan, false, TreeNodeRecursion::Stop));
774+
}
756775
// `f_down` checked that the pulled up filters do not read an
757776
// unnested column.
758777
LogicalPlan::Unnest(_)
@@ -773,11 +792,13 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr {
773792
parent.count_expr_map = out_count_expr_map;
774793
}
775794
} else {
795+
#[expect(deprecated)]
776796
self.correlated_subquery_cols_map.insert(
777797
transformed.data.clone(),
778798
self.correlated_subquery_cols.clone(),
779799
);
780800
if let Some(ref map) = out_count_expr_map {
801+
#[expect(deprecated)]
781802
self.collected_count_expr_map
782803
.insert(transformed.data.clone(), map.clone());
783804
}

‎datafusion/sqllogictest/test_files/subquery.slt‎

Lines changed: 42 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4219,25 +4219,32 @@ WHERE x.d IN (SELECT t.c FROM (SELECT * FROM (VALUES (1)) AS y(c) WHERE y.c = x.
42194219
# Regression test for #26692: decorrelation must not lose correlated columns
42204220
# when a subquery contains two structurally equal subplans.
42214221
statement ok
4222-
CREATE TABLE sq_dup_t(a INT);
4222+
CREATE TABLE sq_dup_t(a INT, b INT);
42234223

42244224
statement ok
4225-
CREATE TABLE sq_dup_t2(a INT);
4225+
CREATE TABLE sq_dup_t2(a INT, b INT);
42264226

42274227
statement ok
4228-
INSERT INTO sq_dup_t VALUES (1), (2), (3);
4228+
CREATE TABLE sq_dup_t3(c INT);
42294229

42304230
statement ok
4231-
INSERT INTO sq_dup_t2 VALUES (1), (2);
4231+
INSERT INTO sq_dup_t VALUES (1, 1), (2, 2);
4232+
4233+
statement ok
4234+
INSERT INTO sq_dup_t2 VALUES (1, 1), (3, 30);
4235+
4236+
statement ok
4237+
INSERT INTO sq_dup_t3 VALUES (10), (20);
42324238

42334239
query I rowsort
4234-
SELECT a FROM sq_dup_t WHERE EXISTS (
4235-
SELECT 1 FROM (SELECT s.a AS x FROM sq_dup_t2 AS s WHERE s.a = sq_dup_t.a)
4236-
CROSS JOIN (SELECT s.a AS y FROM sq_dup_t2 AS s)
4240+
SELECT sq_dup_t.a FROM sq_dup_t, sq_dup_t2 AS s
4241+
WHERE EXISTS (
4242+
SELECT 1 FROM (SELECT * FROM sq_dup_t2 AS s WHERE s.a = sq_dup_t.a) AS x
4243+
CROSS JOIN sq_dup_t2 AS s
42374244
);
42384245
----
42394246
1
4240-
2
4247+
1
42414248

42424249
query I rowsort
42434250
SELECT (
@@ -4247,19 +4254,41 @@ SELECT (
42474254
----
42484255
0
42494256
2
4250-
2
42514257

42524258
query I rowsort
4253-
SELECT a FROM sq_dup_t WHERE EXISTS (
4254-
SELECT 1 FROM (SELECT sq_dup_t2.a FROM sq_dup_t2 WHERE sq_dup_t2.a = sq_dup_t.a) AS x
4255-
CROSS JOIN LATERAL (SELECT sq_dup_t2.a FROM sq_dup_t2 WHERE sq_dup_t2.a = x.a) AS l
4259+
SELECT sq_dup_t.a FROM sq_dup_t
4260+
WHERE EXISTS (
4261+
SELECT 1 FROM (SELECT a FROM sq_dup_t2 WHERE sq_dup_t2.a = sq_dup_t.a) AS x
4262+
CROSS JOIN LATERAL (SELECT b FROM sq_dup_t2 WHERE sq_dup_t2.b = x.a) AS l
42564263
);
42574264
----
42584265
1
4259-
2
4266+
4267+
# Join above a pulled-up groupless scalar aggregate inside EXISTS / IN must
4268+
# not decorrelate into an uncompensated join that drops the empty-input row.
4269+
statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists
4270+
SELECT sq_dup_t.a FROM sq_dup_t WHERE EXISTS (
4271+
SELECT 1 FROM (SELECT count(*) AS c FROM sq_dup_t2 WHERE sq_dup_t2.a = sq_dup_t.a) AS x
4272+
CROSS JOIN sq_dup_t3
4273+
);
4274+
4275+
statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression Exists
4276+
SELECT sq_dup_t.a FROM sq_dup_t WHERE EXISTS (
4277+
SELECT 1 FROM (SELECT count(*) AS c FROM sq_dup_t2 WHERE sq_dup_t2.a = sq_dup_t.a) AS x
4278+
CROSS JOIN sq_dup_t3 WHERE x.c = 0
4279+
);
4280+
4281+
statement error DataFusion error: This feature is not implemented: Physical plan does not support logical expression InSubquery
4282+
SELECT sq_dup_t.a FROM sq_dup_t WHERE sq_dup_t.a IN (
4283+
SELECT x.c + 2 FROM (SELECT count(*) AS c FROM sq_dup_t2 WHERE sq_dup_t2.a = sq_dup_t.a) AS x
4284+
CROSS JOIN sq_dup_t3
4285+
);
42604286

42614287
statement ok
42624288
DROP TABLE sq_dup_t;
42634289

42644290
statement ok
42654291
DROP TABLE sq_dup_t2;
4292+
4293+
statement ok
4294+
DROP TABLE sq_dup_t3;

0 commit comments

Comments
 (0)