Skip to content

Commit 12a12bc

Browse files
crm26claude
andcommitted
Address review feedback from neilconway
- Per-expression early bail-out: check has_subquery(&rewritten) immediately after each rewrite instead of processing all exprs then checking at end. Preserves all-or-nothing semantics with earlier termination. - Add "Optimization:" prefix to alias preservation comment to clarify it is not required for correctness. - Add test projection_mixed_decorrelatable_and_non_bails_out covering one decorrelatable + one non-decorrelatable subquery in the same projection (exercises the partial-rewrite-then-bail path). Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
1 parent f57048f commit 12a12bc

1 file changed

Lines changed: 80 additions & 19 deletions

File tree

‎datafusion/optimizer/src/decorrelate_predicate_subquery.rs‎

Lines changed: 80 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -131,30 +131,34 @@ impl OptimizerRule for DecorrelatePredicateSubquery {
131131
let mut cur_input = Arc::unwrap_or_clone(projection.input);
132132
let mut new_exprs = Vec::with_capacity(projection.expr.len());
133133

134+
// Rewrite each expression in turn. Check per-expression whether
135+
// decorrelation succeeded, and bail out early on the first failure
136+
// (same all-or-nothing semantics as ScalarSubqueryToJoin).
134137
for expr in &projection.expr {
135-
if has_subquery(expr) {
136-
let (plan, rewritten) =
137-
rewrite_inner_subqueries(cur_input, expr.clone(), config)?;
138-
cur_input = plan;
139-
new_exprs.push(rewritten);
140-
} else {
138+
if !has_subquery(expr) {
141139
new_exprs.push(expr.clone());
140+
continue;
142141
}
142+
let (plan, rewritten) =
143+
rewrite_inner_subqueries(cur_input, expr.clone(), config)?;
144+
if has_subquery(&rewritten) {
145+
// Decorrelation failed for this expression — bail out for
146+
// the whole projection and return the original plan.
147+
let original = Projection::try_new_with_schema(
148+
projection.expr,
149+
original_input,
150+
projection.schema,
151+
)?;
152+
return Ok(Transformed::no(LogicalPlan::Projection(original)));
153+
}
154+
cur_input = plan;
155+
new_exprs.push(rewritten);
143156
}
144157

145-
// If any expression still contains a subquery after rewriting,
146-
// decorrelation failed — bail out and return the original plan
147-
// unchanged (same pattern as ScalarSubqueryToJoin).
148-
if new_exprs.iter().any(has_subquery) {
149-
let original = Projection::try_new_with_schema(
150-
projection.expr,
151-
original_input,
152-
projection.schema,
153-
)?;
154-
return Ok(Transformed::no(LogicalPlan::Projection(original)));
155-
}
156-
157-
// Preserve original column names via aliases where the rewrite changed them
158+
// Optimization: preserve original column names via aliases where
159+
// the rewrite changed them. Not required for correctness — the
160+
// rewritten plan is still valid without aliases — but keeps the
161+
// output column names stable for downstream consumers.
158162
let proj_exprs: Vec<Expr> = projection
159163
.expr
160164
.iter()
@@ -2422,4 +2426,61 @@ mod tests {
24222426
"#
24232427
)
24242428
}
2429+
2430+
/// When a projection contains one decorrelatable subquery and one that
2431+
/// cannot be decorrelated, the whole projection should bail out and leave
2432+
/// both subqueries in place — neither should be partially rewritten.
2433+
#[test]
2434+
fn projection_mixed_decorrelatable_and_non_bails_out() -> Result<()> {
2435+
let table_scan = test_table_scan()?;
2436+
2437+
// Decorrelatable subquery: simple correlated IN with no LIMIT
2438+
let sq_ok = Arc::new(
2439+
LogicalPlanBuilder::from(test_table_scan_with_name("sq_ok")?)
2440+
.filter(out_ref_col(DataType::UInt32, "test.a").eq(col("sq_ok.a")))?
2441+
.project(vec![col("sq_ok.c")])?
2442+
.build()?,
2443+
);
2444+
2445+
// Non-decorrelatable subquery: LIMIT prevents decorrelation
2446+
let sq_bad = Arc::new(
2447+
LogicalPlanBuilder::from(test_table_scan_with_name("sq_bad")?)
2448+
.filter(out_ref_col(DataType::UInt32, "test.a").eq(col("sq_bad.a")))?
2449+
.project(vec![col("sq_bad.c")])?
2450+
.limit(0, Some(1))?
2451+
.build()?,
2452+
);
2453+
2454+
let case_ok =
2455+
when(in_subquery(col("c"), sq_ok), lit("ok")).otherwise(lit("not_ok"))?;
2456+
let case_bad =
2457+
when(in_subquery(col("c"), sq_bad), lit("bad")).otherwise(lit("not_bad"))?;
2458+
2459+
// case_ok is listed before case_bad so it would decorrelate first
2460+
// if we processed each expression independently. The per-expression
2461+
// bail-out must detect case_bad's failure and discard case_ok's
2462+
// rewrite too.
2463+
let plan = LogicalPlanBuilder::from(table_scan)
2464+
.project(vec![col("a"), case_ok, case_bad])?
2465+
.build()?;
2466+
2467+
// Both subqueries remain — bail-out is all-or-nothing.
2468+
// The decorrelatable one is NOT rewritten because its sibling can't be.
2469+
assert_optimized_plan_equal!(
2470+
plan,
2471+
@r#"
2472+
Projection: test.a, CASE WHEN test.c IN (<subquery>) THEN Utf8("ok") ELSE Utf8("not_ok") END, CASE WHEN test.c IN (<subquery>) THEN Utf8("bad") ELSE Utf8("not_bad") END [a:UInt32, CASE WHEN IN THEN Utf8("ok") ELSE Utf8("not_ok") END:Utf8, CASE WHEN IN THEN Utf8("bad") ELSE Utf8("not_bad") END:Utf8]
2473+
Subquery: [c:UInt32]
2474+
Projection: sq_ok.c [c:UInt32]
2475+
Filter: outer_ref(test.a) = sq_ok.a [a:UInt32, b:UInt32, c:UInt32]
2476+
TableScan: sq_ok [a:UInt32, b:UInt32, c:UInt32]
2477+
Subquery: [c:UInt32]
2478+
Limit: skip=0, fetch=1 [c:UInt32]
2479+
Projection: sq_bad.c [c:UInt32]
2480+
Filter: outer_ref(test.a) = sq_bad.a [a:UInt32, b:UInt32, c:UInt32]
2481+
TableScan: sq_bad [a:UInt32, b:UInt32, c:UInt32]
2482+
TableScan: test [a:UInt32, b:UInt32, c:UInt32]
2483+
"#
2484+
)
2485+
}
24252486
}

0 commit comments

Comments
 (0)