Skip to content

Commit 7a0d026

Browse files
authored
fix: make PushDownLeafProjections work with unnest (apache#22620)
## Which issue does this PR close? Closes apache#22615. ## Rationale for this change `PushDownLeafProjections` throws an error when it tries to push down an expression through an `Unnest` node. It also tries to push leaf projections incorrectly for `Unnest` nodes. ## What changes are included in this PR? This PR makes PushDownLeafProjections never push down leaf expressions through Unnest nodes. This avoids the code in `try_push_into_inputs` from calling: ``` let new_node = node.with_new_exprs(node.expressions(), new_inputs)?; ``` on an Unnest node, which doesn't work. Changing that line is not enough, as even in that case the code in `try_push_into_inputs` to decide when to push down a projection is not valid for Unnest nodes, as explained in apache#22615 In this PR, I only want to make sure that the optimizer works for Unnest cases, but we could probably do better and allow leaf expressions push downs when the expression refer to a column that is not being unnested. ## Are these changes tested? Added unit tests and `.slt` tests for these cases. ## Are there any user-facing changes? No
1 parent 5c92390 commit 7a0d026

2 files changed

Lines changed: 95 additions & 0 deletions

File tree

‎datafusion/optimizer/src/extract_leaf_expressions.rs‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1143,6 +1143,13 @@ fn try_push_into_inputs(
11431143
return Ok(None);
11441144
}
11451145

1146+
// Unnest may output a column with the same name but different value/type
1147+
// than its input column. Name-based routing cannot distinguish those.
1148+
// On top of that Unnest can't go through the `node.with_new_exprs(node.expressions(), new_inputs)` rebuild
1149+
if matches!(node, LogicalPlan::Unnest(_)) {
1150+
return Ok(None);
1151+
}
1152+
11461153
// SubqueryAlias remaps qualifiers between input and output.
11471154
// Rewrite pairs/columns from alias-space to input-space before routing.
11481155
let remapped = if let LogicalPlan::SubqueryAlias(sa) = node {
@@ -3035,4 +3042,48 @@ mod tests {
30353042

30363043
Ok(())
30373044
}
3045+
3046+
/// Regression test for the `Assertion failed: expr.is_empty(): Unnest`
3047+
/// internal error.
3048+
///
3049+
/// `try_push_into_inputs` rebuilds the parent node via
3050+
/// `node.with_new_exprs(node.expressions(), new_inputs)`. For `Unnest`,
3051+
/// `apply_expressions` exposes the `exec_columns` as `Expr::Column`s
3052+
/// (so `expressions()` is **non-empty**), but `with_new_exprs` for
3053+
/// `Unnest` immediately calls `assert_no_expressions(expr)?` and errors
3054+
/// out. The optimizer should treat `Unnest` as a barrier and bail
3055+
/// instead of attempting to push through it.
3056+
#[test]
3057+
fn test_no_push_through_unnest() -> Result<()> {
3058+
use arrow::datatypes::{DataType, Field, Schema};
3059+
3060+
let schema = Schema::new(vec![
3061+
Field::new("list_col", DataType::new_list(DataType::Int32, true), true),
3062+
Field::new("other_col", DataType::Int32, true),
3063+
]);
3064+
let table_scan =
3065+
datafusion_expr::logical_plan::table_scan(Some("t"), &schema, None)?
3066+
.build()?;
3067+
let plan = LogicalPlanBuilder::from(table_scan)
3068+
.unnest_column("list_col")?
3069+
.filter(leaf_udf(col("list_col"), "x").eq(lit(1i32)))?
3070+
.build()?;
3071+
3072+
let ctx = OptimizerContext::new().with_max_passes(1);
3073+
let optimizer = Optimizer::with_rules(vec![
3074+
Arc::new(ExtractLeafExpressions::new()),
3075+
Arc::new(PushDownLeafProjections::new()),
3076+
]);
3077+
let optimized = optimizer.optimize(plan, &ctx, |_, _| {})?;
3078+
3079+
insta::assert_snapshot!(format!("{optimized}"), @r#"
3080+
Projection: list_col, t.other_col
3081+
Filter: __datafusion_extracted_1 = Int32(1)
3082+
Projection: leaf_udf(list_col, Utf8("x")) AS __datafusion_extracted_1, list_col, t.other_col
3083+
Unnest: lists[t.list_col|depth=1] structs[]
3084+
TableScan: t
3085+
"#);
3086+
3087+
Ok(())
3088+
}
30383089
}

‎datafusion/sqllogictest/test_files/unnest.slt‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1419,3 +1419,47 @@ FROM (
14191419

14201420
statement ok
14211421
DROP TABLE unused_unnest_pruning;
1422+
1423+
## Regression: pushing a leaf-extracted projection (containing get_field,
1424+
## which has MoveTowardsLeafNodes placement) through an `Unnest` used to
1425+
## trip `Assertion failed: expr.is_empty(): Unnest` inside
1426+
## `PushDownLeafProjections`. The optimizer must not try to pushdown these
1427+
## projections through an `Unnest` and should produce a valid plan.
1428+
1429+
statement ok
1430+
CREATE TABLE struct_and_list_table
1431+
AS VALUES
1432+
(struct(1, 2), [10, 20, 30]),
1433+
(struct(3, 4), [40, 50]);
1434+
1435+
query I
1436+
SELECT sum(get_field(s, 'c0'))
1437+
FROM (SELECT s, unnest(arr)
1438+
FROM (SELECT column1 AS s, column2 AS arr
1439+
FROM struct_and_list_table));
1440+
----
1441+
9
1442+
1443+
statement ok
1444+
DROP TABLE struct_and_list_table;
1445+
1446+
## Regression: get_field directly references the struct produced by unnest.
1447+
## This covers the case where the leaf-extracted expression depends on the
1448+
## unnested column itself rather than a sibling input column below the Unnest.
1449+
1450+
statement ok
1451+
CREATE TABLE list_struct_table
1452+
AS VALUES
1453+
([struct(1, 'a'), struct(2, 'b')]),
1454+
([struct(3, 'c')]);
1455+
1456+
query IT
1457+
SELECT get_field(unnest(column1), 'c0'), get_field(unnest(column1), 'c1')
1458+
FROM list_struct_table;
1459+
----
1460+
1 a
1461+
2 b
1462+
3 c
1463+
1464+
statement ok
1465+
DROP TABLE list_struct_table;

0 commit comments

Comments
 (0)