Skip to content

Commit 0fc962f

Browse files
authored
Fix self join leaf expressions (apache#24242)
## Which issue does this PR close? - Closes apache#24241 ## Rationale for this change This is a bugfix for a specific query shape. ## What changes are included in this PR? Changes the way leaf expressions are extracted when the alias is essentially the same as the column ## Are these changes tested? Yes, with a test that fails on main currently ## Are there any user-facing changes? Nope
1 parent 40c208e commit 0fc962f

1 file changed

Lines changed: 96 additions & 6 deletions

File tree

‎datafusion/optimizer/src/extract_leaf_expressions.rs‎

Lines changed: 96 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -757,6 +757,21 @@ fn try_push_input(
757757
split_and_push_projection(proj, alias_generator)
758758
}
759759

760+
/// If this is a passthrough column. I.e,
761+
///
762+
/// - A bare column expression
763+
/// - A trivial alias rename (i.e, `p.column AS column`)
764+
fn passthrough_column(expr: &Expr) -> Option<&Column> {
765+
match expr {
766+
Expr::Column(col) => Some(col),
767+
Expr::Alias(alias) => match alias.expr.as_ref() {
768+
Expr::Column(col) if col.name == alias.name => Some(col),
769+
_ => None,
770+
},
771+
_ => None,
772+
}
773+
}
774+
760775
/// Splits a projection into extractable pieces, pushes them towards leaf
761776
/// nodes, and adds a recovery projection if needed.
762777
///
@@ -831,7 +846,13 @@ fn split_and_push_projection(
831846
let mut proj_exprs_captured: usize = 0;
832847

833848
for (expr, (qualifier, field)) in proj.expr.iter().zip(original_schema.iter()) {
834-
if let Expr::Alias(alias) = expr
849+
if let Some(col) = passthrough_column(expr) {
850+
// Pass-through — the input already produces this column, so
851+
// there is nothing to extract; just track it in the extractor.
852+
extractors[0].columns_needed.insert(col.clone());
853+
recovery_exprs.push(expr.clone());
854+
proj_exprs_captured += 1;
855+
} else if let Expr::Alias(alias) = expr
835856
&& alias.name.starts_with(EXTRACTED_EXPR_PREFIX)
836857
{
837858
// Insert the full Alias expression as the key so that
@@ -847,11 +868,6 @@ fn split_and_push_projection(
847868
.insert(expr.clone(), alias_name.clone());
848869
recovery_exprs.push(Expr::Column(Column::new_unqualified(&alias_name)));
849870
proj_exprs_captured += 1;
850-
} else if let Expr::Column(col) = expr {
851-
// Plain column pass-through — track it in the extractor
852-
extractors[0].columns_needed.insert(col.clone());
853-
recovery_exprs.push(expr.clone());
854-
proj_exprs_captured += 1;
855871
} else {
856872
// Everything else: run through routing_extract
857873
let transformed =
@@ -3163,4 +3179,78 @@ mod tests {
31633179

31643180
Ok(())
31653181
}
3182+
3183+
/// Reproduces a schema-ambiguity failure in `PushDownLeafProjections`.
3184+
///
3185+
/// ```text
3186+
/// Optimizer rule 'push_down_leaf_projections' failed
3187+
/// caused by
3188+
/// Schema error: Schema contains qualified field name p.__datafusion_extracted_1
3189+
/// and unqualified field name __datafusion_extracted_1 which would be ambiguous
3190+
/// ```
3191+
///
3192+
/// Four ingredients, each necessary:
3193+
///
3194+
/// 1. a `SubqueryAlias` over each join input (bare aliased table scans pass),
3195+
/// 2. a **left** join (an inner join passes),
3196+
/// 3. a leaf expression on *each* side, so extraction pushes into both inputs,
3197+
/// 4. an aggregate above the join (a projection passes).
3198+
///
3199+
/// With extraction pushed inside both aliases, the left input re-exposes its
3200+
/// alias qualified as `p.__datafusion_extracted_1`, while the recovery
3201+
/// projection above the join rebuilds the same reference unqualified via
3202+
/// `Column::new_unqualified`, so one schema ends up holding both spellings.
3203+
///
3204+
/// Equivalent SQL:
3205+
/// ```sql
3206+
/// CREATE VIEW p AS SELECT user, id FROM test;
3207+
/// CREATE VIEW c AS SELECT user, id FROM other;
3208+
/// SELECT p.user['a'], count(1)
3209+
/// FROM p LEFT JOIN c ON p.id = c.id
3210+
/// WHERE c.user['t'] IS NOT NULL
3211+
/// GROUP BY 1;
3212+
/// ```
3213+
#[test]
3214+
fn test_leaf_expr_on_both_sides_of_left_join_under_aggregate() -> Result<()> {
3215+
use datafusion_expr::JoinType;
3216+
use datafusion_expr::test::function_stub::count;
3217+
3218+
fn view(name: &str, table: LogicalPlan) -> Result<LogicalPlan> {
3219+
LogicalPlanBuilder::from(table)
3220+
.project(vec![col("user"), col("id")])?
3221+
.alias(name)?
3222+
.build()
3223+
}
3224+
3225+
let left = view("p", test_table_scan_with_struct()?)?;
3226+
let right = view("c", test_table_scan_with_struct_named("right")?)?;
3227+
3228+
let plan = LogicalPlanBuilder::from(left)
3229+
.join(right, JoinType::Left, (vec!["id"], vec!["id"]), None)?
3230+
.filter(leaf_udf(col("c.user"), "t").is_not_null())?
3231+
.aggregate(vec![leaf_udf(col("p.user"), "a")], vec![count(lit(1))])?
3232+
.build()?;
3233+
3234+
// The full optimizer, not `format_optimization_stages`: the triggering
3235+
// shape (`p.__datafusion_extracted_1 AS __datafusion_extracted_1`) only
3236+
// appears once projection merging has collapsed the recovery projection
3237+
// over the `SubqueryAlias`.
3238+
let ctx = OptimizerContext::new();
3239+
let optimized = Optimizer::new().optimize(plan, &ctx, |_, _| {})?;
3240+
insta::assert_snapshot!(format!("{optimized}"), @r#"
3241+
Projection: __datafusion_extracted_1 AS leaf_udf(p.user,Utf8("a")), COUNT(Int32(1))
3242+
Aggregate: groupBy=[[__datafusion_extracted_1]], aggr=[[COUNT(Int32(1))]]
3243+
Projection: p.__datafusion_extracted_1 AS __datafusion_extracted_1
3244+
Inner Join: p.id = c.id
3245+
SubqueryAlias: p
3246+
Projection: test.id, leaf_udf(test.user, Utf8("a")) AS __datafusion_extracted_1
3247+
TableScan: test projection=[id, user]
3248+
SubqueryAlias: c
3249+
Projection: right.id
3250+
Filter: __datafusion_extracted_2 IS NOT NULL
3251+
Projection: right.id, leaf_udf(right.user, Utf8("t")) AS __datafusion_extracted_2
3252+
TableScan: right projection=[id, user]
3253+
"#);
3254+
Ok(())
3255+
}
31663256
}

0 commit comments

Comments
 (0)