Skip to content

Commit f73f01e

Browse files
sgrebnovnuno-faria
andauthored
fix: Ensure columns are casted to the correct names with Unions (apache#20146) (#127)
UPSTREAM NOTE: not required, DF original PR: from apache#20146 ## Which issue does this PR close? - Closes apache#20123. ## Rationale for this change When aliasing a cast, use the original name to prevent duplicate column names errors. ## What changes are included in this PR? - Updated `coerce_exprs_for_schema` to use the correct name. - Updated tests. ## Are these changes tested? Yes. ## Are there any user-facing changes? No. # Conflicts: # datafusion/expr/src/expr_rewriter/mod.rs # datafusion/optimizer/tests/optimizer_integration.rs # datafusion/substrait/tests/cases/logical_plans.rs Co-authored-by: Nuno Faria <[email protected]>
1 parent 5094428 commit f73f01e

3 files changed

Lines changed: 72 additions & 32 deletions

File tree

‎datafusion/expr/src/expr_rewriter/mod.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,18 @@ fn coerce_exprs_for_schema(
255255
}
256256
#[expect(deprecated)]
257257
Expr::Wildcard { .. } => Ok(expr),
258-
_ => expr.cast_to(new_type, src_schema),
258+
_ => {
259+
match expr {
260+
// maintain the original name when casting a column, to avoid the
261+
// tablename being added to it when not explicitly set by the query
262+
// (see: https://github.com/apache/datafusion/issues/18818)
263+
Expr::Column(ref column) => {
264+
let name = column.name().to_owned();
265+
Ok(expr.cast_to(new_type, src_schema)?.alias(name))
266+
}
267+
_ => Ok(expr.cast_to(new_type, src_schema)?),
268+
}
269+
}
259270
}
260271
} else {
261272
Ok(expr)

‎datafusion/optimizer/tests/optimizer_integration.rs‎

Lines changed: 29 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -536,14 +536,15 @@ fn recursive_cte_projection_pushdown() -> Result<()> {
536536
// columns from the base table and recursive table, eliminating unused columns
537537
assert_snapshot!(
538538
format!("{plan}"),
539-
@r#"SubqueryAlias: nodes
540-
RecursiveQuery: is_distinct=false
541-
Projection: test.col_int32 AS id
542-
TableScan: test projection=[col_int32]
543-
Projection: CAST(CAST(nodes.id AS Int64) + Int64(1) AS Int32)
544-
Filter: nodes.id < Int32(3)
545-
TableScan: nodes projection=[id]
546-
"#
539+
@r"
540+
SubqueryAlias: nodes
541+
RecursiveQuery: is_distinct=false
542+
Projection: test.col_int32 AS id
543+
TableScan: test projection=[col_int32]
544+
Projection: CAST(CAST(nodes.id AS Int64) + Int64(1) AS Int32)
545+
Filter: nodes.id < Int32(3)
546+
TableScan: nodes projection=[id]
547+
"
547548
);
548549
Ok(())
549550
}
@@ -559,14 +560,16 @@ fn recursive_cte_with_aliased_self_reference() -> Result<()> {
559560

560561
assert_snapshot!(
561562
format!("{plan}"),
562-
@r#"SubqueryAlias: nodes
563-
RecursiveQuery: is_distinct=false
564-
Projection: test.col_int32 AS id
565-
TableScan: test projection=[col_int32]
566-
Projection: CAST(CAST(child.id AS Int64) + Int64(1) AS Int32)
567-
SubqueryAlias: child
568-
Filter: nodes.id < Int32(3)
569-
TableScan: nodes projection=[id]"#,
563+
@r"
564+
SubqueryAlias: nodes
565+
RecursiveQuery: is_distinct=false
566+
Projection: test.col_int32 AS id
567+
TableScan: test projection=[col_int32]
568+
Projection: CAST(CAST(child.id AS Int64) + Int64(1) AS Int32)
569+
SubqueryAlias: child
570+
Filter: nodes.id < Int32(3)
571+
TableScan: nodes projection=[id]
572+
",
570573
);
571574
Ok(())
572575
}
@@ -618,15 +621,16 @@ fn recursive_cte_projection_pushdown_baseline() -> Result<()> {
618621
// and only the needed column is selected from the recursive table
619622
assert_snapshot!(
620623
format!("{plan}"),
621-
@r#"SubqueryAlias: countdown
622-
RecursiveQuery: is_distinct=false
623-
Projection: test.col_int32 AS n
624-
Filter: test.col_int32 = Int32(5)
625-
TableScan: test projection=[col_int32]
626-
Projection: CAST(CAST(countdown.n AS Int64) - Int64(1) AS Int32)
627-
Filter: countdown.n > Int32(1)
628-
TableScan: countdown projection=[n]
629-
"#
624+
@r"
625+
SubqueryAlias: countdown
626+
RecursiveQuery: is_distinct=false
627+
Projection: test.col_int32 AS n
628+
Filter: test.col_int32 = Int32(5)
629+
TableScan: test projection=[col_int32]
630+
Projection: CAST(CAST(countdown.n AS Int64) - Int64(1) AS Int32)
631+
Filter: countdown.n > Int32(1)
632+
TableScan: countdown projection=[n]
633+
"
630634
);
631635
Ok(())
632636
}

‎datafusion/substrait/tests/cases/logical_plans.rs‎

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -217,12 +217,37 @@ mod tests {
217217

218218
assert_snapshot!(
219219
plan,
220-
@r#"
221-
Projection: lower(sales.product) AS lower(product), sum(count(sales.product)) AS product_count
222-
Aggregate: groupBy=[[sales.product]], aggr=[[sum(count(sales.product))]]
223-
Aggregate: groupBy=[[sales.product]], aggr=[[count(sales.product)]]
224-
TableScan: sales
225-
"#
220+
@r"
221+
Projection: lower(sales.product) AS lower(product), sum(count(sales.product)) AS product_count
222+
Aggregate: groupBy=[[sales.product]], aggr=[[sum(count(sales.product))]]
223+
Aggregate: groupBy=[[sales.product]], aggr=[[count(sales.product)]]
224+
TableScan: sales
225+
"
226+
);
227+
228+
// Trigger execution to ensure plan validity
229+
DataFrame::new(ctx.state(), plan).show().await?;
230+
231+
Ok(())
232+
}
233+
234+
#[tokio::test]
235+
async fn duplicate_name_in_union() -> Result<()> {
236+
let proto_plan =
237+
read_json("tests/testdata/test_plans/duplicate_name_in_union.substrait.json");
238+
let ctx = add_plan_schemas_to_ctx(SessionContext::new(), &proto_plan)?;
239+
let plan = from_substrait_plan(&ctx.state(), &proto_plan).await?;
240+
241+
assert_snapshot!(
242+
plan,
243+
@r"
244+
Projection: foo AS col1, bar AS col2
245+
Union
246+
Projection: foo, bar
247+
Values: (Int64(100), Int64(200))
248+
Projection: x, foo
249+
Values: (Int32(300), Int64(400))
250+
"
226251
);
227252

228253
// Trigger execution to ensure plan validity

0 commit comments

Comments
 (0)