Skip to content

Commit 2277aa4

Browse files
alambkumarUjjawal
andauthored
[branch-56] fix: infer UNION output types before analysis (backport #27139) (#27178)
## Which issue does this PR close? - Backport of #27139 to `branch-56` - Part of #24461 - Closes #27023 Co-authored-by: Kumar Ujjawal <[email protected]>
1 parent 86bb0d3 commit 2277aa4

11 files changed

Lines changed: 615 additions & 135 deletions

File tree

‎datafusion/core/tests/sql/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ mod sql_api;
6161
mod tpcds_planning;
6262
mod union_comparison;
6363
mod union_nullable;
64+
mod union_schema;
6465
mod unparser;
6566

6667
async fn register_aggregate_csv_by_sql(ctx: &SessionContext) {

‎datafusion/core/tests/sql/union_nullable.rs‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,8 @@ async fn union_all_mismatched_nullable(
6161
.await
6262
}
6363

64-
/// The schema DataFusion actually commits to for a query: the logical plan
65-
/// after the `Analyzer` (which includes the `UNION` nullability/type
66-
/// coercion this test targets) and `Optimizer` have run. `DataFrame::schema`
67-
/// alone is not enough here -- it reflects the raw, pre-`Analyzer` plan (see
68-
/// `SessionState::create_logical_plan`), which for a `UNION` still has the
69-
/// first leg's un-coerced type.
64+
/// This test checks the schema after the Analyzer and Optimizer run,
65+
/// because they can change nullability.
7066
fn analyzed_schema(df: &DataFrame) -> Result<Schema> {
7167
Ok(df
7268
.clone()
Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3+
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4+
5+
//! The pre-analysis UNION schema must describe the output types (#27023).
6+
7+
use arrow::datatypes::{DataType, TimeUnit};
8+
use datafusion::prelude::*;
9+
use datafusion_common::{Result, ScalarValue};
10+
11+
fn assert_union_schema(df: &DataFrame, expected: &[(&str, DataType)]) {
12+
let fields = df.schema().fields();
13+
assert_eq!(fields.len(), expected.len());
14+
for (field, (name, data_type)) in fields.iter().zip(expected) {
15+
assert_eq!(field.name(), name);
16+
assert_eq!(field.data_type(), data_type);
17+
}
18+
}
19+
20+
#[tokio::test]
21+
async fn union_infers_types_before_analysis() -> Result<()> {
22+
let ctx = SessionContext::new();
23+
let cases = [
24+
(
25+
r#"arrow_cast(arrow_cast(1, 'Timestamp(Microsecond, None)'), 'Timestamp(Microsecond, Some("UTC"))')"#,
26+
"CAST(NULL AS TIMESTAMP)",
27+
DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())),
28+
),
29+
(
30+
"arrow_cast('a', 'Utf8')",
31+
"arrow_cast('b', 'LargeUtf8')",
32+
DataType::LargeUtf8,
33+
),
34+
(
35+
"arrow_cast('a', 'Utf8')",
36+
"arrow_cast('b', 'Utf8View')",
37+
DataType::Utf8View,
38+
),
39+
("CAST(1 AS SMALLINT)", "2.5", DataType::Float64),
40+
(
41+
"CASE WHEN false THEN 1 ELSE 2.5 END",
42+
"NULL",
43+
DataType::Float64,
44+
),
45+
(
46+
"CASE WHEN false THEN 1 WHEN true THEN 2.5 END",
47+
"NULL",
48+
DataType::Float64,
49+
),
50+
];
51+
for (left, right, expected) in cases {
52+
for (left, right) in [(left, right), (right, left)] {
53+
for operation in ["ALL", "DISTINCT", "ALL BY NAME", "BY NAME"] {
54+
let sql = format!(
55+
"SELECT x FROM (SELECT {left} AS x UNION {operation} SELECT {right} AS x) u"
56+
);
57+
assert_union_schema(&ctx.sql(&sql).await?, &[("x", expected.clone())]);
58+
}
59+
}
60+
}
61+
Ok(())
62+
}
63+
64+
#[tokio::test]
65+
async fn union_infers_dictionary_types_before_analysis() -> Result<()> {
66+
let ctx = SessionContext::new();
67+
for (sql, expected) in [
68+
(
69+
"SELECT NULL AS x UNION ALL SELECT arrow_cast('a', 'Dictionary(Int32, Utf8)') AS x",
70+
DataType::Utf8,
71+
),
72+
(
73+
"SELECT CAST(1 AS SMALLINT) AS x UNION ALL SELECT arrow_cast(2, 'Dictionary(Int32, Int32)') AS x",
74+
DataType::Int32,
75+
),
76+
] {
77+
assert_union_schema(&ctx.sql(sql).await?, &[("x", expected)]);
78+
}
79+
for sql in [
80+
"SELECT 1 AS y UNION ALL BY NAME SELECT arrow_cast('a', 'Dictionary(Int32, Utf8)') AS x",
81+
"SELECT arrow_cast('a', 'Dictionary(Int32, Utf8)') AS x UNION ALL BY NAME SELECT 1 AS y",
82+
] {
83+
let df = ctx.sql(sql).await?.select_columns(&["x", "y"])?;
84+
assert_union_schema(&df, &[("x", DataType::Utf8), ("y", DataType::Int64)]);
85+
}
86+
Ok(())
87+
}
88+
89+
#[tokio::test]
90+
async fn union_dataframe_and_by_name_schemas() -> Result<()> {
91+
let ctx = SessionContext::new();
92+
for variant in 0..4 {
93+
let left = ctx.sql("SELECT CAST(1 AS SMALLINT) AS x").await?;
94+
let right = ctx.sql("SELECT 2.5 AS x").await?;
95+
let df = match variant {
96+
0 => left.union(right)?,
97+
1 => left.union_distinct(right)?,
98+
2 => left.union_by_name(right)?,
99+
_ => left.union_by_name_distinct(right)?,
100+
};
101+
assert_union_schema(&df.select_columns(&["x"])?, &[("x", DataType::Float64)]);
102+
}
103+
for sql in [
104+
"SELECT CAST(1 AS SMALLINT) AS x, 1 AS y UNION ALL BY NAME SELECT 2 AS y, 2.5 AS x",
105+
"SELECT CAST(1 AS SMALLINT) AS x UNION ALL BY NAME SELECT 2.5 AS x, 2 AS y",
106+
] {
107+
assert_union_schema(
108+
&ctx.sql(sql).await?,
109+
&[("x", DataType::Float64), ("y", DataType::Int64)],
110+
);
111+
}
112+
let df = ctx
113+
.sql(
114+
"SELECT CAST(NULL AS SMALLINT) AS x WHERE false \
115+
UNION ALL SELECT CAST(NULL AS BIGINT) AS x WHERE false",
116+
)
117+
.await?;
118+
assert_union_schema(&df, &[("x", DataType::Int64)]);
119+
let df = ctx
120+
.sql(
121+
"SELECT max(x) AS x FROM (SELECT 1 AS x UNION ALL SELECT 2.5 AS x \
122+
UNION ALL SELECT 3 AS x) u",
123+
)
124+
.await?;
125+
assert_union_schema(&df, &[("x", DataType::Float64)]);
126+
Ok(())
127+
}
128+
129+
#[tokio::test]
130+
async fn union_bound_parameters_keep_inferred_type() -> Result<()> {
131+
let ctx = SessionContext::new();
132+
let df = ctx
133+
.sql("SELECT CAST($1 AS INT) AS x UNION ALL SELECT 2.5 AS x")
134+
.await?
135+
.with_param_values(vec![ScalarValue::Int32(Some(1))])?;
136+
assert_union_schema(&df, &[("x", DataType::Float64)]);
137+
Ok(())
138+
}

‎datafusion/expr/src/expr_schema.rs‎

Lines changed: 55 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ use crate::expr::{FieldMetadata, LambdaVariable};
1313
use crate::higher_order_function::HigherOrderReturnFieldArgs;
1414
use crate::type_coercion::functions::value_fields_with_higher_order_udf_and_lambdas;
1515
use crate::type_coercion::functions::{UDFCoercionExt, fields_with_udf};
16+
use crate::type_coercion::other::get_coerce_type_for_case_expression;
1617
use crate::udf::ReturnFieldArgs;
1718
use crate::{
1819
LogicalPlan, Operator, Projection, Subquery, WindowFunctionDefinition, utils,
@@ -33,6 +34,10 @@ use std::sync::Arc;
3334
/// Trait to allow expr to typable with respect to a schema
3435
pub trait ExprSchemable {
3536
/// Given a schema, return the type of the expr
37+
///
38+
/// `CASE` expressions infer a common type from all `THEN` and `ELSE`
39+
/// branches without inserting casts. Incompatible branches retain a
40+
/// provisional type until analysis.
3641
fn get_type(&self, schema: &dyn ExprSchema) -> Result<DataType>;
3742

3843
/// Given a schema, return the nullability of the expr
@@ -177,15 +182,28 @@ impl ExprSchemable for Expr {
177182
Expr::ScalarVariable(field, _) => Ok(field.data_type().clone()),
178183
Expr::Literal(l, _) => Ok(l.data_type()),
179184
Expr::Case(case) => {
180-
for (_, then_expr) in &case.when_then_expr {
181-
let then_type = then_expr.get_type(schema)?;
182-
if !then_type.is_null() {
183-
return Ok(then_type);
184-
}
185-
}
186-
case.else_expr
185+
let then_types = case
186+
.when_then_expr
187+
.iter()
188+
.map(|(_, expr)| expr.get_type(schema))
189+
.collect::<Result<Vec<_>>>()?;
190+
let else_type = case
191+
.else_expr
187192
.as_ref()
188-
.map_or(Ok(DataType::Null), |e| e.get_type(schema))
193+
.map(|expr| expr.get_type(schema))
194+
.transpose()?;
195+
// If the branches have no common type, keep the old provisional
196+
// type. The analyzer reports the error.
197+
let common_type =
198+
get_coerce_type_for_case_expression(&then_types, else_type.as_ref());
199+
let result_type = common_type.unwrap_or_else(|| {
200+
then_types
201+
.into_iter()
202+
.find(|t| !t.is_null())
203+
.or(else_type)
204+
.unwrap_or(DataType::Null)
205+
});
206+
Ok(result_type)
189207
}
190208
Expr::Cast(Cast { field, .. }) | Expr::TryCast(TryCast { field, .. }) => {
191209
Ok(field.data_type().clone())
@@ -1684,4 +1702,33 @@ mod tests {
16841702
}
16851703
}
16861704
}
1705+
1706+
#[test]
1707+
fn case_infers_all_result_types_without_casting() -> Result<()> {
1708+
use crate::expr::Case;
1709+
let schema = DFSchema::empty();
1710+
let nested = Expr::Case(Case::new(
1711+
None,
1712+
vec![(Box::new(lit(false)), Box::new(lit(1_i64)))],
1713+
Some(Box::new(lit(2.5))),
1714+
));
1715+
for (then_values, else_value, expected) in [
1716+
(vec![lit(1_i64)], Some(lit(2.5)), DataType::Float64),
1717+
(vec![lit(1_i64), lit(2.5)], None, DataType::Float64),
1718+
(vec![lit(2.5), lit(1_i64)], None, DataType::Float64),
1719+
(vec![lit(2.5)], Some(lit(1_i64)), DataType::Float64),
1720+
(vec![lit(3_i64)], Some(nested), DataType::Float64),
1721+
(vec![lit(ScalarValue::Null)], None, DataType::Null),
1722+
// Incompatible raw types retain a provisional type for the analyzer.
1723+
(vec![lit(true)], Some(lit("a")), DataType::Boolean),
1724+
] {
1725+
let when_then = then_values
1726+
.into_iter()
1727+
.map(|value| (Box::new(lit(true)), Box::new(value)))
1728+
.collect();
1729+
let expr = Expr::Case(Case::new(None, when_then, else_value.map(Box::new)));
1730+
assert_eq!(expr.get_type(&schema)?, expected);
1731+
}
1732+
Ok(())
1733+
}
16871734
}

‎datafusion/expr/src/logical_plan/builder.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2189,8 +2189,8 @@ pub fn validate_unique_names<'a>(
21892189

21902190
/// Union two [`LogicalPlan`]s.
21912191
///
2192-
/// Constructs the UNION plan, but does not perform type-coercion. Therefore the
2193-
/// subtree expressions will not be properly typed until the optimizer pass.
2192+
/// Infers common output types, but does not insert casts into the inputs.
2193+
/// The analyzer must still resolve input expressions and insert any required casts.
21942194
///
21952195
/// If a properly typed UNION plan is needed, refer to [`TypeCoercionRewriter::coerce_union`]
21962196
/// or alternatively, merge the union input schema using [`coerce_union_schema`] and

0 commit comments

Comments
 (0)