Skip to content

Commit ee1b213

Browse files
committed
fix: reconcile native IF struct fields positionally
1 parent 4101f81 commit ee1b213

2 files changed

Lines changed: 140 additions & 4 deletions

File tree

‎native/spark-expr/src/conditional_funcs/case_when.rs‎

Lines changed: 109 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -86,10 +86,7 @@ pub fn create_if_expr(
8686
) -> Result<Arc<dyn PhysicalExpr>> {
8787
let true_type = true_expr.data_type(input_schema)?;
8888
let false_type = false_expr.data_type(input_schema)?;
89-
// The coercion that `get_coerce_type_for_case_expression` folds over the branches of a CASE
90-
// WHEN, starting from the ELSE branch. Here the THEN branch goes first, so the common type
91-
// takes its struct field names, as Spark's `If.dataType` does.
92-
let Some(common_type) = type_union_coercion(&true_type, &false_type) else {
89+
let Some(common_type) = if_common_type(&true_type, &false_type) else {
9390
return Ok(Arc::new(IfExpr::new(if_expr, true_expr, false_expr)));
9491
};
9592
Ok(Arc::new(IfExpr::new(
@@ -99,6 +96,47 @@ pub fn create_if_expr(
9996
)))
10097
}
10198

99+
/// Reconciles Spark IF branches positionally, retaining THEN names and merging nullability.
100+
/// Spark has already coerced the branches to the same SQL type. DataFusion's struct union may
101+
/// instead match by name, pairing different positions when names differ only in case.
102+
fn if_common_type(then_type: &DataType, else_type: &DataType) -> Option<DataType> {
103+
use arrow::datatypes::FieldRef;
104+
105+
fn field(then_field: &FieldRef, else_field: &FieldRef) -> Option<FieldRef> {
106+
Some(Arc::new(
107+
then_field
108+
.as_ref()
109+
.clone()
110+
.with_data_type(if_common_type(
111+
then_field.data_type(),
112+
else_field.data_type(),
113+
)?)
114+
.with_nullable(then_field.is_nullable() || else_field.is_nullable()),
115+
))
116+
}
117+
118+
match (then_type, else_type) {
119+
(DataType::Struct(then_fields), DataType::Struct(else_fields)) => {
120+
if then_fields.len() != else_fields.len() {
121+
return None;
122+
}
123+
Some(DataType::Struct(
124+
then_fields
125+
.iter()
126+
.zip(else_fields)
127+
.map(|(t, e)| field(t, e))
128+
.collect::<Option<Vec<_>>>()?
129+
.into(),
130+
))
131+
}
132+
(DataType::List(t), DataType::List(e)) => Some(DataType::List(field(t, e)?)),
133+
(DataType::Map(t, t_sorted), DataType::Map(e, e_sorted)) => {
134+
Some(DataType::Map(field(t, e)?, *t_sorted && *e_sorted))
135+
}
136+
_ => type_union_coercion(then_type, else_type),
137+
}
138+
}
139+
102140
/// Casts a CASE WHEN or IF branch whose type is `data_type` to the branches' `common_type`.
103141
///
104142
/// The branches share a Spark type, so any difference is in the Arrow representation. For a
@@ -1320,6 +1358,73 @@ mod tests {
13201358
expr.evaluate(batch).unwrap().into_array(2).unwrap()
13211359
}
13221360

1361+
/// Spark matches struct fields by position even when their case-distinct names are reordered.
1362+
/// Name-based union pairs the first INT with the second DOUBLE and silently widens it.
1363+
#[test]
1364+
fn if_reconciles_case_variant_fields_positionally() {
1365+
use arrow::array::Float64Array;
1366+
1367+
let then_fields: arrow::datatypes::Fields = vec![
1368+
Field::new("x", DataType::Int32, true),
1369+
Field::new("X", DataType::Float64, false),
1370+
]
1371+
.into();
1372+
let else_fields: arrow::datatypes::Fields = vec![
1373+
Field::new("X", DataType::Int32, false),
1374+
Field::new("x", DataType::Float64, true),
1375+
]
1376+
.into();
1377+
let expected_fields: arrow::datatypes::Fields = vec![
1378+
Field::new("x", DataType::Int32, true),
1379+
Field::new("X", DataType::Float64, true),
1380+
]
1381+
.into();
1382+
let schema = Schema::new(vec![
1383+
Field::new("b", DataType::Boolean, false),
1384+
Field::new("t", DataType::Struct(then_fields.clone()), false),
1385+
Field::new("e", DataType::Struct(else_fields.clone()), false),
1386+
]);
1387+
let batch = RecordBatch::try_new(
1388+
Arc::new(schema.clone()),
1389+
vec![
1390+
Arc::new(BooleanArray::from(vec![true, false])),
1391+
Arc::new(StructArray::new(
1392+
then_fields,
1393+
vec![
1394+
Arc::new(Int32Array::from(vec![Some(7), None])),
1395+
Arc::new(Float64Array::from(vec![5.5, 5.5])),
1396+
],
1397+
None,
1398+
)),
1399+
Arc::new(StructArray::new(
1400+
else_fields,
1401+
vec![
1402+
Arc::new(Int32Array::from(vec![0, 0])),
1403+
Arc::new(Float64Array::from(vec![Some(9.5), None])),
1404+
],
1405+
None,
1406+
)),
1407+
],
1408+
)
1409+
.unwrap();
1410+
let c = |name: &str| col(name, &schema).unwrap();
1411+
let expr = create_if_expr(c("b"), c("t"), c("e"), &schema).unwrap();
1412+
assert_eq!(
1413+
expr.data_type(&schema).unwrap(),
1414+
DataType::Struct(expected_fields.clone())
1415+
);
1416+
let result = evaluate_if(&expr, &batch);
1417+
let expected = StructArray::new(
1418+
expected_fields,
1419+
vec![
1420+
Arc::new(Int32Array::from(vec![7, 0])),
1421+
Arc::new(Float64Array::from(vec![Some(5.5), None])),
1422+
],
1423+
None,
1424+
);
1425+
assert_eq!(result.as_ref(), &expected);
1426+
}
1427+
13231428
/// IF branches that share a Spark timestamp type but carry different Arrow timezone labels are
13241429
/// reconciled by relabelling one of them, as CASE branches are. The result has the THEN
13251430
/// branch's label, which `IfExpr` reported before the ELSE branch was cast, unless it has none.

‎spark/src/test/resources/sql-tests/expressions/conditional/if_nested_nullability.sql‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,3 +111,34 @@ SELECT IF(q, array(i), array(0)) FROM test_if_nested
111111
-- CASE WHEN already casts its branches to a common type
112112
query
113113
SELECT CASE WHEN q THEN named_struct('x', i) ELSE named_struct('x', 0) END FROM test_if_nested
114+
115+
-- Case-distinct names repeat across positions in the opposite order. Spark aligns fields by
116+
-- position; name-based type union would pair INT with DOUBLE and make to_json emit 7.0.
117+
statement
118+
CREATE TABLE test_if_positional(q boolean, i int, d double) USING parquet
119+
120+
statement
121+
INSERT INTO test_if_positional VALUES (true, 7, 9.5), (true, NULL, NULL)
122+
123+
-- Pin the all-THEN wrong result before adding rows that exercise ELSE and mixed batches.
124+
query expect_native(if)
125+
SELECT to_json(IF(q, named_struct('x', i, 'X', CAST(5.5 AS DOUBLE)), named_struct('X', 0, 'x', d))) FROM test_if_positional
126+
127+
statement
128+
INSERT INTO test_if_positional VALUES (false, 7, 9.5), (false, NULL, NULL)
129+
130+
statement
131+
INSERT INTO test_if_positional VALUES (true, 7, 9.5), (false, 8, 10.5), (NULL, NULL, NULL)
132+
133+
query expect_native(if)
134+
SELECT to_json(IF(q, named_struct('x', i, 'X', CAST(5.5 AS DOUBLE)), named_struct('X', 0, 'x', d))), to_json(IF(q, named_struct('X', 0, 'x', d), named_struct('x', i, 'X', CAST(5.5 AS DOUBLE)))) FROM test_if_positional
135+
136+
-- Native to_json supports structs, so extract the struct after reconciling the array or map.
137+
query expect_native(if)
138+
SELECT to_json(IF(q, array(named_struct('x', i, 'X', CAST(5.5 AS DOUBLE))), array(named_struct('X', 0, 'x', d)))[0]) FROM test_if_positional
139+
140+
query expect_native(if)
141+
SELECT to_json(IF(q, map('k', named_struct('x', i, 'X', CAST(5.5 AS DOUBLE))), map('k', named_struct('X', 0, 'x', d)))['k']) FROM test_if_positional
142+
143+
query expect_native(if)
144+
SELECT to_json(IF(q, named_struct('s', named_struct('x', i, 'X', CAST(5.5 AS DOUBLE))), named_struct('s', named_struct('X', 0, 'x', d)))) FROM test_if_positional

0 commit comments

Comments
 (0)