Skip to content

Commit 6a83c1b

Browse files
committed
fix: reconcile native IF struct fields positionally
1 parent c48a615 commit 6a83c1b

2 files changed

Lines changed: 140 additions & 4 deletions

File tree

‎native/spark-expr/src/conditional_funcs/if_expr.rs‎

Lines changed: 109 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,7 @@ pub fn create_if_expr(
4141
) -> Result<Arc<dyn PhysicalExpr>> {
4242
let true_type = true_expr.data_type(input_schema)?;
4343
let false_type = false_expr.data_type(input_schema)?;
44-
// The coercion that `get_coerce_type_for_case_expression` folds over the branches of a CASE
45-
// WHEN, starting from the ELSE branch. Here the THEN branch goes first, so the common type
46-
// takes its struct field names, as Spark's `If.dataType` does.
47-
let Some(common_type) = type_union_coercion(&true_type, &false_type) else {
44+
let Some(common_type) = if_common_type(&true_type, &false_type) else {
4845
return Ok(Arc::new(IfExpr::new(if_expr, true_expr, false_expr)));
4946
};
5047
Ok(Arc::new(IfExpr::new(
@@ -54,6 +51,47 @@ pub fn create_if_expr(
5451
)))
5552
}
5653

54+
/// Reconciles Spark IF branches positionally, retaining THEN names and merging nullability.
55+
/// Spark has already coerced the branches to the same SQL type. DataFusion's struct union may
56+
/// instead match by name, pairing different positions when names differ only in case.
57+
fn if_common_type(then_type: &DataType, else_type: &DataType) -> Option<DataType> {
58+
use arrow::datatypes::FieldRef;
59+
60+
fn field(then_field: &FieldRef, else_field: &FieldRef) -> Option<FieldRef> {
61+
Some(Arc::new(
62+
then_field
63+
.as_ref()
64+
.clone()
65+
.with_data_type(if_common_type(
66+
then_field.data_type(),
67+
else_field.data_type(),
68+
)?)
69+
.with_nullable(then_field.is_nullable() || else_field.is_nullable()),
70+
))
71+
}
72+
73+
match (then_type, else_type) {
74+
(DataType::Struct(then_fields), DataType::Struct(else_fields)) => {
75+
if then_fields.len() != else_fields.len() {
76+
return None;
77+
}
78+
Some(DataType::Struct(
79+
then_fields
80+
.iter()
81+
.zip(else_fields)
82+
.map(|(t, e)| field(t, e))
83+
.collect::<Option<Vec<_>>>()?
84+
.into(),
85+
))
86+
}
87+
(DataType::List(t), DataType::List(e)) => Some(DataType::List(field(t, e)?)),
88+
(DataType::Map(t, t_sorted), DataType::Map(e, e_sorted)) => {
89+
Some(DataType::Map(field(t, e)?, *t_sorted && *e_sorted))
90+
}
91+
_ => type_union_coercion(then_type, else_type),
92+
}
93+
}
94+
5795
/// Casts an IF branch whose type is `data_type` to the branches' `common_type`.
5896
///
5997
/// The branches share a Spark type, so any difference is in the Arrow representation. For a
@@ -203,6 +241,73 @@ mod tests {
203241
expr.evaluate(batch).unwrap().into_array(2).unwrap()
204242
}
205243

244+
/// Spark matches struct fields by position even when their case-distinct names are reordered.
245+
/// Name-based union pairs the first INT with the second DOUBLE and silently widens it.
246+
#[test]
247+
fn if_reconciles_case_variant_fields_positionally() {
248+
use arrow::array::Float64Array;
249+
250+
let then_fields: arrow::datatypes::Fields = vec![
251+
Field::new("x", DataType::Int32, true),
252+
Field::new("X", DataType::Float64, false),
253+
]
254+
.into();
255+
let else_fields: arrow::datatypes::Fields = vec![
256+
Field::new("X", DataType::Int32, false),
257+
Field::new("x", DataType::Float64, true),
258+
]
259+
.into();
260+
let expected_fields: arrow::datatypes::Fields = vec![
261+
Field::new("x", DataType::Int32, true),
262+
Field::new("X", DataType::Float64, true),
263+
]
264+
.into();
265+
let schema = Schema::new(vec![
266+
Field::new("b", DataType::Boolean, false),
267+
Field::new("t", DataType::Struct(then_fields.clone()), false),
268+
Field::new("e", DataType::Struct(else_fields.clone()), false),
269+
]);
270+
let batch = RecordBatch::try_new(
271+
Arc::new(schema.clone()),
272+
vec![
273+
Arc::new(BooleanArray::from(vec![true, false])),
274+
Arc::new(StructArray::new(
275+
then_fields,
276+
vec![
277+
Arc::new(Int32Array::from(vec![Some(7), None])),
278+
Arc::new(Float64Array::from(vec![5.5, 5.5])),
279+
],
280+
None,
281+
)),
282+
Arc::new(StructArray::new(
283+
else_fields,
284+
vec![
285+
Arc::new(Int32Array::from(vec![0, 0])),
286+
Arc::new(Float64Array::from(vec![Some(9.5), None])),
287+
],
288+
None,
289+
)),
290+
],
291+
)
292+
.unwrap();
293+
let c = |name: &str| col(name, &schema).unwrap();
294+
let expr = create_if_expr(c("b"), c("t"), c("e"), &schema).unwrap();
295+
assert_eq!(
296+
expr.data_type(&schema).unwrap(),
297+
DataType::Struct(expected_fields.clone())
298+
);
299+
let result = evaluate_if(&expr, &batch);
300+
let expected = StructArray::new(
301+
expected_fields,
302+
vec![
303+
Arc::new(Int32Array::from(vec![7, 0])),
304+
Arc::new(Float64Array::from(vec![Some(5.5), None])),
305+
],
306+
None,
307+
);
308+
assert_eq!(result.as_ref(), &expected);
309+
}
310+
206311
/// IF branches that share a Spark timestamp type but carry different Arrow timezone labels are
207312
/// reconciled by relabelling one of them, as CASE branches are. The result has the THEN
208313
/// branch's label, which `IfExpr` reported before the ELSE branch was cast, unless it has none.

‎spark/src/test/resources/sql-tests/expressions/conditional/if_nested_nullability.sql‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,3 +111,34 @@ SELECT IF(q, array(i), array(0)) FROM test_if_nested
111111
-- CASE WHEN already casts its branches to a common type
112112
query
113113
SELECT CASE WHEN q THEN named_struct('x', i) ELSE named_struct('x', 0) END FROM test_if_nested
114+
115+
-- Case-distinct names repeat across positions in the opposite order. Spark aligns fields by
116+
-- position; name-based type union would pair INT with DOUBLE and make to_json emit 7.0.
117+
statement
118+
CREATE TABLE test_if_positional(q boolean, i int, d double) USING parquet
119+
120+
statement
121+
INSERT INTO test_if_positional VALUES (true, 7, 9.5), (true, NULL, NULL)
122+
123+
-- Pin the all-THEN wrong result before adding rows that exercise ELSE and mixed batches.
124+
query expect_native(if)
125+
SELECT to_json(IF(q, named_struct('x', i, 'X', CAST(5.5 AS DOUBLE)), named_struct('X', 0, 'x', d))) FROM test_if_positional
126+
127+
statement
128+
INSERT INTO test_if_positional VALUES (false, 7, 9.5), (false, NULL, NULL)
129+
130+
statement
131+
INSERT INTO test_if_positional VALUES (true, 7, 9.5), (false, 8, 10.5), (NULL, NULL, NULL)
132+
133+
query expect_native(if)
134+
SELECT to_json(IF(q, named_struct('x', i, 'X', CAST(5.5 AS DOUBLE)), named_struct('X', 0, 'x', d))), to_json(IF(q, named_struct('X', 0, 'x', d), named_struct('x', i, 'X', CAST(5.5 AS DOUBLE)))) FROM test_if_positional
135+
136+
-- Native to_json supports structs, so extract the struct after reconciling the array or map.
137+
query expect_native(if)
138+
SELECT to_json(IF(q, array(named_struct('x', i, 'X', CAST(5.5 AS DOUBLE))), array(named_struct('X', 0, 'x', d)))[0]) FROM test_if_positional
139+
140+
query expect_native(if)
141+
SELECT to_json(IF(q, map('k', named_struct('x', i, 'X', CAST(5.5 AS DOUBLE))), map('k', named_struct('X', 0, 'x', d)))['k']) FROM test_if_positional
142+
143+
query expect_native(if)
144+
SELECT to_json(IF(q, named_struct('s', named_struct('x', i, 'X', CAST(5.5 AS DOUBLE))), named_struct('s', named_struct('X', 0, 'x', d)))) FROM test_if_positional

0 commit comments

Comments
 (0)