Skip to content

Commit 7645048

Browse files
committed
fix: cast native IF branches to a common type like CASE WHEN
Spark adds no cast to IF when its branches differ only in the nullability of a nested field, and the native IfExpr returns a branch's array unchanged when every row of a batch takes that branch, so the projection failed with "column types must match schema types". Cast a branch whose Arrow type differs from the common type, as the planner already does for CASE WHEN.
1 parent c7aa38b commit 7645048

3 files changed

Lines changed: 142 additions & 2 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ use datafusion::common::{
107107
JoinType as DFJoinType, NullEquality, ScalarValue,
108108
};
109109
use datafusion::datasource::listing::PartitionedFile;
110+
use datafusion::logical_expr::type_coercion::binary::type_union_coercion;
110111
use datafusion::logical_expr::type_coercion::functions::fields_with_udf;
111112
use datafusion::logical_expr::type_coercion::other::get_coerce_type_for_case_expression;
112113
use datafusion::logical_expr::{
@@ -802,8 +803,35 @@ impl PhysicalPlanner {
802803
let true_expr =
803804
self.create_expr(expr.true_expr.as_ref().unwrap(), Arc::clone(&input_schema))?;
804805
let false_expr =
805-
self.create_expr(expr.false_expr.as_ref().unwrap(), input_schema)?;
806-
Ok(Arc::new(IfExpr::new(if_expr, true_expr, false_expr)))
806+
self.create_expr(expr.false_expr.as_ref().unwrap(), Arc::clone(&input_schema))?;
807+
// Spark adds no cast when the branches differ only in whether a nested field can
808+
// be NULL, but `IfExpr` reports the THEN branch's type and returns the ELSE
809+
// branch's array unchanged when no row of a batch takes the THEN branch. So cast a
810+
// branch whose type differs from the common type, as `create_case_expr` does for
811+
// CASE WHEN. The THEN branch goes first so that the common type keeps its field
812+
// names, as Spark's `If` does.
813+
let true_type = true_expr.data_type(&input_schema)?;
814+
let false_type = false_expr.data_type(&input_schema)?;
815+
let common_type = type_union_coercion(&true_type, &false_type);
816+
let coerce = |branch, data_type: &DataType| -> Arc<dyn PhysicalExpr> {
817+
match &common_type {
818+
// The branches share a Spark type, so a timestamp can only differ in its
819+
// label, and every TimestampType in a native plan is labelled UTC
820+
Some(common_type) if data_type != common_type => Arc::new(Cast::new(
821+
branch,
822+
common_type.clone(),
823+
SparkCastOptions::new(EvalMode::Legacy, "UTC", false),
824+
None,
825+
None,
826+
)),
827+
_ => branch,
828+
}
829+
};
830+
Ok(Arc::new(IfExpr::new(
831+
if_expr,
832+
coerce(true_expr, &true_type),
833+
coerce(false_expr, &false_type),
834+
)))
807835
}
808836
ExprStruct::NormalizeNanAndZero(expr) => {
809837
let child = self.create_expr(expr.child.as_ref().unwrap(), input_schema)?;
Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- Spark adds no cast to IF when its branches differ only in whether a nested struct field, map
19+
-- value or array element can be NULL, so the native planner has to cast a branch to the common
20+
-- type, as it does for CASE WHEN. Without the cast, a batch in which every row took the ELSE
21+
-- branch returned the ELSE array with the other nullability, and the query failed with
22+
-- "column types must match schema types". A batch that mixed the two failed when the THEN branch
23+
-- was the one that could not be NULL.
24+
-- https://github.com/apache/datafusion-comet/issues/6334
25+
--
26+
-- The harness disables ConstantFolding, so these cover the constructor path. The folded map
27+
-- literal path is covered in CometMapExpressionSuite.
28+
29+
statement
30+
CREATE TABLE test_if_nested(q boolean, i int, m map<string, int>, s struct<x: int>, ms map<string, struct<x: int>>) USING parquet
31+
32+
-- Each INSERT writes its own files, and a batch never spans files. Every row of this one takes the
33+
-- THEN branch of IF(q, ...) and the ELSE branch of IF(m IS NULL, ...).
34+
statement
35+
INSERT INTO test_if_nested VALUES (true, 1, map('a', 1), named_struct('x', 1), map('a', named_struct('x', 1))), (true, NULL, map('b', CAST(NULL AS INT)), named_struct('x', CAST(NULL AS INT)), map('b', CAST(NULL AS STRUCT<x: INT>)))
36+
37+
-- Every row of this one takes the ELSE branch of IF(q, ...) and the THEN branch of IF(m IS NULL, ...)
38+
statement
39+
INSERT INTO test_if_nested VALUES (false, 2, NULL, NULL, NULL), (NULL, NULL, NULL, NULL, NULL)
40+
41+
-- Pairs of rows that take different branches, so these batches mix the two
42+
statement
43+
INSERT INTO test_if_nested VALUES (true, 3, map('c', 3), named_struct('x', 3), map('c', named_struct('x', 3))), (false, 4, NULL, NULL, NULL), (true, NULL, map('d', CAST(NULL AS INT)), named_struct('x', CAST(NULL AS INT)), map('d', named_struct('x', CAST(NULL AS INT)))), (NULL, 6, NULL, NULL, NULL), (true, 7, map('e', 7), named_struct('x', 7), map('e', CAST(NULL AS STRUCT<x: INT>))), (false, NULL, NULL, NULL, NULL), (true, 9, map('f', 9), named_struct('x', 9), map('f', named_struct('x', 9))), (false, 10, NULL, NULL, NULL), (true, 11, map('g', 11), named_struct('x', 11), map('g', named_struct('x', 11))), (NULL, 12, NULL, NULL, NULL)
44+
45+
-- struct constructors: x can be NULL in one branch only
46+
query
47+
SELECT IF(q, named_struct('x', i), named_struct('x', 0)) FROM test_if_nested
48+
49+
query
50+
SELECT IF(q, named_struct('x', 0), named_struct('x', i)) FROM test_if_nested
51+
52+
-- map constructors: the value can be NULL in one branch only
53+
query
54+
SELECT IF(q, map('k', i), map('k', 0)) FROM test_if_nested
55+
56+
query
57+
SELECT IF(q, map('k', 0), map('k', i)) FROM test_if_nested
58+
59+
-- a map column, whose value can be NULL, and a map constructor
60+
query
61+
SELECT IF(q, m, map('z', 0)) FROM test_if_nested
62+
63+
query
64+
SELECT IF(m IS NULL, map('z', 0), m) FROM test_if_nested
65+
66+
-- a struct column and a struct constructor
67+
query
68+
SELECT IF(q, s, named_struct('x', 0)) FROM test_if_nested
69+
70+
-- a struct inside a map value
71+
query
72+
SELECT IF(q, ms, map('z', named_struct('x', 0))) FROM test_if_nested
73+
74+
-- an array inside a map value
75+
query
76+
SELECT IF(q, map('k', array(i)), map('k', array(0))) FROM test_if_nested
77+
78+
-- arrays already agree, because CreateArray casts its elements to one type
79+
query
80+
SELECT IF(q, array(i), array(0)) FROM test_if_nested
81+
82+
-- CASE WHEN already casts its branches to a common type
83+
query
84+
SELECT CASE WHEN q THEN named_struct('x', i) ELSE named_struct('x', 0) END FROM test_if_nested

‎spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -550,6 +550,34 @@ class CometMapExpressionSuite extends CometTestBase {
550550
}
551551
}
552552

553+
// A folded `map('z', 0)` has `valueContainsNull = false`, and a `MAP<STRING, INT>` column has
554+
// `valueContainsNull = true`. Spark's `If` treats the two as one type and adds no cast, so the
555+
// native planner has to cast the branch whose Arrow type differs from the common type (#6334).
556+
// Each INSERT writes its own files and a batch never spans files, so there are batches in which
557+
// every row takes the THEN branch, batches in which every row takes the ELSE branch, and batches
558+
// that mix the two. The ones in which every row takes the ELSE branch failed with "column types
559+
// must match schema types". `if_nested_nullability.sql` covers the constructor path.
560+
test("IF with a folded map literal branch and a map column (multirow)") {
561+
withTable("t") {
562+
sql("CREATE TABLE t(c BOOLEAN, m MAP<STRING, INT>) USING parquet")
563+
// Write the rows in Spark, so that only the queries below run in Comet
564+
withSQLConf(CometConf.COMET_ENABLED.key -> "false") {
565+
// c is true and m is not NULL in every row
566+
sql("INSERT INTO t VALUES (true, map('a', 1)), (true, map('b', CAST(NULL AS INT)))")
567+
// c is false or NULL and m is NULL in every row
568+
sql("INSERT INTO t VALUES (false, NULL), (NULL, NULL)")
569+
// pairs of rows that take different branches
570+
val mixed =
571+
(0 until 10).map(i => if (i % 2 == 0) s"(true, map('k', $i))" else "(false, NULL)")
572+
sql(s"INSERT INTO t VALUES ${mixed.mkString(", ")}")
573+
}
574+
Seq("IF(c, m, map('z', 0))", "IF(c, map('z', 0), m)", "IF(m IS NULL, map('z', 0), m)")
575+
.foreach { expr =>
576+
checkSparkAnswerAndOperator(s"SELECT $expr AS r FROM t")
577+
}
578+
}
579+
}
580+
553581
// The native lookup compares a whole batch of map entries in one pass and then reads each row's
554582
// window out of the resulting mask. A native OFFSET slices the batch, and Arrow keeps a sliced
555583
// MapArray's original entry offsets, so the visible entries start part way into the keys child --

0 commit comments

Comments
 (0)