Skip to content

Commit dd6da42

Browse files
committed
fix: count LEGACY and TRY string-to-integer casts as infallible, and find shared subexpressions lazily
Cast::is_infallible now accepts a cast from Utf8 or LargeUtf8 to an integer in LEGACY or TRY mode, which returns NULL for a string it cannot parse. Comet's parsers for those modes never raise. With ANSI off, the native planner then leaves array_contains(ids, CAST(s AS INT)) unwrapped, and CASE keeps its eager path for such a cast. withSharedSubexpressions now runs EquivalentExpressions on the operator's first canShortCircuitNulls call instead of for every converted operator. canShortCircuitNulls skips it entirely when every later argument is a leaf, which subexpression elimination never shares. array_null_short_circuit_non_ansi.sql runs the array shapes with ANSI off. The casts return NULL there, while a slice start of 0 and a rand argument still depend on the short-circuit. The shuffle fixture pins spark.comet.batchSize, and the map fixture covers map_from_arrays, which its CASE WHEN guards already protect.
1 parent 24c0270 commit dd6da42

6 files changed

Lines changed: 143 additions & 18 deletions

File tree

‎native/spark-expr/src/conditional_funcs/case_when.rs‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1147,6 +1147,7 @@ mod tests {
11471147
Field::new("d", DataType::Float64, true),
11481148
Field::new("e", DataType::Float64, true),
11491149
Field::new("s", DataType::Utf8, true),
1150+
Field::new("ls", DataType::LargeUtf8, true),
11501151
Field::new("dec", DataType::Decimal128(10, 2), true),
11511152
Field::new("l", list_of(DataType::Float64), true),
11521153
Field::new("m", list_of(DataType::Float64), true),
@@ -1160,15 +1161,17 @@ mod tests {
11601161
),
11611162
]);
11621163
let c = |name: &str| col(name, &schema).unwrap();
1163-
let cast = |e: Arc<dyn PhysicalExpr>, to: DataType| -> Arc<dyn PhysicalExpr> {
1164-
Arc::new(Cast::new(
1164+
let cast_in = |mode: EvalMode, e: Arc<dyn PhysicalExpr>, to: DataType| {
1165+
let cast: Arc<dyn PhysicalExpr> = Arc::new(Cast::new(
11651166
e,
11661167
to,
1167-
SparkCastOptions::new_without_timezone(EvalMode::Ansi, false),
1168+
SparkCastOptions::new_without_timezone(mode, false),
11681169
None,
11691170
None,
1170-
))
1171+
));
1172+
cast
11711173
};
1174+
let cast = |e: Arc<dyn PhysicalExpr>, to: DataType| cast_in(EvalMode::Ansi, e, to);
11721175
let infallible = |e: Arc<dyn PhysicalExpr>| is_infallible(&e, &schema);
11731176
// A comparison as the planner builds it, which follows Spark's ordering for floats
11741177
let compare = |left: &str, op: Operator, right: Arc<dyn PhysicalExpr>| {
@@ -1225,9 +1228,32 @@ mod tests {
12251228
assert!(!infallible(checked));
12261229
// Decimal arithmetic can overflow
12271230
assert!(!infallible(binary(c("dec"), Operator::Plus, c("dec"))));
1228-
// Narrowing and parsing casts can fail
1231+
// Narrowing and parsing casts can fail under ANSI
12291232
assert!(!infallible(cast(c("i"), DataType::Int32)));
12301233
assert!(!infallible(cast(c("s"), DataType::Int32)));
1234+
// A string that LEGACY or TRY mode cannot parse as an integer becomes NULL instead
1235+
for mode in [EvalMode::Legacy, EvalMode::Try] {
1236+
for to in [
1237+
DataType::Int8,
1238+
DataType::Int16,
1239+
DataType::Int32,
1240+
DataType::Int64,
1241+
] {
1242+
assert!(
1243+
infallible(cast_in(mode, c("s"), to.clone())),
1244+
"{mode:?} {to}"
1245+
);
1246+
assert!(
1247+
infallible(cast_in(mode, c("ls"), to.clone())),
1248+
"{mode:?} {to}"
1249+
);
1250+
}
1251+
// Other string casts are not claimed
1252+
assert!(
1253+
!infallible(cast_in(mode, c("s"), DataType::Float64)),
1254+
"{mode:?}"
1255+
);
1256+
}
12311257
// So can anything built from a part that can fail
12321258
assert!(!infallible(binary(
12331259
binary(c("i"), Operator::Divide, c("i")),

‎native/spark-expr/src/conversion_funcs/cast.rs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,10 @@ impl Cast {
119119
}
120120
}
121121

122-
/// Whether the cast cannot fail for any value, in any eval mode: a cast to the child's own
123-
/// type, from NULL, or to a wider number. `cast_array` rejects a dictionary target first.
122+
/// Whether the cast cannot fail for any value: a cast to the child's own type, from NULL, or to
123+
/// a wider number, in any eval mode, or a cast from a string to an integer in LEGACY or TRY
124+
/// mode, which returns NULL for a string it cannot parse. `cast_array` rejects a dictionary
125+
/// target first.
124126
pub(crate) fn is_infallible(&self, input_schema: &Schema) -> bool {
125127
use DataType::*;
126128
let Ok(from) = self.child.data_type(input_schema) else {
@@ -136,7 +138,10 @@ impl Cast {
136138
| (Int32, Int64 | Float32 | Float64)
137139
| (Int64, Float32 | Float64)
138140
| (Float32, Float64)
139-
))
141+
)
142+
|| (matches!(from, Utf8 | LargeUtf8)
143+
&& matches!(self.data_type, Int8 | Int16 | Int32 | Int64)
144+
&& self.cast_options.eval_mode != EvalMode::Ansi))
140145
}
141146
}
142147

‎spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1307,17 +1307,23 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
13071307
* projection or an aggregation evaluates each of them for every row, before the expressions
13081308
* that use them. They are treated that way for every operator, and whatever
13091309
* `spark.sql.subexpressionElimination.enabled` says, which only keeps them evaluated for every
1310-
* row where Spark might not.
1310+
* row where Spark might not. They are found on the first `canShortCircuitNulls` call for the
1311+
* operator, so an operator without an argument to skip never runs `EquivalentExpressions`.
13111312
*/
1312-
private val sharedSubexpressions = new DynamicVariable[Seq[Expression]](Nil)
1313+
private val sharedSubexpressions = new DynamicVariable[Option[SharedSubexpressions]](None)
13131314

1314-
/** Runs `f`, which serializes an operator whose expressions are `expressions`. */
1315-
def withSharedSubexpressions[T](expressions: Seq[Expression])(f: => T): T = {
1316-
val equivalence = new EquivalentExpressions
1317-
expressions.foreach(equivalence.addExprTree(_))
1318-
sharedSubexpressions.withValue(equivalence.getCommonSubexpressions)(f)
1315+
private final class SharedSubexpressions(expressions: Seq[Expression]) {
1316+
lazy val common: Seq[Expression] = {
1317+
val equivalence = new EquivalentExpressions
1318+
expressions.foreach(equivalence.addExprTree(_))
1319+
equivalence.getCommonSubexpressions
1320+
}
13191321
}
13201322

1323+
/** Runs `f`, which serializes an operator whose expressions are `expressions`. */
1324+
def withSharedSubexpressions[T](expressions: Seq[Expression])(f: => T): T =
1325+
sharedSubexpressions.withValue(Some(new SharedSubexpressions(expressions)))(f)
1326+
13211327
/**
13221328
* Whether native execution may skip the arguments of `expr` after the first on the rows where
13231329
* an earlier argument is NULL, as Spark's evaluation of a null-intolerant `BinaryExpression` or
@@ -1326,8 +1332,12 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
13261332
* array is NULL.
13271333
*/
13281334
def canShortCircuitNulls(expr: Expression): Boolean = {
1329-
val shared = sharedSubexpressions.value
1330-
!expr.children.tail.exists(_.exists(e => shared.exists(_.semanticEquals(e))))
1335+
val later = expr.children.tail
1336+
// Subexpression elimination never shares a leaf, such as a column or a literal
1337+
later.forall(_.isInstanceOf[LeafExpression]) || {
1338+
val shared = sharedSubexpressions.value.map(_.common).getOrElse(Nil)
1339+
!later.exists(_.exists(e => shared.exists(_.semanticEquals(e))))
1340+
}
13311341
}
13321342

13331343
/**
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- The queries of array_null_short_circuit.sql with ANSI off. A cast of a malformed string then
19+
-- returns NULL instead of raising, so the cast queries match Spark however the arguments are
20+
-- evaluated, and the native planner drops the guard for them. A start of 0 still makes slice raise,
21+
-- and a nondeterministic argument still advances on every row it sees, so those two shapes depend on
22+
-- skipping the rows whose array is NULL in every mode.
23+
-- https://github.com/apache/datafusion-comet/issues/6613
24+
25+
-- Config: spark.sql.ansi.enabled=false
26+
27+
statement
28+
CREATE TABLE array_null_short_circuit_non_ansi(ai array<int>, s string, i string) USING parquet
29+
30+
statement
31+
INSERT INTO array_null_short_circuit_non_ansi SELECT /*+ COALESCE(1) */ * FROM VALUES
32+
(NULL, 'bad', 'bad'),
33+
(array(1, 2), '2', '1'),
34+
(array(3), '7', '0'),
35+
(NULL, 'worse', 'worse'),
36+
(array(4, NULL, 5), '5', '2')
37+
AS v(ai, s, i)
38+
39+
query expect_native(array_contains)
40+
SELECT array_contains(ai, CAST(s AS INT)) FROM array_null_short_circuit_non_ansi
41+
42+
query expect_native(array_position)
43+
SELECT array_position(ai, CAST(s AS INT)) FROM array_null_short_circuit_non_ansi
44+
45+
query expect_native(array_remove)
46+
SELECT array_remove(ai, CAST(s AS INT)) FROM array_null_short_circuit_non_ansi
47+
48+
query expect_native(arrays_overlap)
49+
SELECT arrays_overlap(ai, array(CAST(s AS INT))) FROM array_null_short_circuit_non_ansi
50+
51+
query expect_native(array_union)
52+
SELECT array_union(ai, array(CAST(s AS INT))) FROM array_null_short_circuit_non_ansi
53+
54+
query expect_native(slice)
55+
SELECT slice(ai, CAST(s AS INT), 1), slice(ai, 1, CAST(i AS INT)) FROM array_null_short_circuit_non_ansi
56+
57+
query expect_native(getarrayitem)
58+
SELECT ai[CAST(i AS INT)] FROM array_null_short_circuit_non_ansi
59+
60+
-- A start of 0 makes slice raise in any mode, and only rows whose array is NULL have one
61+
query expect_native(array_union)
62+
SELECT array_union(ai, slice(array(1), IF(ai IS NULL, 0, 1), 1)) FROM array_null_short_circuit_non_ansi
63+
64+
-- A nondeterministic argument sees the same rows as in Spark, so it returns the same values
65+
query expect_native(array_position)
66+
SELECT array_position(ai, CAST(rand(7L) * 3 AS INT)) FROM array_null_short_circuit_non_ansi

‎spark/src/test/resources/sql-tests/expressions/array/array_null_short_circuit_shuffle.sql‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,13 @@
2222
-- https://github.com/apache/datafusion-comet/issues/6613
2323

2424
-- MinSparkVersion: 4.0
25+
-- Config: spark.comet.batchSize=8192
2526

2627
statement
2728
CREATE TABLE array_null_short_circuit_shuffle(id bigint, a array<int>, b array<int>) USING parquet
2829

29-
-- A single file, which the scan reads in batches of 8192 rows. Only the second batch has NULL arrays.
30+
-- A single file, which the scan reads in batches of 8192 rows (the batch size is pinned above). Only
31+
-- the second batch has NULL arrays.
3032
statement
3133
INSERT INTO array_null_short_circuit_shuffle
3234
SELECT id, IF(id >= 8192 AND id % 2 = 0, NULL, array(0)), array(1, 2, 3, 4)

‎spark/src/test/resources/sql-tests/expressions/map/map_null_short_circuit.sql‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,3 +45,19 @@ SELECT m[CAST(s AS INT)] FROM map_null_short_circuit
4545
-- Where the map is not NULL, the key is evaluated and raises as in Spark
4646
query expect_error(CAST_INVALID_INPUT)
4747
SELECT m[CAST(s || 'x' AS INT)] FROM map_null_short_circuit
48+
49+
-- map_from_arrays builds a map rather than reading one, but it is null-intolerant too: Spark skips
50+
-- the values where the keys are NULL. Comet guards it with CASE WHEN keys IS NOT NULL instead.
51+
statement
52+
CREATE TABLE map_from_arrays_null_short_circuit(k array<int>, s string) USING parquet
53+
54+
statement
55+
INSERT INTO map_from_arrays_null_short_circuit SELECT /*+ COALESCE(1) */ * FROM VALUES
56+
(NULL, 'bad'),
57+
(array(1), '10'),
58+
(NULL, 'worse'),
59+
(array(2), '20')
60+
AS v(k, s)
61+
62+
query
63+
SELECT map_from_arrays(k, array(CAST(s AS INT))) FROM map_from_arrays_null_short_circuit

0 commit comments

Comments
 (0)