Skip to content

Commit 88ec629

Browse files
committed
fix: skip array_union's right operand on rows whose left array is NULL
Spark's ArrayUnion returns NULL for a NULL left array without evaluating the right operand. The native function evaluates both over the whole batch, so a right operand that throws, such as slice(b, 0, 1) or an ANSI cast, failed on rows that Spark never evaluates it for. That already happened on main for every element type that runs natively, and this PR would have extended it to float elements on 4.0.5+ and 4.1.4+. CometArrayUnion now guards the call with CASE WHEN <left> IS NOT NULL, as CometElementAt does, unless the right operand is a column or a literal, which cannot throw. A nullable nondeterministic left operand, which the guard would evaluate twice, stays on Spark.
1 parent 2ecb05f commit 88ec629

3 files changed

Lines changed: 147 additions & 4 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/arrays.scala‎

Lines changed: 57 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -605,25 +605,78 @@ object CometArrayDistinct extends CometExpressionSerde[ArrayDistinct] {
605605
}
606606

607607
object CometArrayUnion extends CometExpressionSerde[ArrayUnion] {
608+
609+
/**
610+
* Spark's `ArrayUnion` is a `BinaryExpression`: for a NULL left array it returns NULL without
611+
* evaluating the right operand. The native function evaluates both operands over the whole
612+
* batch first, so a right operand that throws, such as `slice(b, 0, 1)` or an ANSI cast, fails
613+
* on rows Spark never evaluates it for. `convert` reproduces the short-circuit with a `CASE
614+
* WHEN <left> IS NOT NULL` guard, as `CometElementAt` does. A column or literal cannot throw,
615+
* so it needs no guard. The guard serializes the left operand twice, which a stateful operand
616+
* cannot survive, so that shape stays on Spark. See
617+
* https://github.com/apache/datafusion-comet/issues/6613.
618+
*/
619+
private val eagerRightOperandReason: String =
620+
"a nullable nondeterministic left operand: native array_union evaluates the right operand " +
621+
"over the whole batch, where Spark skips it on the rows whose left operand is NULL"
622+
623+
/** True when `convert` has to guard the call to reproduce Spark's NULL short-circuit. */
624+
private def needsNullGuard(expr: ArrayUnion): Boolean =
625+
expr.left.nullable && !expr.right.isInstanceOf[Attribute] && !expr.right.isInstanceOf[Literal]
626+
608627
override def getIncompatibleReasons(): Seq[String] =
609628
Seq(ArraySetSupport.floatingPointReason, ArraySetSupport.collationReason)
610629

630+
override def getUnsupportedReasons(): Seq[String] = Seq(eagerRightOperandReason)
631+
611632
override def getSupportLevel(expr: ArrayUnion): SupportLevel =
612-
ArraySetSupport.supportLevel(expr.dataType)
633+
if (needsNullGuard(expr) && !expr.left.deterministic) {
634+
Unsupported(Some(eagerRightOperandReason))
635+
} else {
636+
ArraySetSupport.supportLevel(expr.dataType)
637+
}
613638

614639
override def convert(
615640
expr: ArrayUnion,
616641
inputs: Seq[Attribute],
617642
binding: Boolean): Option[ExprOuterClass.Expr] = {
618-
val leftArrayExprProto = exprToProtoInternal(expr.children.head, inputs, binding)
619-
val rightArrayExprProto = exprToProtoInternal(expr.children(1), inputs, binding)
643+
val leftArrayExprProto = exprToProtoInternal(expr.left, inputs, binding)
644+
val rightArrayExprProto = exprToProtoInternal(expr.right, inputs, binding)
620645

621646
val arraysUnionScalarExpr =
622647
scalarFunctionExprToProto(
623648
ArraySetSupport.function("array_union", expr.dataType),
624649
leftArrayExprProto,
625650
rightArrayExprProto)
626-
arraysUnionScalarExpr
651+
if (!needsNullGuard(expr)) {
652+
arraysUnionScalarExpr
653+
} else {
654+
// DataFusion's CaseExpr evaluates the THEN branch only on the rows the guard selects.
655+
val isNotNullExpr = createUnaryExpr(
656+
expr,
657+
expr.left,
658+
inputs,
659+
binding,
660+
(builder, unaryExpr) => builder.setIsNotNull(unaryExpr))
661+
val nullLiteralProto =
662+
exprToProtoInternal(Literal(null, expr.dataType), Seq.empty, binding = true)
663+
for {
664+
union <- arraysUnionScalarExpr
665+
notNull <- isNotNullExpr
666+
nullLit <- nullLiteralProto
667+
} yield {
668+
val caseWhenExpr = ExprOuterClass.CaseWhen
669+
.newBuilder()
670+
.addWhen(notNull)
671+
.addThen(union)
672+
.setElseExpr(nullLit)
673+
.build()
674+
ExprOuterClass.Expr
675+
.newBuilder()
676+
.setCaseWhen(caseWhenExpr)
677+
.build()
678+
}
679+
}
627680
}
628681
}
629682

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- Spark's ArrayUnion is a BinaryExpression: for a NULL left array it returns NULL without
19+
-- evaluating the right operand, so a right operand that would throw on that row doesn't. Comet
20+
-- guards the native call with CASE WHEN <left> IS NOT NULL to match.
21+
-- https://github.com/apache/datafusion-comet/issues/6613
22+
23+
statement
24+
CREATE TABLE array_union_null_left(a array<int>, b array<int>, da array<double>, db array<double>) USING parquet
25+
26+
statement
27+
INSERT INTO array_union_null_left VALUES (NULL, array(1), NULL, array(1.0D))
28+
29+
-- slice with a start of 0 throws whenever it is evaluated, in any mode
30+
query
31+
SELECT array_union(a, slice(b, 0, 1)) FROM array_union_null_left
32+
33+
-- Float elements run natively only on the Spark versions whose floats Comet can match, so the
34+
-- path depends on the version
35+
query spark_answer_only
36+
SELECT array_union(da, slice(db, 0, 1)) FROM array_union_null_left
37+
38+
-- The guard would evaluate a nondeterministic left operand twice, so that shape stays on Spark
39+
query expect_fallback(nullable nondeterministic left operand)
40+
SELECT array_union(IF(rand(42) > 0.5D, a, NULL), slice(b, 1, 1)) FROM array_union_null_left
41+
42+
statement
43+
CREATE TABLE array_union_mixed(a array<int>, b array<int>) USING parquet
44+
45+
statement
46+
INSERT INTO array_union_mixed VALUES (NULL, array(1)), (array(2), array(1)), (array(3), NULL)
47+
48+
-- The guard selects only some rows of the batch
49+
query
50+
SELECT array_union(a, slice(b, 1, 1)) FROM array_union_mixed
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
-- Licensed to the Apache Software Foundation (ASF) under one
2+
-- or more contributor license agreements. See the NOTICE file
3+
-- distributed with this work for additional information
4+
-- regarding copyright ownership. The ASF licenses this file
5+
-- to you under the Apache License, Version 2.0 (the
6+
-- "License"); you may not use this file except in compliance
7+
-- with the License. You may obtain a copy of the License at
8+
--
9+
-- http://www.apache.org/licenses/LICENSE-2.0
10+
--
11+
-- Unless required by applicable law or agreed to in writing,
12+
-- software distributed under the License is distributed on an
13+
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
-- KIND, either express or implied. See the License for the
15+
-- specific language governing permissions and limitations
16+
-- under the License.
17+
18+
-- Config: spark.sql.ansi.enabled=true
19+
20+
-- Spark skips the right operand of array_union on rows whose left array is NULL, so an ANSI cast
21+
-- there never runs. https://github.com/apache/datafusion-comet/issues/6613
22+
23+
statement
24+
CREATE TABLE array_union_ansi(a array<int>, s string) USING parquet
25+
26+
statement
27+
INSERT INTO array_union_ansi VALUES (NULL, 'bad'), (array(1), '1')
28+
29+
query
30+
SELECT array_union(a, array(CAST(s AS INT))) FROM array_union_ansi
31+
32+
statement
33+
CREATE TABLE array_union_ansi_bad(a array<int>, s string) USING parquet
34+
35+
statement
36+
INSERT INTO array_union_ansi_bad VALUES (array(1), 'bad')
37+
38+
-- Where the array isn't NULL, both evaluate the cast and fail
39+
query expect_error(CAST_INVALID_INPUT)
40+
SELECT array_union(a, array(CAST(s AS INT))) FROM array_union_ansi_bad

0 commit comments

Comments
 (0)