Skip to content

Commit 35440ba

Browse files
committed
revert: drop the array_union NULL guard in favor of #6716
The CASE WHEN <left> IS NOT NULL guard serialized the left operand twice, in the IS NOT NULL test and in the call, so nested unions grew exponentially: twelve nested array_union calls became 4,095 union nodes, and each level evaluated its left operand twice. #6716 adds a native NullShortCircuit that evaluates the left operand once and the right operand only on the rows where the left is not NULL. It covers array_union for every element type, so it fixes the eager evaluation this guard was for without the duplication. This reverts commit 88ec629.
1 parent 88ec629 commit 35440ba

3 files changed

Lines changed: 4 additions & 147 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/arrays.scala‎

Lines changed: 4 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -605,78 +605,25 @@ object CometArrayDistinct extends CometExpressionSerde[ArrayDistinct] {
605605
}
606606

607607
object CometArrayUnion extends CometExpressionSerde[ArrayUnion] {
608-
609-
/**
610-
* Spark's `ArrayUnion` is a `BinaryExpression`: for a NULL left array it returns NULL without
611-
* evaluating the right operand. The native function evaluates both operands over the whole
612-
* batch first, so a right operand that throws, such as `slice(b, 0, 1)` or an ANSI cast, fails
613-
* on rows Spark never evaluates it for. `convert` reproduces the short-circuit with a `CASE
614-
* WHEN <left> IS NOT NULL` guard, as `CometElementAt` does. A column or literal cannot throw,
615-
* so it needs no guard. The guard serializes the left operand twice, which a stateful operand
616-
* cannot survive, so that shape stays on Spark. See
617-
* https://github.com/apache/datafusion-comet/issues/6613.
618-
*/
619-
private val eagerRightOperandReason: String =
620-
"a nullable nondeterministic left operand: native array_union evaluates the right operand " +
621-
"over the whole batch, where Spark skips it on the rows whose left operand is NULL"
622-
623-
/** True when `convert` has to guard the call to reproduce Spark's NULL short-circuit. */
624-
private def needsNullGuard(expr: ArrayUnion): Boolean =
625-
expr.left.nullable && !expr.right.isInstanceOf[Attribute] && !expr.right.isInstanceOf[Literal]
626-
627608
override def getIncompatibleReasons(): Seq[String] =
628609
Seq(ArraySetSupport.floatingPointReason, ArraySetSupport.collationReason)
629610

630-
override def getUnsupportedReasons(): Seq[String] = Seq(eagerRightOperandReason)
631-
632611
override def getSupportLevel(expr: ArrayUnion): SupportLevel =
633-
if (needsNullGuard(expr) && !expr.left.deterministic) {
634-
Unsupported(Some(eagerRightOperandReason))
635-
} else {
636-
ArraySetSupport.supportLevel(expr.dataType)
637-
}
612+
ArraySetSupport.supportLevel(expr.dataType)
638613

639614
override def convert(
640615
expr: ArrayUnion,
641616
inputs: Seq[Attribute],
642617
binding: Boolean): Option[ExprOuterClass.Expr] = {
643-
val leftArrayExprProto = exprToProtoInternal(expr.left, inputs, binding)
644-
val rightArrayExprProto = exprToProtoInternal(expr.right, inputs, binding)
618+
val leftArrayExprProto = exprToProtoInternal(expr.children.head, inputs, binding)
619+
val rightArrayExprProto = exprToProtoInternal(expr.children(1), inputs, binding)
645620

646621
val arraysUnionScalarExpr =
647622
scalarFunctionExprToProto(
648623
ArraySetSupport.function("array_union", expr.dataType),
649624
leftArrayExprProto,
650625
rightArrayExprProto)
651-
if (!needsNullGuard(expr)) {
652-
arraysUnionScalarExpr
653-
} else {
654-
// DataFusion's CaseExpr evaluates the THEN branch only on the rows the guard selects.
655-
val isNotNullExpr = createUnaryExpr(
656-
expr,
657-
expr.left,
658-
inputs,
659-
binding,
660-
(builder, unaryExpr) => builder.setIsNotNull(unaryExpr))
661-
val nullLiteralProto =
662-
exprToProtoInternal(Literal(null, expr.dataType), Seq.empty, binding = true)
663-
for {
664-
union <- arraysUnionScalarExpr
665-
notNull <- isNotNullExpr
666-
nullLit <- nullLiteralProto
667-
} yield {
668-
val caseWhenExpr = ExprOuterClass.CaseWhen
669-
.newBuilder()
670-
.addWhen(notNull)
671-
.addThen(union)
672-
.setElseExpr(nullLit)
673-
.build()
674-
ExprOuterClass.Expr
675-
.newBuilder()
676-
.setCaseWhen(caseWhenExpr)
677-
.build()
678-
}
679-
}
626+
arraysUnionScalarExpr
680627
}
681628
}
682629

‎spark/src/test/resources/sql-tests/expressions/array/array_union_null_left.sql‎

Lines changed: 0 additions & 50 deletions
This file was deleted.

‎spark/src/test/resources/sql-tests/expressions/array/array_union_null_left_ansi.sql‎

Lines changed: 0 additions & 40 deletions
This file was deleted.

0 commit comments

Comments
 (0)