Skip to content

Commit 02aa2bb

Browse files
committed
perf: check a repeated argument once when matching a UDF's null guard
BooleanSimplification drops a repeated isnull from the predicate that HandleNullInputsForUDF builds, so with a column at two primitive positions, f(a, a) optimizes to if (isnull(a)) null else f(knownnotnull(a), knownnotnull(a)). The predicate rebuilt for the match kept every repeat, so those guards stayed native. It is now rebuilt from an ExpressionSet of the guarded arguments. ExpressionSet keeps every nondeterministic argument, so a guard over one still stays native. The first null guard test also covers addBoth(a, a), addThree(a, b, a) and addBoth(a, 1L), the last with an argument that cannot be null.
1 parent 68aac67 commit 02aa2bb

2 files changed

Lines changed: 17 additions & 5 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ package org.apache.comet.serde
2222
import scala.util.control.NonFatal
2323

2424
import org.apache.spark.SparkEnv
25-
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSeq, BindReferences, Expression, If, IsNull, KnownNotNull, Literal, Or, RuntimeReplaceable, ScalaUDF}
25+
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSeq, BindReferences, Expression, ExpressionSet, If, IsNull, KnownNotNull, Literal, Or, RuntimeReplaceable, ScalaUDF}
2626
import org.apache.spark.sql.types.BinaryType
2727

2828
import org.apache.comet.CometConf
@@ -72,9 +72,9 @@ object CometScalaUDF extends CometExpressionSerde[ScalaUDF] {
7272
* protects. In the kernel it is a branch per row.
7373
*
7474
* Only that shape matches: the predicate must be the one the rule builds from the UDF's guarded
75-
* arguments. The kernel then runs no expression it would not run for the UDF alone, apart from
76-
* the guard's own `if`, `or` and `isnull`, and it runs Spark's code for all of them. In a
77-
* filter or join condition, and in the predicate of a conditional,
75+
* arguments, each checked once. The kernel then runs no expression it would not run for the UDF
76+
* alone, apart from the guard's own `if`, `or` and `isnull`, and it runs Spark's code for all
77+
* of them. In a filter or join condition, and in the predicate of a conditional,
7878
* `ReplaceNullWithFalseInPredicate` replaces the null with `false`, so that form matches too.
7979
*/
8080
def emitNullGuardDispatch(expr: If, inputs: Seq[Attribute], binding: Boolean): Option[Expr] =
@@ -107,8 +107,9 @@ object CometScalaUDF extends CometExpressionSerde[ScalaUDF] {
107107
val guarded = udf.inputPrimitives.zip(udf.children).collect {
108108
case (true, KnownNotNull(arg)) => arg
109109
}
110+
// The optimizer drops a repeated `isnull`, as in `f(a, a)`, so check each argument once.
110111
// `semanticEquals` is false for a nondeterministic argument, so that guard stays native.
111-
guarded
112+
ExpressionSet(guarded).toSeq
112113
.map(IsNull(_))
113114
.reduceLeftOption[Expression](Or)
114115
.exists(_.semanticEquals(predicate))

‎spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1035,6 +1035,8 @@ class CometCodegenSuite
10351035
// goes to the kernel with the call, and neither `if` nor `isnull` is native.
10361036
spark.udf.register("plusOne", (x: Long) => x + 1)
10371037
spark.udf.register("combine", (a: Long, b: Int, s: String) => s"$s:${a + b}")
1038+
spark.udf.register("addBoth", (x: Long, y: Long) => x + y)
1039+
spark.udf.register("addThree", (x: Long, y: Int, z: Long) => x + y + z)
10381040
withTable("t") {
10391041
sql("CREATE TABLE t (a BIGINT, b INT, s STRING) USING parquet")
10401042
sql(
@@ -1053,6 +1055,15 @@ class CometCodegenSuite
10531055
checkSparkAnswerAndImpl(
10541056
sql("SELECT combine(a, b, s) FROM t"),
10551057
dispatched = Seq("combine", "or") ++ guard)
1058+
// The optimizer drops a repeated `isnull`, so a column passed twice is checked once.
1059+
checkSparkAnswerAndImpl(sql("SELECT addBoth(a, a) FROM t"), dispatched = "addboth" +: guard)
1060+
checkSparkAnswerAndImpl(
1061+
sql("SELECT addThree(a, b, a) FROM t"),
1062+
dispatched = Seq("addthree", "or") ++ guard)
1063+
// An argument that cannot be null gets no `isnull`.
1064+
checkSparkAnswerAndImpl(
1065+
sql("SELECT addBoth(a, 1L) FROM t"),
1066+
dispatched = "addboth" +: guard)
10561067
}
10571068
}
10581069

0 commit comments

Comments
 (0)