Skip to content

Commit b3fcf82

Browse files
committed
test: classify expressions nested in a dispatched subtree as dispatched
Also accepts expect_dispatch / expect_native as sentinel queries for the expect_error preflight check.
1 parent b2cab4e commit b3fcf82

3 files changed

Lines changed: 60 additions & 7 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,18 @@ object CometScalaUDF extends CometExpressionSerde[ScalaUDF] {
172172
// their descendants' names.
173173
expr.setTagValue(CometExplainInfo.DISPATCHED_SELF, ())
174174
withCodegenDispatchExpr(expr, exprName)
175+
// The whole subtree under `expr` was bound and closure-serialized into this one kernel, so
176+
// every expression in it ran in the JVM, not just the root. Naming only the root understates
177+
// that: for `hypot(abs(b), c)` the dispatched set would hold `hypot` alone, and a test could
178+
// assert `abs` was native while an `abs` was in fact running inside the kernel. Attribute
179+
// references and literals are the kernel's inputs rather than work it performed, so they are
180+
// left out - which also keeps them from appearing in the coverage stats as "expressions".
181+
target.foreach {
182+
case _: AttributeReference | _: Literal =>
183+
case node if !(node eq target) =>
184+
withCodegenDispatchExpr(expr, CometExplainInfo.exprDisplayName(node))
185+
case _ =>
186+
}
175187
Some(
176188
ExprOuterClass.Expr
177189
.newBuilder()

‎spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala‎

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ class CometCodegenSuite
193193
s"expected a [COMET-INFO: segment, got:\n$explain")
194194
// Names appear alphabetically via `.distinct.sorted` in rollUpInfoMessages.
195195
assert(
196-
explain.contains("JVM codegen dispatcher: hypot, nanvl"),
196+
dispatchedNames(explain).containsSlice(Seq("hypot", "nanvl")),
197197
s"expected combined codegen-dispatch info, got:\n$explain")
198198
}
199199

@@ -243,6 +243,42 @@ class CometCodegenSuite
243243
}
244244
}
245245

246+
test("an expression nested inside a dispatched subtree is classified as dispatched") {
247+
// The whole of `hypot(abs(b), c)` is bound and closure-serialized into one JVM kernel, so the
248+
// inner `abs` ran in the JVM even though the `abs(a)` next to it ran natively. Naming only
249+
// the dispatched root would let `native = Seq("abs")` pass here while an `abs` was running in
250+
// the kernel, which is the one claim this helper exists to make trustworthy.
251+
withTable("t") {
252+
sql("CREATE TABLE t (a DOUBLE, b DOUBLE, c DOUBLE) USING parquet")
253+
sql("INSERT INTO t VALUES (3.0, 4.0, 5.0)")
254+
val query = "SELECT abs(a), hypot(abs(b), c) FROM t"
255+
256+
// `abs` is genuinely on both sides of the fence, so neither claim about it alone holds.
257+
intercept[TestFailedException] {
258+
checkSparkAnswerAndImpl(sql(query), native = Seq("abs"))
259+
}
260+
intercept[TestFailedException] {
261+
checkSparkAnswerAndImpl(sql(query), dispatched = Seq("abs"))
262+
}
263+
// `hypot` is unambiguous, and the same query still classifies it correctly.
264+
checkSparkAnswerAndImpl(sql(query), dispatched = Seq("hypot"))
265+
}
266+
}
267+
268+
/**
269+
* The expression names listed in the `[COMET-INFO: JVM codegen dispatcher: ...]` segment.
270+
*
271+
* Matching the whole segment rather than a `contains` on `"JVM codegen dispatcher: <name>"`,
272+
* because the segment lists every dispatched expression in the operator sorted by name -
273+
* including expressions nested inside a dispatched subtree - so a substring match pinned to one
274+
* name breaks as soon as a query dispatches a second one.
275+
*/
276+
private def dispatchedNames(explain: String): Seq[String] =
277+
"JVM codegen dispatcher: ([^\\]]*)".r
278+
.findFirstMatchIn(explain)
279+
.map(_.group(1).split(",").map(_.trim).filter(_.nonEmpty).toSeq)
280+
.getOrElse(Seq.empty)
281+
246282
private def withSequenceTable(f: => Unit): Unit = {
247283
withTable("t") {
248284
// `stp` carries a sign-correct step so `sequence(a, b, stp)` is legal on both rows:
@@ -286,7 +322,7 @@ class CometCodegenSuite
286322
val explain =
287323
new ExtendedExplainInfo().generateExtendedInfo(df.queryExecution.executedPlan)
288324
assert(
289-
explain.contains("JVM codegen dispatcher: sequence"),
325+
dispatchedNames(explain).contains("sequence"),
290326
s"expected zero-arg-UDF sequence to route through the dispatcher, got:\n$explain")
291327
}
292328
}
@@ -302,7 +338,7 @@ class CometCodegenSuite
302338
val explain =
303339
new ExtendedExplainInfo().generateExtendedInfo(df.queryExecution.executedPlan)
304340
assert(
305-
explain.contains("JVM codegen dispatcher: sequence"),
341+
dispatchedNames(explain).contains("sequence"),
306342
s"expected composed-arg sequence to route through the dispatcher, got:\n$explain")
307343
}
308344
}
@@ -316,7 +352,7 @@ class CometCodegenSuite
316352
val explain =
317353
new ExtendedExplainInfo().generateExtendedInfo(df.queryExecution.executedPlan)
318354
assert(
319-
explain.contains("JVM codegen dispatcher: sequence"),
355+
dispatchedNames(explain).contains("sequence"),
320356
s"expected date sequence to route through the dispatcher, got:\n$explain")
321357
}
322358
}

‎spark/src/test/scala/org/apache/comet/CometSqlFileTestSuite.scala‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,13 @@ class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper {
112112
case _ => false
113113
}
114114
if (!hasExpectError) return
115+
// `ExpectDispatch` and `ExpectNative` both run `checkSparkAnswerAndImpl`, which performs the
116+
// same answer and operator checks as a plain `query` before asserting anything extra, so
117+
// either is a strictly stronger sentinel than `CheckCoverageAndAnswer`. Not accepting them
118+
// meant upgrading a file's last positive query to one of the new modes made preflight reject
119+
// the file for having no sentinel, forcing a redundant plain query alongside it.
115120
val hasSentinel = file.records.exists {
116-
case SqlQuery(_, CheckCoverageAndAnswer, _) => true
121+
case SqlQuery(_, CheckCoverageAndAnswer | _: ExpectDispatch | _: ExpectNative, _) => true
117122
case _ => false
118123
}
119124
assert(
@@ -122,8 +127,8 @@ class CometSqlFileTestSuite extends CometTestBase with AdaptiveSparkPlanHelper {
122127
s"${CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key}=true but is missing a non-error " +
123128
"sentinel query. Without one, a silent dispatcher fallback to Spark would let the " +
124129
"`expect_error` queries pass vacuously (Spark raises the same error on the fallback " +
125-
"path). Add at least one `query` over valid input so `checkSparkAnswerAndOperator` " +
126-
"fails if the expression did not execute natively.")
130+
"path). Add at least one `query`, `expect_dispatch` or `expect_native` over valid " +
131+
"input so the operator check fails if the expression did not execute natively.")
127132
}
128133

129134
private def runTestFile(relativePath: String, file: SqlTestFile): Unit = {

0 commit comments

Comments
 (0)