Skip to content

Commit 275327a

Browse files
committed
test: keep the metrics interval check compiling on Spark 3.x
withSQLConf returns Unit before Spark 4.0, so run the assertions inside the block instead of returning the metric values from it.
1 parent 299144e commit 275327a

1 file changed

Lines changed: 14 additions & 17 deletions

File tree

‎spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala‎

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1157,11 +1157,11 @@ class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper {
11571157
test("native block without a JVM input publishes SQL metrics on the update interval") {
11581158
withTempPath { dir =>
11591159
spark.range(0, 10000, 1, 1).write.parquet(dir.getAbsolutePath)
1160-
// Ten output batches from one task. Returns output_rows as the task sees it right after the
1161-
// first batch, and again after the last. The task creates the native plan, which starts the
1162-
// interval's clock, when it builds the iterator, so the sleep before the first batch runs
1160+
// Ten output batches from one task. Passes `check` the output_rows the task sees right after
1161+
// the first batch, and again after the last. The task creates the native plan, which starts
1162+
// the interval's clock, when it builds the iterator, so the sleep before the first batch runs
11631163
// after that.
1164-
def outputRowsMidStreamAndAtEnd(interval: String): (Long, Long) =
1164+
def checkOutputRows(interval: String)(check: (Long, Long) => Unit): Unit =
11651165
withSQLConf(
11661166
CometConf.COMET_BATCH_SIZE.key -> "1000",
11671167
CometConf.COMET_METRICS_UPDATE_INTERVAL.key -> interval) {
@@ -1172,7 +1172,7 @@ class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper {
11721172
// The task deserializes this metric together with the scan's metric node, so both
11731173
// refer to the task-side copy that native execution publishes into.
11741174
val outputRows = scan.metrics("output_rows")
1175-
SQLExecution.withSQLConfPropagated(spark) {
1175+
val (midStream, atEnd) = SQLExecution.withSQLConfPropagated(spark) {
11761176
scan
11771177
.executeColumnar()
11781178
.mapPartitions { batches =>
@@ -1185,24 +1185,21 @@ class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper {
11851185
.collect()
11861186
.head
11871187
}
1188+
check(midStream, atEnd)
11881189
}
11891190

11901191
// With the interval disabled, the only publish is the one in releasePlan, which a
11911192
// per-batch publish would break after the first batch.
1192-
val (disabledMidStream, disabledAtEnd) = outputRowsMidStreamAndAtEnd("-1")
1193-
assert(disabledMidStream == 0, s"output_rows was published mid-stream: $disabledMidStream")
1194-
assert(
1195-
disabledAtEnd == 10000,
1196-
s"releasePlan should publish the final output_rows, got $disabledAtEnd")
1193+
checkOutputRows("-1") { (midStream, atEnd) =>
1194+
assert(midStream == 0, s"output_rows was published mid-stream: $midStream")
1195+
assert(atEnd == 10000, s"releasePlan should publish the final output_rows, got $atEnd")
1196+
}
11971197

11981198
// A 1 ms interval has passed by the time the first batch arrives, so that batch publishes.
1199-
val (dueMidStream, dueAtEnd) = outputRowsMidStreamAndAtEnd("1")
1200-
assert(
1201-
dueMidStream > 0,
1202-
"output_rows was not published mid-stream once the interval passed")
1203-
assert(
1204-
dueAtEnd == 10000,
1205-
s"releasePlan should publish the final output_rows, got $dueAtEnd")
1199+
checkOutputRows("1") { (midStream, atEnd) =>
1200+
assert(midStream > 0, "output_rows was not published mid-stream once the interval passed")
1201+
assert(atEnd == 10000, s"releasePlan should publish the final output_rows, got $atEnd")
1202+
}
12061203
}
12071204
}
12081205

0 commit comments

Comments
 (0)