Skip to content

Commit 299144e

Browse files
committed
test: check that a native block without a JVM input publishes once its metrics interval passes
The test covered only a disabled interval, so it would also pass if this branch never published mid-stream. Run it with a 1 ms interval too, sleeping before the first batch, and expect output_rows to be published right after that batch.
1 parent 00a55ed commit 299144e

1 file changed

Lines changed: 45 additions & 26 deletions

File tree

‎spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala‎

Lines changed: 45 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1157,33 +1157,52 @@ class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper {
11571157
test("native block without a JVM input publishes SQL metrics on the update interval") {
11581158
withTempPath { dir =>
11591159
spark.range(0, 10000, 1, 1).write.parquet(dir.getAbsolutePath)
1160-
// Ten output batches from one task, so a per-batch publish shows up after the first one.
1161-
// With the interval disabled, the only publish is the one in releasePlan.
1162-
withSQLConf(
1163-
CometConf.COMET_BATCH_SIZE.key -> "1000",
1164-
CometConf.COMET_METRICS_UPDATE_INTERVAL.key -> "-1") {
1165-
val plan = spark.read.parquet(dir.getAbsolutePath).queryExecution.executedPlan
1166-
val scan = find(plan)(_.isInstanceOf[CometNativeScanExec])
1167-
.getOrElse(fail(s"Expected CometNativeScanExec in plan:\n${plan.treeString}"))
1168-
.asInstanceOf[CometNativeScanExec]
1169-
// The task deserializes this metric together with the scan's metric node, so both refer
1170-
// to the task-side copy that native execution publishes into.
1171-
val outputRows = scan.metrics("output_rows")
1172-
val (midStream, atEnd) = SQLExecution.withSQLConfPropagated(spark) {
1173-
scan
1174-
.executeColumnar()
1175-
.mapPartitions { batches =>
1176-
batches.next()
1177-
val mid = outputRows.value
1178-
batches.foreach(_ => ())
1179-
Iterator((mid, outputRows.value))
1180-
}
1181-
.collect()
1182-
.head
1160+
// Ten output batches from one task. Returns output_rows as the task sees it right after the
1161+
// first batch, and again after the last. The task creates the native plan, which starts the
1162+
// interval's clock, when it builds the iterator, so the sleep before the first batch runs
1163+
// after that.
1164+
def outputRowsMidStreamAndAtEnd(interval: String): (Long, Long) =
1165+
withSQLConf(
1166+
CometConf.COMET_BATCH_SIZE.key -> "1000",
1167+
CometConf.COMET_METRICS_UPDATE_INTERVAL.key -> interval) {
1168+
val plan = spark.read.parquet(dir.getAbsolutePath).queryExecution.executedPlan
1169+
val scan = find(plan)(_.isInstanceOf[CometNativeScanExec])
1170+
.getOrElse(fail(s"Expected CometNativeScanExec in plan:\n${plan.treeString}"))
1171+
.asInstanceOf[CometNativeScanExec]
1172+
// The task deserializes this metric together with the scan's metric node, so both
1173+
// refer to the task-side copy that native execution publishes into.
1174+
val outputRows = scan.metrics("output_rows")
1175+
SQLExecution.withSQLConfPropagated(spark) {
1176+
scan
1177+
.executeColumnar()
1178+
.mapPartitions { batches =>
1179+
Thread.sleep(100)
1180+
batches.next()
1181+
val mid = outputRows.value
1182+
batches.foreach(_ => ())
1183+
Iterator((mid, outputRows.value))
1184+
}
1185+
.collect()
1186+
.head
1187+
}
11831188
}
1184-
assert(midStream == 0, s"output_rows was published mid-stream: $midStream")
1185-
assert(atEnd == 10000, s"releasePlan should publish the final output_rows, got $atEnd")
1186-
}
1189+
1190+
// With the interval disabled, the only publish is the one in releasePlan, which a
1191+
// per-batch publish would break after the first batch.
1192+
val (disabledMidStream, disabledAtEnd) = outputRowsMidStreamAndAtEnd("-1")
1193+
assert(disabledMidStream == 0, s"output_rows was published mid-stream: $disabledMidStream")
1194+
assert(
1195+
disabledAtEnd == 10000,
1196+
s"releasePlan should publish the final output_rows, got $disabledAtEnd")
1197+
1198+
// A 1 ms interval has passed by the time the first batch arrives, so that batch publishes.
1199+
val (dueMidStream, dueAtEnd) = outputRowsMidStreamAndAtEnd("1")
1200+
assert(
1201+
dueMidStream > 0,
1202+
"output_rows was not published mid-stream once the interval passed")
1203+
assert(
1204+
dueAtEnd == 10000,
1205+
s"releasePlan should publish the final output_rows, got $dueAtEnd")
11871206
}
11881207
}
11891208

0 commit comments

Comments
 (0)