@@ -1157,11 +1157,11 @@ class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper {
11571157 test(" native block without a JVM input publishes SQL metrics on the update interval" ) {
11581158 withTempPath { dir =>
11591159 spark.range(0 , 10000 , 1 , 1 ).write.parquet(dir.getAbsolutePath)
1160- // Ten output batches from one task. Returns output_rows as the task sees it right after the
1161- // first batch, and again after the last. The task creates the native plan, which starts the
1162- // interval's clock, when it builds the iterator, so the sleep before the first batch runs
1160+ // Ten output batches from one task. Passes `check` the output_rows the task sees right after
1161+ // the first batch, and again after the last. The task creates the native plan, which starts
1162+ // the interval's clock, when it builds the iterator, so the sleep before the first batch runs
11631163 // after that.
1164- def outputRowsMidStreamAndAtEnd (interval : String ): (Long , Long ) =
1164+ def checkOutputRows (interval : String )( check : (Long , Long ) => Unit ) : Unit =
11651165 withSQLConf(
11661166 CometConf .COMET_BATCH_SIZE .key -> " 1000" ,
11671167 CometConf .COMET_METRICS_UPDATE_INTERVAL .key -> interval) {
@@ -1172,7 +1172,7 @@ class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper {
11721172 // The task deserializes this metric together with the scan's metric node, so both
11731173 // refer to the task-side copy that native execution publishes into.
11741174 val outputRows = scan.metrics(" output_rows" )
1175- SQLExecution .withSQLConfPropagated(spark) {
1175+ val (midStream, atEnd) = SQLExecution .withSQLConfPropagated(spark) {
11761176 scan
11771177 .executeColumnar()
11781178 .mapPartitions { batches =>
@@ -1185,24 +1185,21 @@ class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper {
11851185 .collect()
11861186 .head
11871187 }
1188+ check(midStream, atEnd)
11881189 }
11891190
11901191 // With the interval disabled, the only publish is the one in releasePlan, which a
11911192 // per-batch publish would break after the first batch.
1192- val (disabledMidStream, disabledAtEnd) = outputRowsMidStreamAndAtEnd(" -1" )
1193- assert(disabledMidStream == 0 , s " output_rows was published mid-stream: $disabledMidStream" )
1194- assert(
1195- disabledAtEnd == 10000 ,
1196- s " releasePlan should publish the final output_rows, got $disabledAtEnd" )
1193+ checkOutputRows(" -1" ) { (midStream, atEnd) =>
1194+ assert(midStream == 0 , s " output_rows was published mid-stream: $midStream" )
1195+ assert(atEnd == 10000 , s " releasePlan should publish the final output_rows, got $atEnd" )
1196+ }
11971197
11981198 // A 1 ms interval has passed by the time the first batch arrives, so that batch publishes.
1199- val (dueMidStream, dueAtEnd) = outputRowsMidStreamAndAtEnd(" 1" )
1200- assert(
1201- dueMidStream > 0 ,
1202- " output_rows was not published mid-stream once the interval passed" )
1203- assert(
1204- dueAtEnd == 10000 ,
1205- s " releasePlan should publish the final output_rows, got $dueAtEnd" )
1199+ checkOutputRows(" 1" ) { (midStream, atEnd) =>
1200+ assert(midStream > 0 , " output_rows was not published mid-stream once the interval passed" )
1201+ assert(atEnd == 10000 , s " releasePlan should publish the final output_rows, got $atEnd" )
1202+ }
12061203 }
12071204 }
12081205
0 commit comments