@@ -1157,33 +1157,52 @@ class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper {
11571157 test(" native block without a JVM input publishes SQL metrics on the update interval" ) {
11581158 withTempPath { dir =>
11591159 spark.range(0 , 10000 , 1 , 1 ).write.parquet(dir.getAbsolutePath)
1160- // Ten output batches from one task, so a per-batch publish shows up after the first one.
1161- // With the interval disabled, the only publish is the one in releasePlan.
1162- withSQLConf(
1163- CometConf .COMET_BATCH_SIZE .key -> " 1000" ,
1164- CometConf .COMET_METRICS_UPDATE_INTERVAL .key -> " -1" ) {
1165- val plan = spark.read.parquet(dir.getAbsolutePath).queryExecution.executedPlan
1166- val scan = find(plan)(_.isInstanceOf [CometNativeScanExec ])
1167- .getOrElse(fail(s " Expected CometNativeScanExec in plan: \n ${plan.treeString}" ))
1168- .asInstanceOf [CometNativeScanExec ]
1169- // The task deserializes this metric together with the scan's metric node, so both refer
1170- // to the task-side copy that native execution publishes into.
1171- val outputRows = scan.metrics(" output_rows" )
1172- val (midStream, atEnd) = SQLExecution .withSQLConfPropagated(spark) {
1173- scan
1174- .executeColumnar()
1175- .mapPartitions { batches =>
1176- batches.next()
1177- val mid = outputRows.value
1178- batches.foreach(_ => ())
1179- Iterator ((mid, outputRows.value))
1180- }
1181- .collect()
1182- .head
1160+ // Ten output batches from one task. Returns output_rows as the task sees it right after the
1161+ // first batch, and again after the last. The task creates the native plan, which starts the
1162+ // interval's clock, when it builds the iterator, so the sleep before the first batch runs
1163+ // after that.
1164+ def outputRowsMidStreamAndAtEnd (interval : String ): (Long , Long ) =
1165+ withSQLConf(
1166+ CometConf .COMET_BATCH_SIZE .key -> " 1000" ,
1167+ CometConf .COMET_METRICS_UPDATE_INTERVAL .key -> interval) {
1168+ val plan = spark.read.parquet(dir.getAbsolutePath).queryExecution.executedPlan
1169+ val scan = find(plan)(_.isInstanceOf [CometNativeScanExec ])
1170+ .getOrElse(fail(s " Expected CometNativeScanExec in plan: \n ${plan.treeString}" ))
1171+ .asInstanceOf [CometNativeScanExec ]
1172+ // The task deserializes this metric together with the scan's metric node, so both
1173+ // refer to the task-side copy that native execution publishes into.
1174+ val outputRows = scan.metrics(" output_rows" )
1175+ SQLExecution .withSQLConfPropagated(spark) {
1176+ scan
1177+ .executeColumnar()
1178+ .mapPartitions { batches =>
1179+ Thread .sleep(100 )
1180+ batches.next()
1181+ val mid = outputRows.value
1182+ batches.foreach(_ => ())
1183+ Iterator ((mid, outputRows.value))
1184+ }
1185+ .collect()
1186+ .head
1187+ }
11831188 }
1184- assert(midStream == 0 , s " output_rows was published mid-stream: $midStream" )
1185- assert(atEnd == 10000 , s " releasePlan should publish the final output_rows, got $atEnd" )
1186- }
1189+
1190+ // With the interval disabled, the only publish is the one in releasePlan, which a
1191+ // per-batch publish would break after the first batch.
1192+ val (disabledMidStream, disabledAtEnd) = outputRowsMidStreamAndAtEnd(" -1" )
1193+ assert(disabledMidStream == 0 , s " output_rows was published mid-stream: $disabledMidStream" )
1194+ assert(
1195+ disabledAtEnd == 10000 ,
1196+ s " releasePlan should publish the final output_rows, got $disabledAtEnd" )
1197+
1198+ // A 1 ms interval has passed by the time the first batch arrives, so that batch publishes.
1199+ val (dueMidStream, dueAtEnd) = outputRowsMidStreamAndAtEnd(" 1" )
1200+ assert(
1201+ dueMidStream > 0 ,
1202+ " output_rows was not published mid-stream once the interval passed" )
1203+ assert(
1204+ dueAtEnd == 10000 ,
1205+ s " releasePlan should publish the final output_rows, got $dueAtEnd" )
11871206 }
11881207 }
11891208
0 commit comments