Skip to content

Commit 00a4b42

Browse files
viiryaIsaac
andauthored
test: drain the listener bus before registering Iceberg test listeners (#6809)
Co-authored-by: Isaac <[email protected]>
1 parent 14e49a0 commit 00a4b42

4 files changed

Lines changed: 10 additions & 0 deletions

File tree

‎spark/src/test/scala/org/apache/comet/CometIcebergRewriteActionSuite.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,7 @@ class CometIcebergRewriteActionSuite extends CometTestBase with CometIcebergTest
373373
case _ =>
374374
}
375375
}
376+
CometListenerBusUtils.waitUntilEmpty(spark.sparkContext)
376377
spark.sparkContext.addSparkListener(listener)
377378
try {
378379
body

‎spark/src/test/scala/org/apache/comet/CometIcebergTestBase.scala‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,9 @@ trait CometIcebergTestBase { this: CometTestBase =>
237237
override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit =
238238
if (includeFailures) captured += qe.executedPlan
239239
}
240+
// Events from earlier queries may still be queued; drain them so they do not reach the
241+
// listener.
242+
CometListenerBusUtils.waitUntilEmpty(spark.sparkContext)
240243
spark.listenerManager.register(listener)
241244
try {
242245
action
@@ -259,6 +262,7 @@ trait CometIcebergTestBase { this: CometTestBase =>
259262
override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit =
260263
captured += qe.executedPlan
261264
}
265+
CometListenerBusUtils.waitUntilEmpty(spark.sparkContext)
262266
spark.listenerManager.register(listener)
263267
try {
264268
val error =

‎spark/src/test/scala/org/apache/comet/CometIcebergWriteActionSuite.scala‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3411,6 +3411,8 @@ class CometIcebergWriteActionSuite
34113411
override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit =
34123412
if (taskEnd.reason == Success) JobAbortGate.taskFinished()
34133413
}
3414+
// Drain the setup writes' task-end events first, or they can count toward the gate.
3415+
CometListenerBusUtils.waitUntilEmpty(spark.sparkContext)
34143416
spark.sparkContext.addSparkListener(listener)
34153417
try {
34163418
val run = () =>
@@ -4309,6 +4311,8 @@ class CometIcebergWriteActionSuite
43094311
val listener = new IcebergWriteReportListener(
43104312
new SparkConf()
43114313
.set(CometConf.COMET_ICEBERG_WRITE_REPORT_DIR.key, reportDir.getAbsolutePath))
4314+
// Drain the setup writes' events first, or they can reach the listener and be reported.
4315+
CometListenerBusUtils.waitUntilEmpty(spark.sparkContext)
43124316
spark.listenerManager.register(listener)
43134317
try {
43144318
action

‎spark/src/test/scala/org/apache/spark/sql/benchmark/CometIcebergWriteBenchmark.scala‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -529,6 +529,7 @@ object CometIcebergWriteBenchmark extends CometBenchmarkBase {
529529
qe: QueryExecution,
530530
exception: Exception): Unit = {}
531531
}
532+
CometListenerBusUtils.waitUntilEmpty(spark.sparkContext)
532533
spark.listenerManager.register(listener)
533534
try {
534535
action

0 commit comments

Comments
 (0)