Skip to content

Commit df15126

Browse files
committed
chore: drop a redundant interpolator and state the listener-order caveat in the scaladoc
1 parent 5d727c1 commit df15126

3 files changed

Lines changed: 10 additions & 5 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,12 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM
110110
* a limit, leaves the final publish to that close.
111111
*
112112
* Adds to the task's counters instead of replacing them, so bytes that a fallback Spark scan
113-
* accumulated in the same task survive. Trees registered on one task may share accumulators
114-
* (see [[reportSpillMetrics]]), so each accumulator is counted once per task.
113+
* accumulated in the same task survive. That holds only when the fallback scan registers its
114+
* completion listener after this one, which a `CometSparkToColumnarExec` input always does and
115+
* a coalesced Spark-scan partition computed first does not: `FileScanRDD`'s close then runs
116+
* last and sets bytesRead from the value it snapshotted at construction. Trees registered on
117+
* one task may share accumulators (see [[reportSpillMetrics]]), so each accumulator is counted
118+
* once per task.
115119
*/
116120
def reportScanInputMetrics(ctx: TaskContext): Unit = {
117121
val seenMetrics = CometMetricNode.taskSeenMetrics(ctx).scanInput

‎spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3779,8 +3779,9 @@ class CometIcebergNativeSuite
37793779
spark.sparkContext.addSparkListener(listener)
37803780

37813781
try {
3782-
// The limit stops pulling before the scan is exhausted, so the final metric publish
3783-
// happens in the iterator's completion-time close.
3782+
// This Iceberg scan is its own native block with no JVM input, so its metrics publish
3783+
// per batch and this covers the registration site rather than the listener order,
3784+
// which stays unguarded for a fused Iceberg scan until the scan-input gate widens.
37843785
val query = "SELECT * FROM test_cat.db.task_metrics_limit_test LIMIT 3"
37853786
Seq("-1", CometConf.COMET_METRICS_UPDATE_INTERVAL.defaultValueString).foreach {
37863787
interval =>

‎spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1202,7 +1202,7 @@ class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper {
12021202
s"recordsRead mismatch: comet=$cometRecordsReversed, spark=$sparkRecordsReversed")
12031203
assert(
12041204
cometBytesReversed >= fallbackBytes && cometBytesReversed < nativeBytes + fallbackBytes,
1205-
s"bytesRead with the fallback side first keeps only its own bytes today: " +
1205+
"bytesRead with the fallback side first keeps only its own bytes today: " +
12061206
s"comet=$cometBytesReversed, native=$nativeBytes, fallback=$fallbackBytes")
12071207
}
12081208
}

0 commit comments

Comments
 (0)