@@ -77,11 +77,7 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM
7777 seenMetrics : IdentityHashMap [SQLMetric , java.lang.Boolean ]): Long = {
7878 def sumFromNode (metricNode : CometMetricNode ): Long = {
7979 val nodeValue = metricNode.metrics.get(metricName).fold(0L ) { metric =>
80- if (seenMetrics.put(metric, java.lang.Boolean .TRUE ) == null ) {
81- math.max(metric.value, 0L )
82- } else {
83- 0L
84- }
80+ CometMetricNode .claimMetricValue(metric, seenMetrics)
8581 }
8682 nodeValue + metricNode.children.iterator.map(sumFromNode).sum
8783 }
@@ -105,24 +101,38 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM
105101 })
106102
107103 /**
108- * Reports aggregated scan input metrics (bytesRead, recordsRead) to Spark's task metrics.
109- * Aggregates across all scan leaf nodes to handle plans with multiple scans (e.g., joins). Must
110- * be called in a TaskCompletionListener after the iterator is fully consumed.
104+ * Reports the scan leaves' bytes and rows (summed across joins and unions) to Spark's task
105+ * input metrics, which drive the Input column on the UI's Stages and Executors tabs.
106+ *
107+ * Must be registered on the task thread before [[org.apache.comet.CometExecIterator ]] so its
108+ * completion listener publishes final SQL metrics before this listener runs. A block with a JVM
109+ * input only publishes on the metrics update interval, and a consumer that stops early, such as
110+ * a limit, leaves the final publish to that close.
111+ *
112+ * Adds to the task's counters instead of replacing them, so bytes that a fallback Spark scan
113+ * accumulated in the same task survive. That holds only when the fallback scan registers its
114+ * completion listener after this one, which a `CometSparkToColumnarExec` input always does and
115+ * a coalesced Spark-scan partition computed first does not: `FileScanRDD`'s close then runs
116+ * last and sets bytesRead from the value it snapshotted at construction. Trees registered on
117+ * one task may share accumulators (see [[reportSpillMetrics ]]), so each accumulator is counted
118+ * once per task.
111119 */
112120 def reportScanInputMetrics (ctx : TaskContext ): Unit = {
121+ val seenMetrics = CometMetricNode .taskSeenMetrics(ctx).scanInput
113122 ctx.addTaskCompletionListener[Unit ] { _ =>
114123 val scanLeaves = leafNodes.filter(_.metrics.contains(" bytes_scanned" ))
115- if (scanLeaves.nonEmpty) {
116- val totalBytes = scanLeaves.map(_.metrics(" bytes_scanned" ).value).sum
117- val totalRows = scanLeaves.map { leaf =>
118- val outputRows =
119- leaf.metrics.get(" output_rows" ).map(_.value).getOrElse(0L )
120- val prunedRows =
121- leaf.metrics.get(" pushdown_rows_pruned" ).map(_.value).getOrElse(0L )
122- outputRows + prunedRows
123- }.sum
124- ctx.taskMetrics().inputMetrics.setBytesRead(totalBytes)
125- ctx.taskMetrics().inputMetrics.setRecordsRead(totalRows)
124+ def claimed (leaf : CometMetricNode , metricName : String ): Long =
125+ leaf.metrics.get(metricName).fold(0L )(CometMetricNode .claimMetricValue(_, seenMetrics))
126+
127+ val totalBytes = scanLeaves.map(claimed(_, " bytes_scanned" )).sum
128+ val totalRows = scanLeaves.map { leaf =>
129+ claimed(leaf, " output_rows" ) + claimed(leaf, " pushdown_rows_pruned" )
130+ }.sum
131+ if (totalBytes > 0L ) {
132+ ctx.taskMetrics().inputMetrics.incBytesRead(totalBytes)
133+ }
134+ if (totalRows > 0L ) {
135+ ctx.taskMetrics().inputMetrics.incRecordsRead(totalRows)
126136 }
127137 }
128138 }
@@ -135,6 +145,10 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM
135145 * Must be registered on the task thread before [[org.apache.comet.CometExecIterator ]] so
136146 * Spark's completion listener stack invokes the iterator `close` (final SQL metric update)
137147 * before this listener runs.
148+ *
149+ * The native writer is the root of its task's plan and runs once per task, so its values
150+ * replace the task's output counters without the per-task registry that the scan input and
151+ * spill reports share.
138152 */
139153 def reportNativeWriteOutputMetrics (ctx : TaskContext ): Unit = {
140154 ctx.addTaskCompletionListener[Unit ] { _ =>
@@ -161,7 +175,7 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM
161175 * per-task registry, so each accumulator is counted once while disjoint trees still all report.
162176 */
163177 def reportSpillMetrics (ctx : TaskContext ): Unit = {
164- val seenMetrics = CometMetricNode .taskSeenSpillMetrics (ctx)
178+ val seenMetrics = CometMetricNode .taskSeenMetrics (ctx)
165179 ctx.addTaskCompletionListener[Unit ] { _ =>
166180 val diskBytesSpilled = sumMetricValues(" spilled_bytes" , seenMetrics.disk)
167181 if (diskBytesSpilled > 0L ) {
@@ -225,30 +239,39 @@ object CometMetricNode {
225239 private val aggregateMetricNames =
226240 Set (" spill_count" , " spilled_bytes" , " spilled_rows" , " peak_mem_used" )
227241
228- private case class SeenSpillMetrics (
229- disk : IdentityHashMap [SQLMetric , java.lang.Boolean ],
230- memory : IdentityHashMap [SQLMetric , java.lang.Boolean ])
242+ private type SeenMetricSet = IdentityHashMap [SQLMetric , java.lang.Boolean ]
243+
244+ private case class SeenMetrics (
245+ disk : SeenMetricSet ,
246+ memory : SeenMetricSet ,
247+ scanInput : SeenMetricSet )
231248
232- // Per running task attempt: the spill accumulators already claimed by a reporting listener,
233- // one identity set per metric name (see reportSpillMetrics). The first registration installs
234- // a cleanup listener ahead of every reporting listener, so it runs last (reverse registration
235- // order) and removes the entry.
236- private val seenSpillMetricsByTask = new ConcurrentHashMap [Long , SeenSpillMetrics ]()
249+ // Per running task attempt: the accumulators already claimed by a reporting listener, one
250+ // identity set per reported task metric (see reportSpillMetrics and reportScanInputMetrics).
251+ // The first registration installs a cleanup listener ahead of every reporting listener, so it
252+ // runs last (reverse registration order) and removes the entry.
253+ private val seenMetricsByTask = new ConcurrentHashMap [Long , SeenMetrics ]()
237254
238- private def taskSeenSpillMetrics (ctx : TaskContext ): SeenSpillMetrics = {
255+ private def taskSeenMetrics (ctx : TaskContext ): SeenMetrics = {
239256 val attemptId = ctx.taskAttemptId()
240- val existing = seenSpillMetricsByTask .get(attemptId)
257+ val existing = seenMetricsByTask .get(attemptId)
241258 if (existing != null ) {
242259 existing
243260 } else {
244261 // The task thread is the only registrant for its attempt id, so there is no put race.
245- val created = SeenSpillMetrics (new IdentityHashMap (), new IdentityHashMap ())
246- seenSpillMetricsByTask.put(attemptId, created)
247- ctx.addTaskCompletionListener[Unit ](_ => seenSpillMetricsByTask.remove(attemptId))
262+ val created =
263+ SeenMetrics (new IdentityHashMap (), new IdentityHashMap (), new IdentityHashMap ())
264+ seenMetricsByTask.put(attemptId, created)
265+ ctx.addTaskCompletionListener[Unit ](_ => seenMetricsByTask.remove(attemptId))
248266 created
249267 }
250268 }
251269
270+ /** The metric's value the first time it is claimed for a task, zero afterwards. */
271+ private def claimMetricValue (metric : SQLMetric , seenMetrics : SeenMetricSet ): Long =
272+ if (seenMetrics.put(metric, java.lang.Boolean .TRUE ) == null ) math.max(metric.value, 0L )
273+ else 0L
274+
252275 /**
253276 * The baseline SQL metrics for DataFusion `BaselineMetrics`.
254277 */
0 commit comments