Skip to content

Commit f088a54

Browse files
authored
fix: report task input metrics after the native iterator closes and add to Spark's counters (apache#5880)
* fix: report task input metrics after the native iterator closes and add to Spark's counters Register the scan input metrics listener before super.compute creates the CometExecIterator at the native scan, Iceberg scan and CometExecRDD sites. Spark runs completion listeners in reverse registration order, so the report now runs after the iterator's close publishes the final native metrics. A block with a JVM input only publishes on the metrics update interval, and a consumer that stops early, such as a limit, left the report reading zero. Add to the task's bytesRead and recordsRead instead of replacing them, so bytes a fallback Spark scan or a cached input accumulated in the same task survive, and claim each accumulator once per task through the registry the spill reporter already uses. * chore: drop a redundant interpolator and state the listener-order caveat in the scaladoc
1 parent 31b3819 commit f088a54

6 files changed

Lines changed: 512 additions & 42 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/CometIcebergNativeScanExec.scala‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -287,9 +287,10 @@ case class CometIcebergNativeScanExec(
287287
nativeMetrics = nativeMetrics,
288288
subqueries = Seq.empty) {
289289
override def compute(split: Partition, context: TaskContext): Iterator[ColumnarBatch] = {
290-
val res = super.compute(split, context)
290+
// Register before super.compute creates the CometExecIterator, so this listener runs
291+
// after the iterator's close has published the final scan metrics.
291292
Option(context).foreach(nativeMetrics.reportScanInputMetrics)
292-
res
293+
super.compute(split, context)
293294
}
294295
}
295296
}

‎spark/src/main/scala/org/apache/spark/sql/comet/CometMetricNode.scala‎

Lines changed: 56 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -77,11 +77,7 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM
7777
seenMetrics: IdentityHashMap[SQLMetric, java.lang.Boolean]): Long = {
7878
def sumFromNode(metricNode: CometMetricNode): Long = {
7979
val nodeValue = metricNode.metrics.get(metricName).fold(0L) { metric =>
80-
if (seenMetrics.put(metric, java.lang.Boolean.TRUE) == null) {
81-
math.max(metric.value, 0L)
82-
} else {
83-
0L
84-
}
80+
CometMetricNode.claimMetricValue(metric, seenMetrics)
8581
}
8682
nodeValue + metricNode.children.iterator.map(sumFromNode).sum
8783
}
@@ -105,24 +101,38 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM
105101
})
106102

107103
/**
108-
* Reports aggregated scan input metrics (bytesRead, recordsRead) to Spark's task metrics.
109-
* Aggregates across all scan leaf nodes to handle plans with multiple scans (e.g., joins). Must
110-
* be called in a TaskCompletionListener after the iterator is fully consumed.
104+
* Reports the scan leaves' bytes and rows (summed across joins and unions) to Spark's task
105+
* input metrics, which drive the Input column on the UI's Stages and Executors tabs.
106+
*
107+
* Must be registered on the task thread before [[org.apache.comet.CometExecIterator]] so its
108+
* completion listener publishes final SQL metrics before this listener runs. A block with a JVM
109+
* input only publishes on the metrics update interval, and a consumer that stops early, such as
110+
* a limit, leaves the final publish to that close.
111+
*
112+
* Adds to the task's counters instead of replacing them, so bytes that a fallback Spark scan
113+
* accumulated in the same task survive. That holds only when the fallback scan registers its
114+
* completion listener after this one, which a `CometSparkToColumnarExec` input always does and
115+
* a coalesced Spark-scan partition computed first does not: `FileScanRDD`'s close then runs
116+
* last and sets bytesRead from the value it snapshotted at construction. Trees registered on
117+
* one task may share accumulators (see [[reportSpillMetrics]]), so each accumulator is counted
118+
* once per task.
111119
*/
112120
def reportScanInputMetrics(ctx: TaskContext): Unit = {
121+
val seenMetrics = CometMetricNode.taskSeenMetrics(ctx).scanInput
113122
ctx.addTaskCompletionListener[Unit] { _ =>
114123
val scanLeaves = leafNodes.filter(_.metrics.contains("bytes_scanned"))
115-
if (scanLeaves.nonEmpty) {
116-
val totalBytes = scanLeaves.map(_.metrics("bytes_scanned").value).sum
117-
val totalRows = scanLeaves.map { leaf =>
118-
val outputRows =
119-
leaf.metrics.get("output_rows").map(_.value).getOrElse(0L)
120-
val prunedRows =
121-
leaf.metrics.get("pushdown_rows_pruned").map(_.value).getOrElse(0L)
122-
outputRows + prunedRows
123-
}.sum
124-
ctx.taskMetrics().inputMetrics.setBytesRead(totalBytes)
125-
ctx.taskMetrics().inputMetrics.setRecordsRead(totalRows)
124+
def claimed(leaf: CometMetricNode, metricName: String): Long =
125+
leaf.metrics.get(metricName).fold(0L)(CometMetricNode.claimMetricValue(_, seenMetrics))
126+
127+
val totalBytes = scanLeaves.map(claimed(_, "bytes_scanned")).sum
128+
val totalRows = scanLeaves.map { leaf =>
129+
claimed(leaf, "output_rows") + claimed(leaf, "pushdown_rows_pruned")
130+
}.sum
131+
if (totalBytes > 0L) {
132+
ctx.taskMetrics().inputMetrics.incBytesRead(totalBytes)
133+
}
134+
if (totalRows > 0L) {
135+
ctx.taskMetrics().inputMetrics.incRecordsRead(totalRows)
126136
}
127137
}
128138
}
@@ -135,6 +145,10 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM
135145
* Must be registered on the task thread before [[org.apache.comet.CometExecIterator]] so
136146
* Spark's completion listener stack invokes the iterator `close` (final SQL metric update)
137147
* before this listener runs.
148+
*
149+
* The native writer is the root of its task's plan and runs once per task, so its values
150+
* replace the task's output counters without the per-task registry that the scan input and
151+
* spill reports share.
138152
*/
139153
def reportNativeWriteOutputMetrics(ctx: TaskContext): Unit = {
140154
ctx.addTaskCompletionListener[Unit] { _ =>
@@ -161,7 +175,7 @@ case class CometMetricNode(metrics: Map[String, SQLMetric], children: Seq[CometM
161175
* per-task registry, so each accumulator is counted once while disjoint trees still all report.
162176
*/
163177
def reportSpillMetrics(ctx: TaskContext): Unit = {
164-
val seenMetrics = CometMetricNode.taskSeenSpillMetrics(ctx)
178+
val seenMetrics = CometMetricNode.taskSeenMetrics(ctx)
165179
ctx.addTaskCompletionListener[Unit] { _ =>
166180
val diskBytesSpilled = sumMetricValues("spilled_bytes", seenMetrics.disk)
167181
if (diskBytesSpilled > 0L) {
@@ -225,30 +239,39 @@ object CometMetricNode {
225239
private val aggregateMetricNames =
226240
Set("spill_count", "spilled_bytes", "spilled_rows", "peak_mem_used")
227241

228-
private case class SeenSpillMetrics(
229-
disk: IdentityHashMap[SQLMetric, java.lang.Boolean],
230-
memory: IdentityHashMap[SQLMetric, java.lang.Boolean])
242+
private type SeenMetricSet = IdentityHashMap[SQLMetric, java.lang.Boolean]
243+
244+
private case class SeenMetrics(
245+
disk: SeenMetricSet,
246+
memory: SeenMetricSet,
247+
scanInput: SeenMetricSet)
231248

232-
// Per running task attempt: the spill accumulators already claimed by a reporting listener,
233-
// one identity set per metric name (see reportSpillMetrics). The first registration installs
234-
// a cleanup listener ahead of every reporting listener, so it runs last (reverse registration
235-
// order) and removes the entry.
236-
private val seenSpillMetricsByTask = new ConcurrentHashMap[Long, SeenSpillMetrics]()
249+
// Per running task attempt: the accumulators already claimed by a reporting listener, one
250+
// identity set per reported task metric (see reportSpillMetrics and reportScanInputMetrics).
251+
// The first registration installs a cleanup listener ahead of every reporting listener, so it
252+
// runs last (reverse registration order) and removes the entry.
253+
private val seenMetricsByTask = new ConcurrentHashMap[Long, SeenMetrics]()
237254

238-
private def taskSeenSpillMetrics(ctx: TaskContext): SeenSpillMetrics = {
255+
private def taskSeenMetrics(ctx: TaskContext): SeenMetrics = {
239256
val attemptId = ctx.taskAttemptId()
240-
val existing = seenSpillMetricsByTask.get(attemptId)
257+
val existing = seenMetricsByTask.get(attemptId)
241258
if (existing != null) {
242259
existing
243260
} else {
244261
// The task thread is the only registrant for its attempt id, so there is no put race.
245-
val created = SeenSpillMetrics(new IdentityHashMap(), new IdentityHashMap())
246-
seenSpillMetricsByTask.put(attemptId, created)
247-
ctx.addTaskCompletionListener[Unit](_ => seenSpillMetricsByTask.remove(attemptId))
262+
val created =
263+
SeenMetrics(new IdentityHashMap(), new IdentityHashMap(), new IdentityHashMap())
264+
seenMetricsByTask.put(attemptId, created)
265+
ctx.addTaskCompletionListener[Unit](_ => seenMetricsByTask.remove(attemptId))
248266
created
249267
}
250268
}
251269

270+
/** The metric's value the first time it is claimed for a task, zero afterwards. */
271+
private def claimMetricValue(metric: SQLMetric, seenMetrics: SeenMetricSet): Long =
272+
if (seenMetrics.put(metric, java.lang.Boolean.TRUE) == null) math.max(metric.value, 0L)
273+
else 0L
274+
252275
/**
253276
* The baseline SQL metrics for DataFusion `BaselineMetrics`.
254277
*/

‎spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -278,12 +278,10 @@ case class CometNativeScanExec(
278278
encryptedFilePaths,
279279
perPartitionFilePaths = perPartitionFilePaths) {
280280
override def compute(split: Partition, context: TaskContext): Iterator[ColumnarBatch] = {
281-
val res = super.compute(split, context)
282-
283-
// Report scan input metrics after the iterator is fully consumed.
281+
// Register before super.compute creates the CometExecIterator, so this listener runs
282+
// after the iterator's close has published the final scan metrics.
284283
Option(context).foreach(nativeMetrics.reportScanInputMetrics)
285-
286-
res
284+
super.compute(split, context)
287285
}
288286
}
289287
}

‎spark/src/main/scala/org/apache/spark/sql/comet/operators.scala‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -854,11 +854,12 @@ abstract class CometNativeExec extends CometExec {
854854
ctx.shuffleScanIndices,
855855
perPartitionFilePaths = ctx.perPartitionFilePaths) {
856856
override def compute(split: Partition, context: TaskContext): Iterator[ColumnarBatch] = {
857-
val res = super.compute(split, context)
857+
// Register before super.compute creates the CometExecIterator, so this listener runs
858+
// after the iterator's close has published the final scan metrics.
858859
if (ctx.hasScanInput) {
859860
Option(context).foreach(nativeMetrics.reportScanInputMetrics)
860861
}
861-
res
862+
super.compute(split, context)
862863
}
863864
}
864865
}

‎spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4290,6 +4290,49 @@ class CometIcebergNativeSuite
42904290
}
42914291
}
42924292

4293+
test("Iceberg native scan left unconsumed by a limit still reports task input metrics") {
4294+
assume(icebergAvailable, "Iceberg not available in classpath")
4295+
4296+
withTempIcebergDir { warehouseDir =>
4297+
withSQLConf(
4298+
"spark.sql.catalog.test_cat" -> "org.apache.iceberg.spark.SparkCatalog",
4299+
"spark.sql.catalog.test_cat.type" -> "hadoop",
4300+
"spark.sql.catalog.test_cat.warehouse" -> warehouseDir.getAbsolutePath,
4301+
CometConf.COMET_ENABLED.key -> "true",
4302+
CometConf.COMET_EXEC_ENABLED.key -> "true",
4303+
CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true") {
4304+
4305+
createInputMetricsTable("test_cat.db.task_metrics_limit_test")
4306+
try {
4307+
// This Iceberg scan is its own native block with no JVM input, so its metrics publish
4308+
// per batch and this covers the registration site rather than the listener order. A
4309+
// fused Iceberg scan reports from the same CometNativeExec site as a fused Parquet
4310+
// scan, whose order CometTaskMetricsSuite's broadcast join limit test guards.
4311+
val query = "SELECT * FROM test_cat.db.task_metrics_limit_test LIMIT 3"
4312+
Seq("-1", CometConf.COMET_METRICS_UPDATE_INTERVAL.defaultValueString).foreach {
4313+
interval =>
4314+
withSQLConf(CometConf.COMET_METRICS_UPDATE_INTERVAL.key -> interval) {
4315+
val df = spark.sql(query)
4316+
val (bytesRead, recordsRead) = taskInputMetrics(df.collect())
4317+
assert(
4318+
collectIcebergNativeScans(df.queryExecution.executedPlan).nonEmpty,
4319+
"Expected CometIcebergNativeScanExec in plan")
4320+
assert(
4321+
bytesRead > 0,
4322+
s"bytesRead should be > 0 at interval $interval, got $bytesRead")
4323+
assert(
4324+
recordsRead >= 3 && recordsRead <= inputMetricsRows,
4325+
s"recordsRead should cover at least the limit at interval $interval, " +
4326+
s"got $recordsRead")
4327+
}
4328+
}
4329+
} finally {
4330+
spark.sql("DROP TABLE test_cat.db.task_metrics_limit_test")
4331+
}
4332+
}
4333+
}
4334+
}
4335+
42934336
test("exchange reuse must not collapse scans with different pushed filters (#4774)") {
42944337
assume(icebergAvailable, "Iceberg not available")
42954338

0 commit comments

Comments
 (0)