Skip to content

Commit c190f6e

Browse files
committed
Expose Spark's own scan, not the cached plan, as the subquery
Spark 4.2's SQLLastAttemptAccumulator walks a plan's subqueries to find a metric's stages and gives up on a shuffle it does not know. With the cached plan as the subquery it reached the cached plan's Comet shuffle, so a last-attempt metric used outside the cache returned None. SparkPlanInfo draws the cached plan below Spark's own InMemoryTableScanExec, matched by class, so expose that scan instead. Every other walker of subqueries stops at it, as in Spark's own plans. Adds a check on every version that collectWithSubqueries does not reach the cached plan's shuffle, and a Spark 4.2 suite for the outside-cache metric.
1 parent 6685125 commit c190f6e

5 files changed

Lines changed: 93 additions & 9 deletions

File tree

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -518,6 +518,7 @@ jobs:
518518
org.apache.comet.exec.CometInMemoryCacheKryoSuite
519519
org.apache.comet.exec.CometInMemoryCacheKryoUnregisteredSuite
520520
org.apache.comet.exec.CometInMemoryCacheKryoClassesToRegisterSuite
521+
org.apache.comet.exec.CometInMemoryCacheLastAttemptMetricSuite
521522
org.apache.comet.exec.CometGenerateExecSuite
522523
org.apache.comet.exec.CometWindowExecSuite
523524
org.apache.comet.exec.CometJoinSuite

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,7 @@ jobs:
222222
org.apache.comet.exec.CometInMemoryCacheKryoSuite
223223
org.apache.comet.exec.CometInMemoryCacheKryoUnregisteredSuite
224224
org.apache.comet.exec.CometInMemoryCacheKryoClassesToRegisterSuite
225+
org.apache.comet.exec.CometInMemoryCacheLastAttemptMetricSuite
225226
org.apache.comet.exec.CometGenerateExecSuite
226227
org.apache.comet.exec.CometWindowExecSuite
227228
org.apache.comet.exec.CometJoinSuite

‎spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -86,13 +86,14 @@ case class CometInMemoryTableScanExec(
8686
// ExtendedExplainInfo.executionInnerChildren leaves them out of Comet's own reporting.
8787
override def innerChildren: Seq[QueryPlan[_]] = Seq(originalPlan.relation)
8888

89-
// SparkPlanInfo, behind the SQL tab's graph and the event log, gives Spark's own scan (matched
90-
// by class) its cached plan as a child, and any other node its children and subqueries. Expose
91-
// the cached plan as a subquery: unlike a child it is only walked, never run, since subqueries
92-
// run from expressions. Walkers such as collectWithSubqueries reach it too. innerChildren above
93-
// leaves out super.innerChildren, which is this list, or EXPLAIN would draw the cached plan
94-
// twice. A lazy val overrides both Spark 3.x's lazy val and Spark 4's def.
95-
@transient override lazy val subqueries: Seq[SparkPlan] = Seq(originalPlan.relation.cachedPlan)
89+
// SparkPlanInfo, behind the SQL tab's graph and the event log, draws Spark's own scan (matched
90+
// by class) with its cached plan below, and any other node with its children and subqueries.
91+
// So expose that scan as the one subquery. It never runs, since subqueries run from
92+
// expressions, and other walkers of subqueries stop at it, as at Spark's scan in Spark's own
93+
// plans. Exposing the cached plan instead lets them in: Spark 4.2's last-attempt metrics then
94+
// give up on its Comet shuffle. innerChildren above must not add super.innerChildren, which is
95+
// this list. A lazy val overrides both Spark 3.x's lazy val and Spark 4's def.
96+
@transient override lazy val subqueries: Seq[SparkPlan] = Seq(originalPlan)
9697

9798
// `originalPlan` is a plan-typed field rather than a child, so QueryPlan's canonicalization
9899
// walks straight past it: its attributes and predicates keep the expression IDs of whichever

‎spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -456,10 +456,18 @@ class CometInMemoryCacheSuite extends CometTestBase {
456456
val plan = df.queryExecution.executedPlan
457457
val scans = collect(plan) { case s: CometInMemoryTableScanExec => s }
458458
assert(scans.size == 1, plan)
459+
val sparkScan = scans.head.originalPlan
459460
val cachedPlanInfo =
460-
CometSparkPlanInfoHelper.fromSparkPlan(scans.head.originalPlan.relation.cachedPlan)
461+
CometSparkPlanInfoHelper.fromSparkPlan(sparkScan.relation.cachedPlan)
462+
// Spark's own scan of the cache sits below, and the cached plan below that.
461463
val infos = scanInfos(CometSparkPlanInfoHelper.fromSparkPlan(plan))
462-
assert(infos.map(_.children) == Seq(Seq(cachedPlanInfo)), plan)
464+
assert(
465+
infos.map(_.children.map(info => (info.nodeName, info.children))) ==
466+
Seq(Seq((sparkScan.nodeName, Seq(cachedPlanInfo)))),
467+
plan)
468+
// Other walkers of subqueries stop at Spark's scan, as in Spark's own plans, so they
469+
// do not find the cached plan's shuffle in this query, which has none of its own.
470+
assert(collectWithSubqueries(plan) { case e: ShuffleExchangeLike => e }.isEmpty, plan)
463471
}
464472
}
465473
}
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.comet.exec
21+
22+
import org.apache.spark.SparkConf
23+
import org.apache.spark.sql.CometTestBase
24+
import org.apache.spark.sql.comet.CometInMemoryTableScanExec
25+
import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec
26+
import org.apache.spark.sql.execution.columnar.CometInMemoryRelationHelper
27+
import org.apache.spark.sql.execution.metric.SQLLastAttemptMetrics
28+
29+
import org.apache.comet.CometConf
30+
31+
/** Spark 4.2's last-attempt metrics over a relation cached in Comet's format. */
32+
class CometInMemoryCacheLastAttemptMetricSuite extends CometTestBase {
33+
34+
import testImplicits._
35+
36+
override protected def beforeAll(): Unit = {
37+
CometInMemoryRelationHelper.clearSerializer()
38+
super.beforeAll()
39+
}
40+
41+
override protected def afterAll(): Unit = {
42+
try {
43+
super.afterAll()
44+
} finally {
45+
CometInMemoryRelationHelper.clearSerializer()
46+
}
47+
}
48+
49+
override protected def sparkConf: SparkConf = super.sparkConf
50+
.set("spark.plugins", "org.apache.spark.CometPlugin")
51+
.set(
52+
"spark.sql.cache.serializer",
53+
"org.apache.spark.sql.comet.execution.arrow.ArrowCachedBatchSerializer")
54+
55+
test("a last-attempt metric outside the cache keeps its value") {
56+
// Spark finds the metric's stages by walking the plan and its subqueries, and gives up on a
57+
// shuffle it does not know. A Comet shuffle inside the cached plan must stay out of that walk.
58+
// https://github.com/apache/datafusion-comet/pull/6577#discussion_r4174502176
59+
withSQLConf(CometConf.COMET_EXEC_IN_MEMORY_CACHE_ENABLED.key -> "true") {
60+
val metric = SQLLastAttemptMetrics.createMetric(spark.sparkContext, "rows")
61+
val cached = spark.range(0, 100, 1, 2).repartition(2).cache()
62+
cached.count()
63+
val df = cached.map { id => metric.add(1); id }
64+
df.collect()
65+
val plan = df.queryExecution.executedPlan
66+
val scans = collect(plan) { case s: CometInMemoryTableScanExec => s }
67+
assert(scans.size == 1, plan)
68+
val cachedPlan = scans.head.originalPlan.relation.cachedPlan
69+
assert(collect(cachedPlan) { case s: CometShuffleExchangeExec => s }.nonEmpty, cachedPlan)
70+
assert(metric.lastAttemptValueForDataset(df) == Some(100L))
71+
}
72+
}
73+
}

0 commit comments

Comments
 (0)