Skip to content

Commit 8c93fbb

Browse files
authored
fix: draw the cached plan below CometInMemoryTableScan in EXPLAIN (apache#6574)
* fix: draw the cached plan below CometInMemoryTableScan in EXPLAIN Spark's InMemoryTableScanExec lists its InMemoryRelation, and through it the cached plan, as inner children, so EXPLAIN draws the plan that built the cache below the scan. Do the same for CometInMemoryTableScanExec. ExtendedExplainInfo walks inner children, so leave the scan's out of Comet's coverage and fallback reporting, as it already does for CometEmptyRelationExec. The cached plan runs when the relation is materialized, not as part of every query that reads it. Closes apache#6572. * Simplify the cached plan EXPLAIN test and its comment
1 parent 569eaa5 commit 8c93fbb

3 files changed

Lines changed: 77 additions & 3 deletions

File tree

‎spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import scala.collection.mutable
2626
import org.apache.spark.sql.ExtendedExplainGenerator
2727
import org.apache.spark.sql.catalyst.expressions.{Attribute, BoundReference, Expression, Literal, ScalaUDF}
2828
import org.apache.spark.sql.catalyst.trees.{TreeNode, TreeNodeTag}
29-
import org.apache.spark.sql.comet.{CometColumnarToRowExec, CometEmptyRelationExec, CometNativeColumnarToRowExec, CometPlan, CometSparkToColumnarExec}
29+
import org.apache.spark.sql.comet.{CometColumnarToRowExec, CometEmptyRelationExec, CometInMemoryTableScanExec, CometNativeColumnarToRowExec, CometPlan, CometSparkToColumnarExec}
3030
import org.apache.spark.sql.execution.{ColumnarToRowExec, InputAdapter, ReusedSubqueryExec, RowToColumnarExec, SparkPlan, WholeStageCodegenExec}
3131
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AQEShuffleReadExec, QueryStageExec}
3232
import org.apache.spark.sql.execution.exchange.{Exchange, ReusedExchangeExec}
@@ -109,6 +109,10 @@ class ExtendedExplainInfo extends ExtendedExplainGenerator {
109109
// Spark's treeString displays the eliminated plan, but it does not execute and must not
110110
// contribute Spark operators, fallback reasons, or expressions to Comet's reporting.
111111
case _: CometEmptyRelationExec => Seq.empty
112+
// Spark's treeString displays the cached relation and the plan that built it. That plan runs
113+
// when the relation is materialized, not as part of every query that reads it, so it must not
114+
// count toward the coverage or the fallback reasons of each of those queries.
115+
case _: CometInMemoryTableScanExec => Seq.empty
112116
case _ => node.innerChildren
113117
}
114118

‎spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import scala.collection.JavaConverters._
2323

2424
import org.apache.spark.rdd.RDD
2525
import org.apache.spark.sql.catalyst.expressions.Attribute
26+
import org.apache.spark.sql.catalyst.plans.QueryPlan
2627
import org.apache.spark.sql.catalyst.plans.logical.Statistics
2728
import org.apache.spark.sql.columnar.{CachedBatch, CachedBatchSerializer}
2829
import org.apache.spark.sql.comet.shims.ShimCometInMemoryTableScanExec
@@ -80,6 +81,11 @@ case class CometInMemoryTableScanExec(
8081
// newlines, which breaks the tree of every plan that reads the cache.
8182
override def stringArgs: Iterator[Any] = Iterator(originalPlan)
8283

84+
// Spark's own scan lists its InMemoryRelation as an inner child, and the relation lists the
85+
// cached plan, so EXPLAIN draws the plan that built the cache below the scan. Do the same.
86+
// ExtendedExplainInfo.executionInnerChildren leaves them out of Comet's own reporting.
87+
override def innerChildren: Seq[QueryPlan[_]] = Seq(originalPlan.relation)
88+
8389
// `originalPlan` is a plan-typed field rather than a child, so QueryPlan's canonicalization
8490
// walks straight past it: its attributes and predicates keep the expression IDs of whichever
8591
// occurrence of the cached relation produced them. Two scans of one cache then compare unequal,

‎spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala‎

Lines changed: 66 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ import org.apache.spark.sql.columnar.{CachedBatch, SimpleMetricsCachedBatch}
4040
import org.apache.spark.sql.comet.{CometBroadcastHashJoinExec, CometInMemoryTableScanExec, CometSortExec, CometSortMergeJoinExec}
4141
import org.apache.spark.sql.comet.execution.arrow.{ArrowCachedBatchSerializer, CometCachedBatchHelper}
4242
import org.apache.spark.sql.comet.util.Utils
43-
import org.apache.spark.sql.execution.SortExec
43+
import org.apache.spark.sql.execution.{FormattedMode, SortExec}
4444
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AQEShuffleReadExec, QueryStageExec, ShuffleQueryStageExec}
4545
import org.apache.spark.sql.execution.columnar.{CometInMemoryRelationHelper, InMemoryRelation, InMemoryTableScanExec}
4646
import org.apache.spark.sql.execution.exchange.{Exchange, ReusedExchangeExec, ShuffleExchangeLike}
@@ -352,9 +352,13 @@ class CometInMemoryCacheSuite extends CometTestBase {
352352
line.startsWith("CometInMemoryTableScan Scan In-memory table explain_cache ["),
353353
line)
354354
assert(line.contains("= 3)"), s"the pruning predicates should be shown: $line")
355+
// EXPLAIN FORMATTED also details the InMemoryRelation drawn below the scan. Before Spark
356+
// 4.0 that detail prints the relation's CachedRDDBuilder below Spark's own scan as well
357+
// (SPARK-51861), so there only the scan's own detail is checked.
355358
Seq(
356359
plan.treeString,
357-
df.queryExecution.explainString(org.apache.spark.sql.execution.FormattedMode))
360+
if (isSpark40Plus) df.queryExecution.explainString(FormattedMode)
361+
else scans.head.verboseStringWithOperatorId())
358362
.foreach { text =>
359363
assert(!text.contains("CachedRDDBuilder"), text)
360364
assert(!text.contains(classOf[ArrowCachedBatchSerializer].getName), text)
@@ -365,6 +369,66 @@ class CometInMemoryCacheSuite extends CometTestBase {
365369
}
366370
}
367371

372+
test("EXPLAIN draws the cached plan below CometInMemoryTableScan") {
373+
// Spark's own cache scan draws its InMemoryRelation, and below that the plan that built the
374+
// relation. See https://github.com/apache/datafusion-comet/issues/6572.
375+
def nodeLines(tree: String): Seq[String] =
376+
tree.linesIterator.map(_.replaceAll("^[ :+-]*", "")).toSeq
377+
378+
Seq("false", "true").foreach { aqe =>
379+
withClue(s"AQE $aqe: ") {
380+
withSQLConf(
381+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> aqe,
382+
CometConf.COMET_EXEC_IN_MEMORY_CACHE_ENABLED.key -> "true") {
383+
withTempView("explain_cached_plan") {
384+
spark
385+
.range(1000)
386+
.selectExpr("id AS key", "id % 8 AS value")
387+
.createOrReplaceTempView("explain_cached_plan")
388+
// The cached plan is planned here. Keeping its Range on Spark gives it a fallback
389+
// reason that the reporting for a query reading the cache must leave out.
390+
withSQLConf(
391+
CometConf.COMET_EXEC_RANGE_ENABLED.key -> "false",
392+
CometConf.COMET_SPARK_TO_ARROW_ENABLED.key -> "false") {
393+
spark.catalog.cacheTable("explain_cached_plan")
394+
}
395+
val df = spark.sql("SELECT value, count(*) FROM explain_cached_plan GROUP BY value")
396+
df.collect()
397+
val plan = df.queryExecution.executedPlan
398+
val scans = collect(plan) { case s: CometInMemoryTableScanExec => s }
399+
assert(scans.size == 1, plan)
400+
val relation = scans.head.originalPlan.relation
401+
402+
// The relation's line, then the cached plan's lines, follow the scan's line.
403+
val lines = nodeLines(plan.treeString)
404+
val below = nodeLines(relation.treeString)
405+
assert(below.size > 1, relation)
406+
val scanAt = lines.indexWhere(_.startsWith("CometInMemoryTableScan "))
407+
assert(lines.startsWith(below, scanAt + 1), plan)
408+
409+
// EXPLAIN FORMATTED numbers the relation and draws it below the scan too.
410+
val formatted = df.queryExecution.explainString(FormattedMode)
411+
val formattedLines = nodeLines(formatted)
412+
val formattedScanAt =
413+
formattedLines.indexWhere(_.startsWith("CometInMemoryTableScan ("))
414+
assert(
415+
formattedLines(formattedScanAt + 1).startsWith("InMemoryRelation ("),
416+
formatted)
417+
418+
// Comet's own reporting leaves the cached plan out.
419+
val info = new ExtendedExplainInfo()
420+
val cachedReasons = info.getFallbackReasons(relation.cachedPlan)
421+
assert(cachedReasons.nonEmpty, relation.cachedPlan)
422+
val reasons = info.getFallbackReasons(plan)
423+
assert(cachedReasons.intersect(reasons).isEmpty, reasons)
424+
val verbose = info.generateVerboseInfo(plan)
425+
assert(!verbose.contains("InMemoryRelation"), verbose)
426+
}
427+
}
428+
}
429+
}
430+
}
431+
368432
test("Comet in-memory cache disabled keeps SparkToColumnar fallback path") {
369433
withSQLConf(
370434
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",

0 commit comments

Comments
 (0)