@@ -40,7 +40,7 @@ import org.apache.spark.sql.columnar.{CachedBatch, SimpleMetricsCachedBatch}
4040import org .apache .spark .sql .comet .{CometBroadcastHashJoinExec , CometInMemoryTableScanExec , CometSortExec , CometSortMergeJoinExec }
4141import org .apache .spark .sql .comet .execution .arrow .{ArrowCachedBatchSerializer , CometCachedBatchHelper }
4242import org .apache .spark .sql .comet .util .Utils
43- import org .apache .spark .sql .execution .SortExec
43+ import org .apache .spark .sql .execution .{ FormattedMode , SortExec }
4444import org .apache .spark .sql .execution .adaptive .{AdaptiveSparkPlanExec , AQEShuffleReadExec , QueryStageExec , ShuffleQueryStageExec }
4545import org .apache .spark .sql .execution .columnar .{CometInMemoryRelationHelper , InMemoryRelation , InMemoryTableScanExec }
4646import org .apache .spark .sql .execution .exchange .{Exchange , ReusedExchangeExec , ShuffleExchangeLike }
@@ -352,9 +352,13 @@ class CometInMemoryCacheSuite extends CometTestBase {
352352 line.startsWith(" CometInMemoryTableScan Scan In-memory table explain_cache [" ),
353353 line)
354354 assert(line.contains(" = 3)" ), s " the pruning predicates should be shown: $line" )
355+ // EXPLAIN FORMATTED also details the InMemoryRelation drawn below the scan. Before Spark
356+ // 4.0 that detail prints the relation's CachedRDDBuilder below Spark's own scan as well
357+ // (SPARK-51861), so there only the scan's own detail is checked.
355358 Seq (
356359 plan.treeString,
357- df.queryExecution.explainString(org.apache.spark.sql.execution.FormattedMode ))
360+ if (isSpark40Plus) df.queryExecution.explainString(FormattedMode )
361+ else scans.head.verboseStringWithOperatorId())
358362 .foreach { text =>
359363 assert(! text.contains(" CachedRDDBuilder" ), text)
360364 assert(! text.contains(classOf [ArrowCachedBatchSerializer ].getName), text)
@@ -365,6 +369,66 @@ class CometInMemoryCacheSuite extends CometTestBase {
365369 }
366370 }
367371
372+ test(" EXPLAIN draws the cached plan below CometInMemoryTableScan" ) {
373+ // Spark's own cache scan draws its InMemoryRelation, and below that the plan that built the
374+ // relation. See https://github.com/apache/datafusion-comet/issues/6572.
375+ def nodeLines (tree : String ): Seq [String ] =
376+ tree.linesIterator.map(_.replaceAll(" ^[ :+-]*" , " " )).toSeq
377+
378+ Seq (" false" , " true" ).foreach { aqe =>
379+ withClue(s " AQE $aqe: " ) {
380+ withSQLConf(
381+ SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> aqe,
382+ CometConf .COMET_EXEC_IN_MEMORY_CACHE_ENABLED .key -> " true" ) {
383+ withTempView(" explain_cached_plan" ) {
384+ spark
385+ .range(1000 )
386+ .selectExpr(" id AS key" , " id % 8 AS value" )
387+ .createOrReplaceTempView(" explain_cached_plan" )
388+ // The cached plan is planned here. Keeping its Range on Spark gives it a fallback
389+ // reason that the reporting for a query reading the cache must leave out.
390+ withSQLConf(
391+ CometConf .COMET_EXEC_RANGE_ENABLED .key -> " false" ,
392+ CometConf .COMET_SPARK_TO_ARROW_ENABLED .key -> " false" ) {
393+ spark.catalog.cacheTable(" explain_cached_plan" )
394+ }
395+ val df = spark.sql(" SELECT value, count(*) FROM explain_cached_plan GROUP BY value" )
396+ df.collect()
397+ val plan = df.queryExecution.executedPlan
398+ val scans = collect(plan) { case s : CometInMemoryTableScanExec => s }
399+ assert(scans.size == 1 , plan)
400+ val relation = scans.head.originalPlan.relation
401+
402+ // The relation's line, then the cached plan's lines, follow the scan's line.
403+ val lines = nodeLines(plan.treeString)
404+ val below = nodeLines(relation.treeString)
405+ assert(below.size > 1 , relation)
406+ val scanAt = lines.indexWhere(_.startsWith(" CometInMemoryTableScan " ))
407+ assert(lines.startsWith(below, scanAt + 1 ), plan)
408+
409+ // EXPLAIN FORMATTED numbers the relation and draws it below the scan too.
410+ val formatted = df.queryExecution.explainString(FormattedMode )
411+ val formattedLines = nodeLines(formatted)
412+ val formattedScanAt =
413+ formattedLines.indexWhere(_.startsWith(" CometInMemoryTableScan (" ))
414+ assert(
415+ formattedLines(formattedScanAt + 1 ).startsWith(" InMemoryRelation (" ),
416+ formatted)
417+
418+ // Comet's own reporting leaves the cached plan out.
419+ val info = new ExtendedExplainInfo ()
420+ val cachedReasons = info.getFallbackReasons(relation.cachedPlan)
421+ assert(cachedReasons.nonEmpty, relation.cachedPlan)
422+ val reasons = info.getFallbackReasons(plan)
423+ assert(cachedReasons.intersect(reasons).isEmpty, reasons)
424+ val verbose = info.generateVerboseInfo(plan)
425+ assert(! verbose.contains(" InMemoryRelation" ), verbose)
426+ }
427+ }
428+ }
429+ }
430+ }
431+
368432 test(" Comet in-memory cache disabled keeps SparkToColumnar fallback path" ) {
369433 withSQLConf(
370434 SQLConf .ADAPTIVE_EXECUTION_ENABLED .key -> " false" ,
0 commit comments