Skip to content

Commit 3123088

Browse files
committed
fix: scope plan-only report dedupe to the query and detect AQE re-plans
Render plan-only reports in the verbose format regardless of spark.comet.explain.format, so the coverage summary is never dropped. Tell AQE re-planning apart from initial preparation by whether the rule is running inside AdaptiveSparkPlanExec.reOptimize, instead of guessing from maxRows. An initially empty query is now reported under AQE, and a re-plan that keeps operators above an empty join is no longer reported twice. Deduplicate repeated subqueries within the query being prepared on the current thread rather than within a SQL execution ID, so queries planned without one (toRdd, df.rdd, executedPlan) no longer repeat reports.
1 parent a80ae3f commit 3123088

3 files changed

Lines changed: 101 additions & 54 deletions

File tree

‎spark/src/main/scala/org/apache/comet/ExtendedExplainInfo.scala‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,18 +42,24 @@ class ExtendedExplainInfo extends ExtendedExplainGenerator {
4242
def generateExtendedInfo(plan: SparkPlan): String = {
4343
CometConf.COMET_EXTENDED_EXPLAIN_FORMAT.get() match {
4444
case CometConf.COMET_EXTENDED_EXPLAIN_FORMAT_VERBOSE =>
45-
// Generates the extended info in a verbose manner, printing each node along with the
46-
// extended information in a tree display.
47-
val planStats = new CometCoverageStats()
48-
val outString = new StringBuilder()
49-
generateTreeString(getActualPlan(plan), 0, Seq(), 0, outString, planStats)
50-
s"${outString.toString()}\n$planStats"
45+
generateVerboseInfo(plan)
5146
case CometConf.COMET_EXTENDED_EXPLAIN_FORMAT_FALLBACK =>
5247
// Generates the extended info as a list of fallback reasons
5348
getFallbackReasons(plan).mkString("\n").trim
5449
}
5550
}
5651

52+
/**
53+
* The `verbose` format regardless of `spark.comet.explain.format`: each node along with its
54+
* extended information in a tree display, followed by the coverage summary.
55+
*/
56+
def generateVerboseInfo(plan: SparkPlan): String = {
57+
val planStats = new CometCoverageStats()
58+
val outString = new StringBuilder()
59+
generateTreeString(getActualPlan(plan), 0, Seq(), 0, outString, planStats)
60+
s"${outString.toString()}\n$planStats"
61+
}
62+
5763
def getFallbackReasons(plan: SparkPlan): Seq[String] = {
5864
fallbackReasons(plan).toSeq.sorted
5965
}

‎spark/src/main/scala/org/apache/comet/rules/CometRule.scala‎

Lines changed: 43 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,14 @@
1919

2020
package org.apache.comet.rules
2121

22+
import scala.collection.mutable
2223
import scala.util.control.NonFatal
2324

2425
import org.apache.spark.sql.SparkSession
2526
import org.apache.spark.sql.catalyst.rules.Rule
2627
import org.apache.spark.sql.catalyst.trees.TreeNodeTag
27-
import org.apache.spark.sql.execution.{ApplyColumnarRulesAndInsertTransitions, BaseSubqueryExec, ColumnarToRowExec, ExecSubqueryExpression, InputAdapter, ReusedSubqueryExec, RowToColumnarExec, SparkPlan, SQLExecution, WholeStageCodegenExec}
28-
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec}
28+
import org.apache.spark.sql.execution.{ApplyColumnarRulesAndInsertTransitions, BaseSubqueryExec, ColumnarToRowExec, ExecSubqueryExpression, InputAdapter, QueryExecution, ReusedSubqueryExec, RowToColumnarExec, SparkPlan, WholeStageCodegenExec}
29+
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, InsertAdaptiveSparkPlan, QueryStageExec}
2930
import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange}
3031
import org.apache.spark.sql.execution.reuse.ReuseExchangeAndSubquery
3132

@@ -42,33 +43,44 @@ object CometRule {
4243
EliminateRedundantTransitions(session))
4344

4445
/**
45-
* `executionId:canonicalPlanHash` keys already reported, LRU-bounded for long-lived drivers.
46+
* Canonical hashes of the subquery plans reported for the query this thread is preparing. Spark
47+
* prepares a query's subqueries synchronously, before the query itself, so the scope is reset
48+
* when the query's own plan arrives. This works whether or not a SQL execution ID is set.
4649
*/
47-
private val planOnlyReportedPlans: java.util.Set[String] =
48-
java.util.Collections.newSetFromMap(
49-
java.util.Collections.synchronizedMap(
50-
new java.util.LinkedHashMap[String, java.lang.Boolean](16, 0.75f, true) {
51-
override def removeEldestEntry(
52-
eldest: java.util.Map.Entry[String, java.lang.Boolean]): Boolean = size() > 1024
53-
}))
50+
private val reportedSubqueries = new ThreadLocal[mutable.Set[Int]] {
51+
override def initialValue(): mutable.Set[Int] = mutable.Set.empty
52+
}
5453

5554
/**
5655
* Marks the root of a reported plan. Catalyst copies tags onto replacement nodes, so later
5756
* applications of the rule can recognize the plan however Spark rewrote it in between.
5857
*/
5958
private val PLAN_ONLY_REPORTED: TreeNodeTag[Unit] = TreeNodeTag[Unit]("comet.planOnlyReported")
6059

60+
/** Where in Spark's planning the rule is running, read off the call stack. */
61+
private case class PlanningContext(replanning: Boolean, subquery: Boolean)
62+
63+
private def planningContext(): PlanningContext = {
64+
val frames = Thread.currentThread().getStackTrace
65+
def within(cls: Class[_], method: String): Boolean =
66+
frames.exists(f => f.getMethodName == method && f.getClassName == cls.getName)
67+
PlanningContext(
68+
replanning = within(classOf[AdaptiveSparkPlanExec], "reOptimize"),
69+
// Subqueries are prepared by `PlanSubqueries` and `PlanDynamicPruningFilters` without AQE,
70+
// and by `InsertAdaptiveSparkPlan` and `PlanAdaptiveDynamicPruningFilters` with it.
71+
subquery = within(QueryExecution.getClass, "prepareExecutedPlan") ||
72+
within(classOf[InsertAdaptiveSparkPlan], "compileSubquery"))
73+
}
74+
6175
/**
6276
* Whether plan-only mode should report `plan`, marking it reported if so.
6377
*
6478
* Under AQE the rule sees one query several times: the initial plan (prep rule, the one to
6579
* report), the same plan wrapped in `AdaptiveSparkPlanExec`, each query stage, each
6680
* re-optimization and the final plan. Only the first is reported. Each scalar or DPP subquery
67-
* is prepared as a top-level plan of its own and gets its own report. None of this can rely on
68-
* a SQL execution ID, since `df.rdd.count()` and `executedPlan` plan without one.
81+
* is prepared as a plan of its own and gets its own report.
6982
*/
7083
private[comet] def shouldReportPlanOnly(
71-
executionId: Option[String],
7284
plan: SparkPlan,
7385
queryStagePrep: Boolean,
7486
aqeEnabled: Boolean): Boolean = {
@@ -80,19 +92,25 @@ object CometRule {
8092
p.isInstanceOf[QueryStageExec] || p.isInstanceOf[AdaptiveSparkPlanExec] ||
8193
p.getTagValue(PLAN_ONLY_REPORTED).isDefined)
8294
if (isQueryStage || isReapplication) {
95+
// Execution is under way, so any subquery prepared from here on belongs to a new scope.
96+
reportedSubqueries.get().clear()
8397
false
8498
} else {
85-
// Mark even when not reporting, so the final plan built from an empty re-plan is recognized.
99+
// Mark even when not reporting, so the final plan built from a re-plan is recognized.
86100
plan.setTagValue(PLAN_ONLY_REPORTED, ())
87-
// AQE re-plans to an empty relation when a stage materializes empty, sharing no nodes or
88-
// stages with the reported plan. Only the prep rule sees re-plans, so a genuinely empty
89-
// query still reaches the columnar rule and is reported.
90-
val replannedToNothing =
91-
aqeEnabled && queryStagePrep && plan.logicalLink.exists(_.maxRows.contains(0L))
92-
// A subquery referenced twice is prepared twice, differing only in expression IDs, so dedupe
93-
// on the canonical plan within an execution. Tags are not part of the canonical form.
94-
!replannedToNothing &&
95-
executionId.forall(id => planOnlyReportedPlans.add(s"$id:${plan.canonicalized.hashCode()}"))
101+
val context = planningContext()
102+
if (context.replanning) {
103+
// AQE re-plans a query mid-execution, for example to an empty relation once a stage
104+
// materializes empty, and the result can share no nodes or stages with the reported plan.
105+
false
106+
} else if (context.subquery) {
107+
// A subquery referenced twice is prepared twice, differing only in expression IDs.
108+
// Tags are not part of the canonical form.
109+
reportedSubqueries.get().add(plan.canonicalized.hashCode())
110+
} else {
111+
reportedSubqueries.get().clear()
112+
true
113+
}
96114
}
97115
}
98116
}
@@ -142,16 +160,10 @@ case class CometRule(session: SparkSession, queryStagePrep: Boolean = false)
142160
/** Logs the Comet plan for `plan` unless already reported. Never fails the query. */
143161
private def reportPlanOnlyCoverage(plan: SparkPlan): Unit = {
144162
try {
145-
val executionId = Option(
146-
session.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY))
147-
if (CometRule.shouldReportPlanOnly(
148-
executionId,
149-
plan,
150-
queryStagePrep,
151-
conf.adaptiveExecutionEnabled)) {
163+
if (CometRule.shouldReportPlanOnly(plan, queryStagePrep, conf.adaptiveExecutionEnabled)) {
152164
val preview = buildPreview(plan, topLevel = true)
153165
logWarning(
154-
s"[Comet plan-only]\n${new ExtendedExplainInfo().generateExtendedInfo(preview)}")
166+
s"[Comet plan-only]\n${new ExtendedExplainInfo().generateVerboseInfo(preview)}")
155167
}
156168
} catch {
157169
case NonFatal(e) =>

‎spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala‎

Lines changed: 46 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1288,7 +1288,9 @@ class CometExecRuleSuite extends CometTestBase {
12881288
"collect" -> (df => df.collect()),
12891289
"toRdd.count" -> (df => df.queryExecution.toRdd.count()))
12901290
// In the second shape `RemoveRedundantSorts` drops the root sort, so the mark lands on the
1291-
// join and the empty re-plan's root inherits the unmarked sort's tags.
1291+
// join and the empty re-plan's root inherits the unmarked sort's tags. In the third the
1292+
// global aggregate survives the empty join, so the re-plan is not itself empty. The fourth
1293+
// is empty from the start, which must not be mistaken for a re-plan.
12921294
(shape, query, marker) <- Seq(
12931295
(
12941296
"aggregate",
@@ -1300,7 +1302,14 @@ class CometExecRuleSuite extends CometTestBase {
13001302
|JOIN range(0, 20, 1, 2) b ON a.id % 7 = b.id % 7
13011303
|WHERE a.id < 0
13021304
|SORT BY a.id % 7""".stripMargin,
1303-
"SortMergeJoin"))
1305+
"SortMergeJoin"),
1306+
(
1307+
"global aggregate over a join",
1308+
"""SELECT count(*) FROM range(0, 20, 1, 2) a
1309+
|JOIN range(0, 20, 1, 2) b ON a.id % 7 = b.id % 7
1310+
|WHERE a.id < 0""".stripMargin,
1311+
"SortMergeJoin"),
1312+
("initially empty", "SELECT id FROM range(0) DISTRIBUTE BY id", "Exchange"))
13041313
} {
13051314
test(s"plan-only mode: one report when the plan becomes empty ($shape, AQE=$aqe, $action)") {
13061315
withSQLConf(
@@ -1319,22 +1328,42 @@ class CometExecRuleSuite extends CometTestBase {
13191328
}
13201329
}
13211330

1322-
test("plan-only mode: a subquery referenced twice is reported once") {
1323-
withPlanOnlyTable(aqe = false) {
1324-
val reports = capturePlanOnlyReports {
1325-
spark
1326-
.sql("""SELECT _1,
1327-
| (SELECT max(_2) FROM tbl
1328-
| WHERE _1 > (SELECT min(_2) FROM tbl)) AS a,
1329-
| (SELECT max(_2) FROM tbl
1330-
| WHERE _1 > (SELECT min(_2) FROM tbl)) AS b
1331-
|FROM tbl""".stripMargin)
1332-
.collect()
1331+
for {
1332+
aqe <- Seq(true, false)
1333+
(action, runIt) <- Seq[(String, org.apache.spark.sql.DataFrame => Unit)](
1334+
"collect" -> (df => df.collect()),
1335+
"toRdd.count" -> (df => df.queryExecution.toRdd.count()))
1336+
} {
1337+
test(s"plan-only mode: a subquery referenced twice is reported once (AQE=$aqe, $action)") {
1338+
withPlanOnlyTable(aqe = aqe) {
1339+
val reports = capturePlanOnlyReports {
1340+
runIt(spark.sql("""SELECT _1,
1341+
| (SELECT max(_2) FROM tbl
1342+
| WHERE _1 > (SELECT min(_2) FROM tbl)) AS a,
1343+
| (SELECT max(_2) FROM tbl
1344+
| WHERE _1 > (SELECT min(_2) FROM tbl)) AS b
1345+
|FROM tbl""".stripMargin))
1346+
}
1347+
// The `min` subquery, the `max` subquery and the outer query.
1348+
assert(
1349+
reports.size == 3,
1350+
s"expected three reports, got ${reports.size}:\n${reports.mkString("\n\n")}")
1351+
}
1352+
}
1353+
}
1354+
1355+
test("plan-only mode: the report does not depend on spark.comet.explain.format") {
1356+
withSQLConf(
1357+
CometConf.COMET_EXTENDED_EXPLAIN_FORMAT.key ->
1358+
CometConf.COMET_EXTENDED_EXPLAIN_FORMAT_FALLBACK) {
1359+
withPlanOnlyTable() {
1360+
val reports = capturePlanOnlyReports(sql("SELECT _1 + 1 FROM tbl").collect())
1361+
assert(reports.size == 1, s"expected one report, got:\n${reports.mkString("\n\n")}")
1362+
val (accelerated, eligible) = coverageOf(reports.head)
1363+
assert(
1364+
accelerated > 0 && accelerated == eligible,
1365+
s"unexpected coverage:\n${reports.head}")
13331366
}
1334-
// The `min` subquery, the `max` subquery and the outer query.
1335-
assert(
1336-
reports.size == 3,
1337-
s"expected three reports, got ${reports.size}:\n${reports.mkString("\n\n")}")
13381367
}
13391368
}
13401369

0 commit comments

Comments
 (0)