1919
2020package org .apache .comet .rules
2121
22+ import scala .collection .mutable
2223import scala .util .control .NonFatal
2324
2425import org .apache .spark .sql .SparkSession
2526import org .apache .spark .sql .catalyst .rules .Rule
2627import org .apache .spark .sql .catalyst .trees .TreeNodeTag
27- import org .apache .spark .sql .execution .{ApplyColumnarRulesAndInsertTransitions , BaseSubqueryExec , ColumnarToRowExec , ExecSubqueryExpression , InputAdapter , ReusedSubqueryExec , RowToColumnarExec , SparkPlan , SQLExecution , WholeStageCodegenExec }
28- import org .apache .spark .sql .execution .adaptive .{AdaptiveSparkPlanExec , QueryStageExec }
28+ import org .apache .spark .sql .execution .{ApplyColumnarRulesAndInsertTransitions , BaseSubqueryExec , ColumnarToRowExec , ExecSubqueryExpression , InputAdapter , QueryExecution , ReusedSubqueryExec , RowToColumnarExec , SparkPlan , WholeStageCodegenExec }
29+ import org .apache .spark .sql .execution .adaptive .{AdaptiveSparkPlanExec , InsertAdaptiveSparkPlan , QueryStageExec }
2930import org .apache .spark .sql .execution .exchange .{BroadcastExchangeExec , Exchange }
3031import org .apache .spark .sql .execution .reuse .ReuseExchangeAndSubquery
3132
@@ -42,33 +43,44 @@ object CometRule {
4243 EliminateRedundantTransitions (session))
4344
4445 /**
45- * `executionId:canonicalPlanHash` keys already reported, LRU-bounded for long-lived drivers.
46+ * Canonical hashes of the subquery plans reported for the query this thread is preparing. Spark
47+ * prepares a query's subqueries synchronously, before the query itself, so the scope is reset
48+ * when the query's own plan arrives. This works whether or not a SQL execution ID is set.
4649 */
47- private val planOnlyReportedPlans : java.util.Set [String ] =
48- java.util.Collections .newSetFromMap(
49- java.util.Collections .synchronizedMap(
50- new java.util.LinkedHashMap [String , java.lang.Boolean ](16 , 0.75f , true ) {
51- override def removeEldestEntry (
52- eldest : java.util.Map .Entry [String , java.lang.Boolean ]): Boolean = size() > 1024
53- }))
50+ private val reportedSubqueries = new ThreadLocal [mutable.Set [Int ]] {
51+ override def initialValue (): mutable.Set [Int ] = mutable.Set .empty
52+ }
5453
5554 /**
5655 * Marks the root of a reported plan. Catalyst copies tags onto replacement nodes, so later
5756 * applications of the rule can recognize the plan however Spark rewrote it in between.
5857 */
5958 private val PLAN_ONLY_REPORTED : TreeNodeTag [Unit ] = TreeNodeTag [Unit ](" comet.planOnlyReported" )
6059
60+ /** Where in Spark's planning the rule is running, read off the call stack. */
61+ private case class PlanningContext (replanning : Boolean , subquery : Boolean )
62+
63+ private def planningContext (): PlanningContext = {
64+ val frames = Thread .currentThread().getStackTrace
65+ def within (cls : Class [_], method : String ): Boolean =
66+ frames.exists(f => f.getMethodName == method && f.getClassName == cls.getName)
67+ PlanningContext (
68+ replanning = within(classOf [AdaptiveSparkPlanExec ], " reOptimize" ),
69+ // Subqueries are prepared by `PlanSubqueries` and `PlanDynamicPruningFilters` without AQE,
70+ // and by `InsertAdaptiveSparkPlan` and `PlanAdaptiveDynamicPruningFilters` with it.
71+ subquery = within(QueryExecution .getClass, " prepareExecutedPlan" ) ||
72+ within(classOf [InsertAdaptiveSparkPlan ], " compileSubquery" ))
73+ }
74+
6175 /**
6276 * Whether plan-only mode should report `plan`, marking it reported if so.
6377 *
6478 * Under AQE the rule sees one query several times: the initial plan (prep rule, the one to
6579 * report), the same plan wrapped in `AdaptiveSparkPlanExec`, each query stage, each
6680 * re-optimization and the final plan. Only the first is reported. Each scalar or DPP subquery
67- * is prepared as a top-level plan of its own and gets its own report. None of this can rely on
68- * a SQL execution ID, since `df.rdd.count()` and `executedPlan` plan without one.
81+ * is prepared as a plan of its own and gets its own report.
6982 */
7083 private [comet] def shouldReportPlanOnly (
71- executionId : Option [String ],
7284 plan : SparkPlan ,
7385 queryStagePrep : Boolean ,
7486 aqeEnabled : Boolean ): Boolean = {
@@ -80,19 +92,25 @@ object CometRule {
8092 p.isInstanceOf [QueryStageExec ] || p.isInstanceOf [AdaptiveSparkPlanExec ] ||
8193 p.getTagValue(PLAN_ONLY_REPORTED ).isDefined)
8294 if (isQueryStage || isReapplication) {
95+ // Execution is under way, so any subquery prepared from here on belongs to a new scope.
96+ reportedSubqueries.get().clear()
8397 false
8498 } else {
85- // Mark even when not reporting, so the final plan built from an empty re-plan is recognized.
99+ // Mark even when not reporting, so the final plan built from a re-plan is recognized.
86100 plan.setTagValue(PLAN_ONLY_REPORTED , ())
87- // AQE re-plans to an empty relation when a stage materializes empty, sharing no nodes or
88- // stages with the reported plan. Only the prep rule sees re-plans, so a genuinely empty
89- // query still reaches the columnar rule and is reported.
90- val replannedToNothing =
91- aqeEnabled && queryStagePrep && plan.logicalLink.exists(_.maxRows.contains(0L ))
92- // A subquery referenced twice is prepared twice, differing only in expression IDs, so dedupe
93- // on the canonical plan within an execution. Tags are not part of the canonical form.
94- ! replannedToNothing &&
95- executionId.forall(id => planOnlyReportedPlans.add(s " $id: ${plan.canonicalized.hashCode()}" ))
101+ val context = planningContext()
102+ if (context.replanning) {
103+ // AQE re-plans a query mid-execution, for example to an empty relation once a stage
104+ // materializes empty, and the result can share no nodes or stages with the reported plan.
105+ false
106+ } else if (context.subquery) {
107+ // A subquery referenced twice is prepared twice, differing only in expression IDs.
108+ // Tags are not part of the canonical form.
109+ reportedSubqueries.get().add(plan.canonicalized.hashCode())
110+ } else {
111+ reportedSubqueries.get().clear()
112+ true
113+ }
96114 }
97115 }
98116}
@@ -142,16 +160,10 @@ case class CometRule(session: SparkSession, queryStagePrep: Boolean = false)
142160 /** Logs the Comet plan for `plan` unless already reported. Never fails the query. */
143161 private def reportPlanOnlyCoverage (plan : SparkPlan ): Unit = {
144162 try {
145- val executionId = Option (
146- session.sparkContext.getLocalProperty(SQLExecution .EXECUTION_ID_KEY ))
147- if (CometRule .shouldReportPlanOnly(
148- executionId,
149- plan,
150- queryStagePrep,
151- conf.adaptiveExecutionEnabled)) {
163+ if (CometRule .shouldReportPlanOnly(plan, queryStagePrep, conf.adaptiveExecutionEnabled)) {
152164 val preview = buildPreview(plan, topLevel = true )
153165 logWarning(
154- s " [Comet plan-only] \n ${new ExtendedExplainInfo ().generateExtendedInfo (preview)}" )
166+ s " [Comet plan-only] \n ${new ExtendedExplainInfo ().generateVerboseInfo (preview)}" )
155167 }
156168 } catch {
157169 case NonFatal (e) =>
0 commit comments