@@ -32,7 +32,7 @@ import org.apache.spark.sql.comet._
3232import org .apache .spark .sql .comet .execution .shuffle .{CometColumnarShuffle , CometNativeShuffle , CometShuffleExchangeExec }
3333import org .apache .spark .sql .comet .util .Utils
3434import org .apache .spark .sql .execution ._
35- import org .apache .spark .sql .execution .adaptive .{AdaptiveSparkPlanExec , AQEShuffleReadExec , BroadcastQueryStageExec , ShuffleQueryStageExec }
35+ import org .apache .spark .sql .execution .adaptive .{AdaptiveSparkPlanExec , AQEShuffleReadExec , BroadcastQueryStageExec , QueryStageExec , ShuffleQueryStageExec }
3636import org .apache .spark .sql .execution .aggregate .{BaseAggregateExec , HashAggregateExec , ObjectHashAggregateExec }
3737import org .apache .spark .sql .execution .command .{DataWritingCommandExec , ExecutedCommandExec }
3838import org .apache .spark .sql .execution .datasources .WriteFilesExec
@@ -117,32 +117,89 @@ object CometExecRule {
117117 org.apache.spark.sql.catalyst.trees.TreeNodeTag [Unit ](" comet.skipCometBroadcast" )
118118
119119 /**
120- * SQL execution IDs for which plan-only mode has already logged its report, as a bounded LRU.
121- * Dedupes the WARN under AQE, where `CometExecRule` fires once against the full initial plan
122- * and again per stage. Same synchronized-`LinkedHashMap` LRU pattern used by
123- * `IcebergPlanDataInjector.commonCache`.
120+ * A bounded set of keys, used for plan-only reporting state. Evicts in LRU order once `limit`
121+ * keys are held, so a long-lived driver retains a fixed amount of reporting state. Same
122+ * synchronized-`LinkedHashMap` pattern used by `IcebergPlanDataInjector.commonCache`.
124123 */
124+ private class BoundedKeySet (limit : Int ) {
125+ private val keys : java.util.Map [String , java.lang.Boolean ] =
126+ java.util.Collections .synchronizedMap(
127+ new java.util.LinkedHashMap [String , java.lang.Boolean ](16 , 0.75f , true ) {
128+ override def removeEldestEntry (
129+ eldest : java.util.Map .Entry [String , java.lang.Boolean ]): Boolean = size() > limit
130+ })
131+
132+ /** Adds `key`, returning true if it was not already present. */
133+ def add (key : String ): Boolean = keys.put(key, java.lang.Boolean .TRUE ) == null
134+
135+ def contains (key : String ): Boolean = keys.containsKey(key)
136+ }
137+
125138 private val PLAN_ONLY_REPORTED_LIMIT = 1024
126- private val planOnlyReportedIds : java.util.Map [String , java.lang.Boolean ] =
127- java.util.Collections .synchronizedMap(
128- new java.util.LinkedHashMap [String , java.lang.Boolean ](16 , 0.75f , false ) {
129- override def removeEldestEntry (
130- eldest : java.util.Map .Entry [String , java.lang.Boolean ]): Boolean =
131- size() > PLAN_ONLY_REPORTED_LIMIT
132- })
133-
134- private [comet] def markPlanOnlyReported (executionId : Option [String ]): Boolean = {
139+
140+ /** `executionId:planFingerprint` keys that plan-only mode has already reported. */
141+ private val planOnlyReportedPlans = new BoundedKeySet (PLAN_ONLY_REPORTED_LIMIT )
142+
143+ /** Execution IDs whose plan-only report came from the query-stage-prep rule. */
144+ private val planOnlyPrepReportedIds = new BoundedKeySet (PLAN_ONLY_REPORTED_LIMIT )
145+
146+ /**
147+ * Whether plan-only mode should report `plan`, recording that it did so.
148+ *
149+ * Spark applies this rule many times during one SQL execution, and only some of those
150+ * applications correspond to a plan the user is asking about:
151+ *
152+ * - Each scalar subquery and DPP subquery is prepared as its own top-level plan, and that
153+ * happens *before* the outer plan reaches the conversion rules. Keying the report on the
154+ * execution ID alone therefore let a nested subquery consume the slot and suppressed the
155+ * outer plan, which is the plan being evaluated. Keying on the execution ID *and* the plan
156+ * gives the outer plan its own report and each separately prepared subquery theirs.
157+ * - Under AQE the rule also runs once per query stage (as a columnar rule) and again on every
158+ * re-optimization (as a query-stage-prep rule). Those are re-planning of a plan already
159+ * reported, so a `plan` containing query stages is skipped, and once the query-stage-prep
160+ * rule has reported an execution the columnar applications for it stay quiet.
161+ *
162+ * @param queryStagePrep
163+ * whether the calling rule instance is registered as a query-stage-prep rule.
164+ */
165+ private [comet] def shouldReportPlanOnly (
166+ executionId : Option [String ],
167+ plan : SparkPlan ,
168+ queryStagePrep : Boolean ): Boolean = {
135169 executionId match {
136- case None => true
137- case Some (id) => planOnlyReportedIds.put(id, java.lang.Boolean .TRUE ) == null
170+ case None =>
171+ // No execution ID means the plan is being built outside an action (`df.explain`, or
172+ // reading `queryExecution.executedPlan` directly). AQE creates no stages in that case, so
173+ // there is nothing to dedupe against.
174+ true
175+ case Some (id) =>
176+ if (plan.exists(_.isInstanceOf [QueryStageExec ])) {
177+ // An AQE stage plan or a re-optimized plan: a re-plan of what we already reported.
178+ false
179+ } else {
180+ if (queryStagePrep) {
181+ planOnlyPrepReportedIds.add(id)
182+ } else if (planOnlyPrepReportedIds.contains(id)) {
183+ return false
184+ }
185+ // The plan's structural hash identifies it: node tags are not part of it, so the same
186+ // plan applied twice keys the same and is reported once.
187+ planOnlyReportedPlans.add(s " $id: ${plan.hashCode()}" )
188+ }
138189 }
139190 }
140191}
141192
142193/**
143194 * Spark physical optimizer rule for replacing Spark operators with Comet operators.
195+ *
196+ * @param queryStagePrep
197+ * true for the instance registered with `injectQueryStagePrepRule`, which under AQE sees the
198+ * whole initial plan, and false for the one registered as a columnar rule, which under AQE sees
199+ * one query stage at a time. Only plan-only reporting reads this; see
200+ * [[CometExecRule.shouldReportPlanOnly ]].
144201 */
145- case class CometExecRule (session : SparkSession )
202+ case class CometExecRule (session : SparkSession , queryStagePrep : Boolean = false )
146203 extends Rule [SparkPlan ]
147204 with ShimSubqueryBroadcast {
148205
@@ -599,9 +656,23 @@ case class CometExecRule(session: SparkSession)
599656 * Build the Comet plan we would have executed and log it. Called from `_apply` in plan-only
600657 * mode; the built plan is discarded. Passes `forPreview = true` through the nested calls so
601658 * both rules run their normal transforms instead of short-circuiting.
659+ *
660+ * Conversion is only the first half of Comet planning. Normally Spark then inserts the columnar
661+ * transitions and runs Comet's post-columnar rules (see
662+ * `CometSparkSessionExtensions.CometExecColumnar.postColumnarTransitions`), which can revert
663+ * whole stages back to Spark and drop redundant transitions. Those steps run here too, so the
664+ * report describes the plan that would really have executed and counts the transitions that
665+ * would really have been there, rather than the pre-transition conversion result.
666+ *
667+ * `RevertNativeForTransitionHeavyStages` is applied with `applyToAllStages` because the preview
668+ * holds the whole plan at once, whereas under AQE Spark hands that rule one stage at a time.
602669 */
603670 private def reportPlanOnlyCoverage (plan : SparkPlan ): Unit = {
604- val preview = _apply(CometScanRule (session)._apply(plan), forPreview = true )
671+ val converted = _apply(CometScanRule (session)._apply(plan), forPreview = true )
672+ val withTransitions =
673+ ApplyColumnarRulesAndInsertTransitions (Seq .empty, outputsColumnar = false ).apply(converted)
674+ val reverted = RevertNativeForTransitionHeavyStages (session).applyToAllStages(withTransitions)
675+ val preview = EliminateRedundantTransitions (session).apply(reverted)
605676 logWarning(s " [Comet plan-only] \n ${new ExtendedExplainInfo ().generateExtendedInfo(preview)}" )
606677 }
607678
@@ -629,7 +700,7 @@ case class CometExecRule(session: SparkSession)
629700 if (! forPreview && CometConf .COMET_EXPLAIN_PLAN_ONLY_ENABLED .get()) {
630701 val executionId = Option (
631702 session.sparkContext.getLocalProperty(SQLExecution .EXECUTION_ID_KEY ))
632- if (CometExecRule .markPlanOnlyReported (executionId)) {
703+ if (CometExecRule .shouldReportPlanOnly (executionId, plan, queryStagePrep )) {
633704 reportPlanOnlyCoverage(plan)
634705 }
635706 return plan
0 commit comments