@@ -43,8 +43,9 @@ import org.apache.spark.sql.execution.datasources.v2.{BatchScanExec, V2CommandEx
4343import org .apache .spark .sql .execution .datasources .v2 .csv .CSVScan
4444import org .apache .spark .sql .execution .datasources .v2 .json .JsonScan
4545import org .apache .spark .sql .execution .datasources .v2 .parquet .ParquetScan
46- import org .apache .spark .sql .execution .exchange .{BroadcastExchangeExec , ReusedExchangeExec , ShuffleExchangeExec }
46+ import org .apache .spark .sql .execution .exchange .{BroadcastExchangeExec , Exchange , ReusedExchangeExec , ShuffleExchangeExec }
4747import org .apache .spark .sql .execution .joins .{BroadcastHashJoinExec , BroadcastNestedLoopJoinExec , ShuffledHashJoinExec , SortMergeJoinExec }
48+ import org .apache .spark .sql .execution .reuse .ReuseExchangeAndSubquery
4849import org .apache .spark .sql .execution .window .WindowExec
4950import org .apache .spark .sql .internal .SQLConf
5051import org .apache .spark .sql .types ._
@@ -131,61 +132,94 @@ object CometExecRule {
131132
132133 /** Adds `key`, returning true if it was not already present. */
133134 def add (key : String ): Boolean = keys.put(key, java.lang.Boolean .TRUE ) == null
134-
135- def contains (key : String ): Boolean = keys.containsKey(key)
136135 }
137136
138137 private val PLAN_ONLY_REPORTED_LIMIT = 1024
139138
140139 /** `executionId:planFingerprint` keys that plan-only mode has already reported. */
141140 private val planOnlyReportedPlans = new BoundedKeySet (PLAN_ONLY_REPORTED_LIMIT )
142141
143- /** Execution IDs whose plan-only report came from the query-stage-prep rule. */
144- private val planOnlyPrepReportedIds = new BoundedKeySet (PLAN_ONLY_REPORTED_LIMIT )
142+ /**
143+ * Set on the root of every plan that plan-only mode has reported. Catalyst copies a node's tags
144+ * onto the node that replaces it (`TreeNode.copyTagsFrom`), so the mark survives the rewrites
145+ * Spark applies between one application of this rule and the next, which is what lets a later
146+ * application recognize a plan it has already described.
147+ */
148+ private val PLAN_ONLY_REPORTED : TreeNodeTag [Unit ] = TreeNodeTag [Unit ](" comet.planOnlyReported" )
149+
150+ /**
151+ * Whether `plan` is an application of this rule to a plan already reported, rather than to a
152+ * plan the user is asking about. All the state lives on the plan itself, so this holds whether
153+ * or not the application carries a SQL execution ID - `df.rdd.count()` and reading
154+ * `executedPlan` without an action both plan, and in the first case execute AQE stages, with no
155+ * execution ID set.
156+ *
157+ * Under AQE one query reaches the conversion rules five times, and only the first is the plan
158+ * being evaluated:
159+ *
160+ * - the initial plan, through the query-stage-prep rule - the one to report;
161+ * - the same plan again as a columnar rule, now wrapped in `AdaptiveSparkPlanExec`;
162+ * - each query stage as it is created, again as a columnar rule, rooted at the `Exchange` the
163+ * stage was cut at;
164+ * - the re-optimized plan after each stage materializes, through the prep rule;
165+ * - the final plan once every stage has materialized, as a columnar rule.
166+ *
167+ * `AdaptiveSparkPlanExec` is a leaf as far as `exists` is concerned, so the wrapper is matched
168+ * by name rather than through its contents. The two re-optimization passes hold
169+ * `QueryStageExec` nodes for the stages already materialized. The remaining two - a stage, and
170+ * a final plan for a query AQE never had to cut into stages - carry the mark left by the prep
171+ * rule.
172+ *
173+ * @param queryStagePrep
174+ * whether the calling rule instance is registered as a query-stage-prep rule.
175+ * @param aqeEnabled
176+ * whether AQE is enabled for the session. A plan rooted at an `Exchange` is a stage only when
177+ * AQE cuts stages; with AQE off it is an ordinary plan (`df.repartition(n)`, say) and must
178+ * still be reported.
179+ */
180+ private def isReapplication (
181+ plan : SparkPlan ,
182+ queryStagePrep : Boolean ,
183+ aqeEnabled : Boolean ): Boolean = {
184+ plan.exists(p =>
185+ p.isInstanceOf [QueryStageExec ] || p.isInstanceOf [AdaptiveSparkPlanExec ] ||
186+ p.getTagValue(PLAN_ONLY_REPORTED ).isDefined) ||
187+ (aqeEnabled && ! queryStagePrep && plan.isInstanceOf [Exchange ])
188+ }
145189
146190 /**
147- * Whether plan-only mode should report `plan`, recording that it did so.
191+ * Whether plan-only mode should report `plan`, marking it reported if so.
148192 *
149- * Spark applies this rule many times during one SQL execution , and only some of those
193+ * Spark applies this rule many times while executing one query , and only some of those
150194 * applications correspond to a plan the user is asking about:
151195 *
152196 * - Each scalar subquery and DPP subquery is prepared as its own top-level plan, and that
153- * happens *before* the outer plan reaches the conversion rules. Keying the report on the
154- * execution ID alone therefore let a nested subquery consume the slot and suppressed the
155- * outer plan, which is the plan being evaluated. Keying on the execution ID *and* the plan
156- * gives the outer plan its own report and each separately prepared subquery theirs.
157- * - Under AQE the rule also runs once per query stage (as a columnar rule) and again on every
158- * re-optimization (as a query-stage-prep rule). Those are re-planning of a plan already
159- * reported, so a `plan` containing query stages is skipped, and once the query-stage-prep
160- * rule has reported an execution the columnar applications for it stay quiet.
197+ * happens *before* the outer plan reaches the conversion rules. A single report slot per
198+ * SQL execution therefore let a nested subquery consume the slot and suppressed the outer
199+ * plan, which is the plan being evaluated. Marking plans individually gives the outer plan
200+ * its own report and each separately prepared subquery theirs.
201+ * - Under AQE the rule also runs per query stage and again after every re-optimization. Those
202+ * are re-planning of a plan already reported; see [[isReapplication ]].
203+ * - A subquery referenced from more than one place in the outer plan is prepared once per
204+ * reference, as a separate but identical plan each time. The mark cannot catch those - they
205+ * share no nodes - so within one SQL execution the plan's structural hash dedupes them.
161206 *
162207 * @param queryStagePrep
163208 * whether the calling rule instance is registered as a query-stage-prep rule.
164209 */
165210 private [comet] def shouldReportPlanOnly (
166211 executionId : Option [String ],
167212 plan : SparkPlan ,
168- queryStagePrep : Boolean ): Boolean = {
169- executionId match {
170- case None =>
171- // No execution ID means the plan is being built outside an action (`df.explain`, or
172- // reading `queryExecution.executedPlan` directly). AQE creates no stages in that case, so
173- // there is nothing to dedupe against.
174- true
175- case Some (id) =>
176- if (plan.exists(_.isInstanceOf [QueryStageExec ])) {
177- // An AQE stage plan or a re-optimized plan: a re-plan of what we already reported.
178- false
179- } else {
180- if (queryStagePrep) {
181- planOnlyPrepReportedIds.add(id)
182- } else if (planOnlyPrepReportedIds.contains(id)) {
183- return false
184- }
185- // The plan's structural hash identifies it: node tags are not part of it, so the same
186- // plan applied twice keys the same and is reported once.
187- planOnlyReportedPlans.add(s " $id: ${plan.hashCode()}" )
188- }
213+ queryStagePrep : Boolean ,
214+ aqeEnabled : Boolean ): Boolean = {
215+ if (isReapplication(plan, queryStagePrep, aqeEnabled)) {
216+ false
217+ } else {
218+ plan.setTagValue(PLAN_ONLY_REPORTED , ())
219+ // Node tags are not part of a plan's structural hash, so the mark set above does not
220+ // perturb the key. Without an execution ID there is nothing to scope the state to, and
221+ // `isReapplication` has already ruled out the repeat applications AQE makes, so report.
222+ executionId.forall(id => planOnlyReportedPlans.add(s " $id: ${plan.hashCode()}" ))
189223 }
190224 }
191225}
@@ -654,8 +688,16 @@ case class CometExecRule(session: SparkSession, queryStagePrep: Boolean = false)
654688
655689 /**
656690 * Build the Comet plan we would have executed and log it. Called from `_apply` in plan-only
657- * mode; the built plan is discarded. Passes `forPreview = true` through the nested calls so
658- * both rules run their normal transforms instead of short-circuiting.
691+ * mode; the built plan is discarded.
692+ */
693+ private def reportPlanOnlyCoverage (plan : SparkPlan ): Unit = {
694+ val preview = buildPreview(plan, topLevel = true )
695+ logWarning(s " [Comet plan-only] \n ${new ExtendedExplainInfo ().generateExtendedInfo(preview)}" )
696+ }
697+
698+ /**
699+ * The plan Comet would have executed for `plan`. Passes `forPreview = true` through the nested
700+ * calls so both conversion rules run their normal transforms instead of short-circuiting.
659701 *
660702 * Conversion is only the first half of Comet planning. Normally Spark then inserts the columnar
661703 * transitions and runs Comet's post-columnar rules (see
@@ -666,14 +708,68 @@ case class CometExecRule(session: SparkSession, queryStagePrep: Boolean = false)
666708 *
667709 * `RevertNativeForTransitionHeavyStages` is applied with `applyToAllStages` because the preview
668710 * holds the whole plan at once, whereas under AQE Spark hands that rule one stage at a time.
711+ *
712+ * @param topLevel
713+ * false when previewing the plan behind a subquery expression. `ReuseExchangeAndSubquery` is
714+ * the last step of Spark's preparation and `QueryExecution.preparations` omits it for a
715+ * subquery, so the preview follows suit.
669716 */
670- private def reportPlanOnlyCoverage (plan : SparkPlan ): Unit = {
671- val converted = _apply(CometScanRule (session)._apply(plan), forPreview = true )
717+ private def buildPreview (plan : SparkPlan , topLevel : Boolean ): SparkPlan = {
718+ val converted =
719+ _apply(CometScanRule (session)._apply(previewSubqueriesOf(plan)), forPreview = true )
672720 val withTransitions =
673721 ApplyColumnarRulesAndInsertTransitions (Seq .empty, outputsColumnar = false ).apply(converted)
674722 val reverted = RevertNativeForTransitionHeavyStages (session).applyToAllStages(withTransitions)
675723 val preview = EliminateRedundantTransitions (session).apply(reverted)
676- logWarning(s " [Comet plan-only] \n ${new ExtendedExplainInfo ().generateExtendedInfo(preview)}" )
724+ if (topLevel) ReuseExchangeAndSubquery .apply(preview) else preview
725+ }
726+
727+ /**
728+ * `plan` with the plan behind each of its subquery expressions replaced by that plan's own
729+ * preview.
730+ *
731+ * Extended explain walks a node's `innerChildren`, which for a `SparkPlan` are the plans owned
732+ * by its expressions, and counts their operators towards the report. Normal planning has
733+ * already converted those plans by the time the outer plan reaches this rule - Spark prepares a
734+ * scalar subquery through the full preparation sequence, columnar rules included, before
735+ * substituting it into the outer plan - so leaving them untouched here would report every
736+ * subquery operator as un-accelerated Spark and understate coverage relative to what Comet
737+ * really executes.
738+ *
739+ * Each subquery is also reported in its own right, because Spark prepares it as a top-level
740+ * plan of its own; those reports and the counts here therefore describe overlapping sets of
741+ * operators.
742+ */
743+ private def previewSubqueriesOf (plan : SparkPlan ): SparkPlan = {
744+ plan.transformAllExpressions { case subquery : ExecSubqueryExpression =>
745+ subquery.withNewPlan(previewSubquery(subquery.plan))
746+ }
747+ }
748+
749+ private def previewSubquery (subquery : BaseSubqueryExec ): BaseSubqueryExec = subquery match {
750+ // Reuse bookkeeping: the plan to preview is one level further down.
751+ case reused : ReusedSubqueryExec => reused.copy(child = previewSubquery(reused.child))
752+ case other =>
753+ val preview = buildPreview(stripPreparation(other.child), topLevel = false )
754+ other.withNewChildren(Seq (preview)).asInstanceOf [BaseSubqueryExec ]
755+ }
756+
757+ /**
758+ * `plan` with the artifacts of a finished plan preparation removed: whole-stage codegen
759+ * wrappers and the columnar transitions Spark inserted.
760+ *
761+ * A subquery arrives inside the outer plan fully prepared -
762+ * `ApplyColumnarRulesAndInsertTransitions` and `CollapseCodegenStages` have both run over it -
763+ * whereas the conversion rules only ever see a plan midway through preparation. A
764+ * `HashAggregateExec` still wrapped in `WholeStageCodegenExec` is left unconverted, so
765+ * previewing the prepared form would report a subquery as falling back that Comet in fact
766+ * accelerates. [[buildPreview ]] re-inserts the transitions once conversion is done.
767+ */
768+ private def stripPreparation (plan : SparkPlan ): SparkPlan = plan.transformUp {
769+ case WholeStageCodegenExec (child) => child
770+ case InputAdapter (child) => child
771+ case ColumnarToRowExec (child) => child
772+ case RowToColumnarExec (child) => child
677773 }
678774
679775 private def _apply (plan : SparkPlan , forPreview : Boolean ): SparkPlan = {
@@ -700,7 +796,11 @@ case class CometExecRule(session: SparkSession, queryStagePrep: Boolean = false)
700796 if (! forPreview && CometConf .COMET_EXPLAIN_PLAN_ONLY_ENABLED .get()) {
701797 val executionId = Option (
702798 session.sparkContext.getLocalProperty(SQLExecution .EXECUTION_ID_KEY ))
703- if (CometExecRule .shouldReportPlanOnly(executionId, plan, queryStagePrep)) {
799+ if (CometExecRule .shouldReportPlanOnly(
800+ executionId,
801+ plan,
802+ queryStagePrep,
803+ conf.adaptiveExecutionEnabled)) {
704804 reportPlanOnlyCoverage(plan)
705805 }
706806 return plan
0 commit comments