@@ -1262,8 +1262,9 @@ case class CometExecRule(session: SparkSession)
12621262 * Inspect a failed repair's buffer path without rewriting it or materializing any stage. Report
12631263 * only a native Partial/PartialMerge whose emitted state is not known to be Spark-compatible.
12641264 * Spark Partials and completed aggregates establish new buffers, so stop there rather than
1265- * finding an unrelated native producer below them. Only known aggregate and exchange wrappers
1266- * forward the same buffer path; an arbitrary operator is not evidence of a mixed boundary.
1265+ * finding an unrelated native producer below them. Only known aggregate, sort and exchange
1266+ * wrappers forward the same buffer path; an arbitrary operator is not evidence of a mixed
1267+ * boundary.
12671268 */
12681269 private def hasUnrepairedNativeBuffer (plan : SparkPlan ): Boolean = plan match {
12691270 case agg : CometBaseAggregateExec if agg.aggregateExpressions.isEmpty =>
@@ -1279,6 +1280,9 @@ case class CometExecRule(session: SparkSession)
12791280 agg.aggregateExpressions.forall(e => e.mode == Partial || e.mode == PartialMerge ) &&
12801281 hasUnrepairedNativeBuffer(agg.child)
12811282 case placeholder : CometSinkPlaceHolder => hasUnrepairedNativeBuffer(placeholder.child)
1283+ // A sort aggregate reads its buffers through the sort that orders them.
1284+ case sort : SortExec => hasUnrepairedNativeBuffer(sort.child)
1285+ case sort : CometSortExec => hasUnrepairedNativeBuffer(sort.child)
12821286 case read : AQEShuffleReadExec => hasUnrepairedNativeBuffer(read.child)
12831287 case stage : ShuffleQueryStageExec => hasUnrepairedNativeBuffer(stage.plan)
12841288 case reused : ReusedExchangeExec => hasUnrepairedNativeBuffer(reused.child)
@@ -1290,9 +1294,10 @@ case class CometExecRule(session: SparkSession)
12901294 /**
12911295 * The early tagging pass cannot know whether a Final's child will become native. Check the
12921296 * actual conversion result before serialization or AQE stage creation, restoring the feeding
1293- * aggregate/exchange chain while keeping native work below its Partial. Return the repaired
1294- * plan, or preserve an unrepairable path and record one warning on its Spark Final if an unsafe
1295- * native producer remains. Existing stages and their buffers are never rewritten by this pass.
1297+ * aggregate/sort/exchange chain while keeping native work below its Partial. Return the
1298+ * repaired plan, or preserve an unrepairable path and record one warning on its Spark Final if
1299+ * an unsafe native producer remains. Existing stages and their buffers are never rewritten by
1300+ * this pass.
12961301 */
12971302 private [rules] def revertUnsafePartialAggregates (plan : SparkPlan ): SparkPlan = {
12981303 def revertChain (node : SparkPlan ): Option [SparkPlan ] = node match {
@@ -1318,6 +1323,12 @@ case class CometExecRule(session: SparkSession)
13181323 if agg.aggregateExpressions.forall(e => e.mode == Partial || e.mode == PartialMerge ) =>
13191324 revertChain(agg.child).map(child => agg.withNewChildren(Seq (child)))
13201325
1326+ // A sort aggregate reads its buffers through the sort that orders them.
1327+ case sort : CometSortExec =>
1328+ revertChain(sort.child).map(child => sort.originalPlan.withNewChildren(Seq (child)))
1329+ case sort : SortExec =>
1330+ revertChain(sort.child).map(child => sort.withNewChildren(Seq (child)))
1331+
13211332 case CometSinkPlaceHolder (_, _, shuffle : CometShuffleExchangeExec ) =>
13221333 revertChain(shuffle)
13231334 case shuffle : CometShuffleExchangeExec =>
@@ -1357,11 +1368,12 @@ case class CometExecRule(session: SparkSession)
13571368
13581369 /**
13591370 * Look for the bottom Partial-mode aggregate that feeds into the given plan (the child of a
1360- * Final). Walks through exchanges and AQE stages, and continues down through intermediate
1361- * aggregate stages whose modes are all Partial / PartialMerge - these are the PartialMerge (and
1362- * mixed Partial/PartialMerge) stages that Spark's distinct-aggregate rewrite inserts between
1363- * the Partial and the Final. Stops at anything else. Requires `aggregateExpressions.nonEmpty`
1364- * so that group-by-only dedup stages are traversed rather than mistaken for the partial.
1371+ * Final). Walks through exchanges, AQE stages and the sorts that Spark inserts below sort
1372+ * aggregates, and continues down through intermediate aggregate stages whose modes are all
1373+ * Partial / PartialMerge - these are the PartialMerge (and mixed Partial/PartialMerge) stages
1374+ * that Spark's distinct-aggregate rewrite inserts between the Partial and the Final. Stops at
1375+ * anything else. Requires `aggregateExpressions.nonEmpty` so that group-by-only dedup stages
1376+ * are traversed rather than mistaken for the partial.
13651377 */
13661378 private def findPartialAggInPlan (plan : SparkPlan ): Option [BaseAggregateExec ] = plan match {
13671379 case agg : BaseAggregateExec
@@ -1376,6 +1388,7 @@ case class CometExecRule(session: SparkSession)
13761388 case a : AQEShuffleReadExec => findPartialAggInPlan(a.child)
13771389 case s : ShuffleQueryStageExec => findPartialAggInPlan(s.plan)
13781390 case e : ShuffleExchangeExec => findPartialAggInPlan(e.child)
1391+ case s : SortExec => findPartialAggInPlan(s.child)
13791392 case other =>
13801393 logDebug(s " findPartialAggInPlan: stopping at ${other.nodeName}; not a known passthrough " )
13811394 None
0 commit comments