@@ -220,6 +220,16 @@ case class CometExecRule(session: SparkSession)
220220
221221 private def isCometNative (op : SparkPlan ): Boolean = op.isInstanceOf [CometNativeExec ]
222222
223+ /**
224+ * Restore a Spark Partial while retaining its current children. The tag prevents reconversion
225+ * when AQE replans the exchange without its Final, and records why the Partial stays in Spark.
226+ */
227+ private def restoreSparkPartial (agg : CometHashAggregateExec , reason : String ): SparkPlan = {
228+ val partial = agg.originalPlan.withNewChildren(agg.children)
229+ partial.setTagValue(CometExecRule .COMET_UNSAFE_PARTIAL , reason)
230+ withFallbackReason(partial, reason)
231+ }
232+
223233 /**
224234 * A Celeborn exchange can fall back after its child has been converted, for example because of
225235 * the partition threshold or an unsupported hash key. Keep incompatible partial aggregate
@@ -237,10 +247,8 @@ case class CometExecRule(session: SparkSession)
237247 case _ : QueryStageExec | _ : ShuffleExchangeLike | _ : BroadcastExchangeLike => plan
238248 case agg : CometHashAggregateExec
239249 if agg.modes == Seq (Partial ) &&
240- ! QueryPlanSerde .allAggsSupportMixedExecution(agg.aggregateExpressions) =>
241- val sparkAggregate = agg.originalPlan.withNewChildren(agg.children)
242- sparkAggregate.setTagValue(CometExecRule .COMET_UNSAFE_PARTIAL , reason)
243- withFallbackReason(sparkAggregate, reason)
250+ ! QueryPlanSerde .allAggsSupportNativePartialToSparkFinal(agg.aggregateExpressions) =>
251+ restoreSparkPartial(agg, reason)
244252 // Final output is ordinary SQL data; any partial below it belongs to another aggregate.
245253 case agg : CometHashAggregateExec if agg.modes.contains(Final ) => agg
246254 case agg : BaseAggregateExec if agg.aggregateExpressions.exists(_.mode == Final ) => agg
@@ -758,7 +766,7 @@ case class CometExecRule(session: SparkSession)
758766 // during the bottom-up conversion. Tags persist through AQE stage creation.
759767 tagUnsafePartialAggregates(planWithJoinRewritten)
760768
761- var newPlan = transform(planWithJoinRewritten)
769+ var newPlan = revertUnsafePartialAggregates( transform(planWithJoinRewritten) )
762770
763771 // if the plan cannot be run fully natively then explain why (when appropriate
764772 // config is enabled)
@@ -1130,7 +1138,7 @@ case class CometExecRule(session: SparkSession)
11301138 val consumerMode : AggregateMode =
11311139 if (modes.contains(PartialMerge )) PartialMerge else Final
11321140 if (consumesBuffers &&
1133- ! QueryPlanSerde .allAggsSupportMixedExecution (agg.aggregateExpressions) &&
1141+ ! QueryPlanSerde .allAggsSupportNativePartialToSparkFinal (agg.aggregateExpressions) &&
11341142 ! canAggregateBeConverted(agg, consumerMode)) {
11351143 findPartialAggInPlan(agg.child).foreach { partial =>
11361144 // Only tag if the Partial would otherwise have been converted. If the Partial itself
@@ -1173,6 +1181,103 @@ case class CometExecRule(session: SparkSession)
11731181 }
11741182 }
11751183
1184+ /**
1185+ * Inspect a failed repair's buffer path without rewriting it or materializing any stage. Report
1186+ * only a native Partial/PartialMerge whose emitted state is not known to be Spark-compatible.
1187+ * Spark Partials and completed aggregates establish new buffers, so stop there rather than
1188+ * finding an unrelated native producer below them. Only known aggregate and exchange wrappers
1189+ * forward the same buffer path; an arbitrary operator is not evidence of a mixed boundary.
1190+ */
1191+ private def hasUnrepairedNativeBuffer (plan : SparkPlan ): Boolean = plan match {
1192+ case agg : CometHashAggregateExec if agg.aggregateExpressions.isEmpty =>
1193+ hasUnrepairedNativeBuffer(agg.child)
1194+ case agg : CometHashAggregateExec =>
1195+ agg.modes.forall(m => m == Partial || m == PartialMerge ) &&
1196+ ! QueryPlanSerde .allAggsSupportNativePartialToSparkFinal(agg.aggregateExpressions)
1197+ case agg : BaseAggregateExec
1198+ if agg.aggregateExpressions.nonEmpty &&
1199+ agg.aggregateExpressions.forall(_.mode == Partial ) =>
1200+ false
1201+ case agg : BaseAggregateExec =>
1202+ agg.aggregateExpressions.forall(e => e.mode == Partial || e.mode == PartialMerge ) &&
1203+ hasUnrepairedNativeBuffer(agg.child)
1204+ case placeholder : CometSinkPlaceHolder => hasUnrepairedNativeBuffer(placeholder.child)
1205+ case read : AQEShuffleReadExec => hasUnrepairedNativeBuffer(read.child)
1206+ case stage : ShuffleQueryStageExec => hasUnrepairedNativeBuffer(stage.plan)
1207+ case reused : ReusedExchangeExec => hasUnrepairedNativeBuffer(reused.child)
1208+ case shuffle : CometShuffleExchangeExec => hasUnrepairedNativeBuffer(shuffle.child)
1209+ case shuffle : ShuffleExchangeExec => hasUnrepairedNativeBuffer(shuffle.child)
1210+ case _ => false
1211+ }
1212+
1213+ /**
1214+ * The early tagging pass cannot know whether a Final's child will become native. Check the
1215+ * actual conversion result before serialization or AQE stage creation, restoring the feeding
1216+ * aggregate/exchange chain while keeping native work below its Partial. Return the repaired
1217+ * plan, or preserve an unrepairable path and record one warning on its Spark Final if an unsafe
1218+ * native producer remains. Existing stages and their buffers are never rewritten by this pass.
1219+ */
1220+ private [rules] def revertUnsafePartialAggregates (plan : SparkPlan ): SparkPlan = {
1221+ def revertChain (node : SparkPlan ): Option [SparkPlan ] = node match {
1222+ case agg : CometHashAggregateExec if agg.modes == Seq (Partial ) =>
1223+ Some (
1224+ restoreSparkPartial(
1225+ agg,
1226+ " Partial aggregate disabled: corresponding final aggregate " +
1227+ " cannot be converted to Comet and intermediate buffer formats are incompatible" ))
1228+
1229+ case agg : CometHashAggregateExec
1230+ if agg.modes.forall(m => m == Partial || m == PartialMerge ) =>
1231+ revertChain(agg.child).map(child => agg.originalPlan.withNewChildren(Seq (child)))
1232+
1233+ case agg : BaseAggregateExec
1234+ if agg.aggregateExpressions.nonEmpty &&
1235+ agg.aggregateExpressions.forall(_.mode == Partial ) =>
1236+ // This producer already emits Spark buffers. Do not reach through it to an unrelated
1237+ // aggregate below it.
1238+ None
1239+
1240+ case agg : BaseAggregateExec
1241+ if agg.aggregateExpressions.forall(e => e.mode == Partial || e.mode == PartialMerge ) =>
1242+ revertChain(agg.child).map(child => agg.withNewChildren(Seq (child)))
1243+
1244+ case CometSinkPlaceHolder (_, _, shuffle : CometShuffleExchangeExec ) =>
1245+ revertChain(shuffle)
1246+ case shuffle : CometShuffleExchangeExec =>
1247+ revertChain(shuffle.child).map(child => shuffle.originalPlan.withNewChildren(Seq (child)))
1248+ case shuffle : ShuffleExchangeExec =>
1249+ revertChain(shuffle.child).map(child => shuffle.withNewChildren(Seq (child)))
1250+
1251+ // Stop at materialized stages and operators outside the feeding aggregate/exchange chain.
1252+ case _ => None
1253+ }
1254+
1255+ plan.transformUp {
1256+ case agg : BaseAggregateExec
1257+ if agg.aggregateExpressions.map(_.mode).distinct == Seq (Final ) &&
1258+ ! QueryPlanSerde .allAggsSupportNativePartialToSparkFinal(agg.aggregateExpressions) =>
1259+ revertChain(agg.child)
1260+ // Rebuild native consumers and shuffles from their original Spark operators. Merely
1261+ // replacing their children would leave a native protobuf reading the old buffers.
1262+ .map(child => transform(agg.withNewChildren(Seq (child))))
1263+ .getOrElse {
1264+ if (hasUnrepairedNativeBuffer(agg.child)) {
1265+ val reason = " Comet could not restore a native intermediate buffer producer " +
1266+ " below Spark final aggregate; the remaining buffer may be incompatible"
1267+ // AQE can revisit the same consumer. Record the explanation and warn once,
1268+ // regardless of whether general fallback logging is enabled.
1269+ if (! agg
1270+ .getTagValue(CometExplainInfo .FALLBACK_REASONS )
1271+ .exists(_.contains(reason))) {
1272+ if (! CometConf .COMET_EXPLAIN_FALLBACK_LOG_ENABLED .get()) logWarning(reason)
1273+ withFallbackReason(agg, reason)
1274+ }
1275+ }
1276+ agg
1277+ }
1278+ }
1279+ }
1280+
11761281 /**
11771282 * Look for the bottom Partial-mode aggregate that feeds into the given plan (the child of a
11781283 * Final). Walks through exchanges and AQE stages, and continues down through intermediate
@@ -1202,8 +1307,8 @@ case class CometExecRule(session: SparkSession)
12021307 /**
12031308 * Conservative check for whether an aggregate could be converted to Comet. Checks operator
12041309 * enablement, grouping expressions, aggregate expressions, and result expressions.
1205- * Intentionally skips the sparkFinalMode / child-native checks since those depend on
1206- * transformation state .
1310+ * Intentionally skips the child-native checks since those depend on transformation state;
1311+ * [[ revertUnsafePartialAggregates ]] checks the actual conversion result before execution .
12071312 *
12081313 * WARNING: this intentionally mirrors the predicate checks in `CometBaseAggregate.doConvert`
12091314 * (operators.scala). Any change to the convertibility rules there must be reflected here or
0 commit comments