@@ -206,6 +206,18 @@ case class CometExecRule(session: SparkSession)
206206
207207 private def isCometNative (op : SparkPlan ): Boolean = op.isInstanceOf [CometNativeExec ]
208208
209+ /**
210+ * Restore a native Partial's original Spark operator with its current children. Both repair
211+ * paths use this to preserve native work below the Partial, prevent reconversion, and record
212+ * the explanation immediately, even if a non-native child prevents another serde attempt. The
213+ * caller establishes buffer incompatibility; this does not traverse or rewrite children.
214+ */
215+ private def restoreSparkPartial (agg : CometHashAggregateExec , reason : String ): SparkPlan = {
216+ val partial = agg.originalPlan.withNewChildren(agg.children)
217+ partial.setTagValue(CometExecRule .COMET_UNSAFE_PARTIAL , reason)
218+ withFallbackReason(partial, reason)
219+ }
220+
209221 /**
210222 * A Celeborn exchange can fall back after its child has been converted, for example because of
211223 * the partition threshold or an unsupported hash key. Keep incompatible partial aggregate
@@ -224,9 +236,7 @@ case class CometExecRule(session: SparkSession)
224236 case agg : CometHashAggregateExec
225237 if agg.modes == Seq (Partial ) &&
226238 ! QueryPlanSerde .allAggsSupportNativePartialToSparkFinal(agg.aggregateExpressions) =>
227- val sparkAggregate = agg.originalPlan.withNewChildren(agg.children)
228- sparkAggregate.setTagValue(CometExecRule .COMET_UNSAFE_PARTIAL , reason)
229- withFallbackReason(sparkAggregate, reason)
239+ restoreSparkPartial(agg, reason)
230240 // Final output is ordinary SQL data; any partial below it belongs to another aggregate.
231241 case agg : CometHashAggregateExec if agg.modes.contains(Final ) => agg
232242 case agg : BaseAggregateExec if agg.aggregateExpressions.exists(_.mode == Final ) => agg
@@ -1133,20 +1143,51 @@ case class CometExecRule(session: SparkSession)
11331143 }
11341144 }
11351145
1146+ /**
1147+ * Inspect an unrepaired buffer path without modifying it, including already-created stages.
1148+ * Unlike a descendant search, stop at Spark producers and completed aggregates so an unrelated
1149+ * inner native Partial cannot trigger a warning. Merge stages still consume intermediate
1150+ * buffers. Unknown unary wrappers are inspected only when they preserve all output attributes;
1151+ * changed-output and branching nodes are not assumed to forward the same buffers.
1152+ */
1153+ private def hasUnrepairedNativePartial (plan : SparkPlan ): Boolean = plan match {
1154+ case agg : CometHashAggregateExec if agg.aggregateExpressions.isEmpty =>
1155+ hasUnrepairedNativePartial(agg.child)
1156+ case agg : CometHashAggregateExec =>
1157+ agg.modes.forall(m => m == Partial || m == PartialMerge ) &&
1158+ ! QueryPlanSerde .allAggsSupportNativePartialToSparkFinal(agg.aggregateExpressions)
1159+ case agg : BaseAggregateExec
1160+ if agg.aggregateExpressions.nonEmpty &&
1161+ agg.aggregateExpressions.forall(_.mode == Partial ) =>
1162+ false
1163+ case agg : BaseAggregateExec =>
1164+ agg.aggregateExpressions.forall(e => e.mode == Partial || e.mode == PartialMerge ) &&
1165+ hasUnrepairedNativePartial(agg.child)
1166+ case stage : QueryStageExec => hasUnrepairedNativePartial(stage.plan)
1167+ case reused : ReusedExchangeExec => hasUnrepairedNativePartial(reused.child)
1168+ case read : AQEShuffleReadExec => hasUnrepairedNativePartial(read.child)
1169+ case placeholder : CometSinkPlaceHolder => hasUnrepairedNativePartial(placeholder.child)
1170+ case shuffle : CometShuffleExchangeExec => hasUnrepairedNativePartial(shuffle.child)
1171+ case other if other.children.size == 1 && other.output == other.children.head.output =>
1172+ hasUnrepairedNativePartial(other.children.head)
1173+ case _ => false
1174+ }
1175+
11361176 /**
11371177 * The early tagging pass cannot know whether a Final's child will become native. Check the
11381178 * actual conversion result as well, before native blocks are serialized or AQE launches stages.
1139- * Restore only the feeding aggregate/exchange chain; keep native work below its Partial.
1179+ * Restore only the feeding aggregate/exchange chain; keep native work below its Partial. If a
1180+ * remaining native buffer producer cannot be restored, warn and annotate the Spark Final; do
1181+ * not rewrite materialized stages or assume unknown operators can safely be reconstructed.
11401182 */
1141- private def revertUnsafePartialAggregates (plan : SparkPlan ): SparkPlan = {
1183+ private [rules] def revertUnsafePartialAggregates (plan : SparkPlan ): SparkPlan = {
11421184 def revertChain (node : SparkPlan ): Option [SparkPlan ] = node match {
11431185 case agg : CometHashAggregateExec if agg.modes == Seq (Partial ) =>
1144- val partial = agg.originalPlan.withNewChildren(Seq (agg.child))
1145- partial.setTagValue(
1146- CometExecRule .COMET_UNSAFE_PARTIAL ,
1147- " Partial aggregate disabled: corresponding final aggregate " +
1148- " cannot be converted to Comet and intermediate buffer formats are incompatible" )
1149- Some (partial)
1186+ Some (
1187+ restoreSparkPartial(
1188+ agg,
1189+ " Partial aggregate disabled: corresponding final aggregate " +
1190+ " cannot be converted to Comet and intermediate buffer formats are incompatible" ))
11501191
11511192 case agg : CometHashAggregateExec
11521193 if agg.modes.forall(m => m == Partial || m == PartialMerge ) =>
@@ -1186,7 +1227,25 @@ case class CometExecRule(session: SparkSession)
11861227 // Rebuild native consumers and shuffles from their original Spark operators. Merely
11871228 // replacing their children would leave a native protobuf reading the old buffers.
11881229 .map(child => transform(agg.withNewChildren(Seq (child))))
1189- .getOrElse(agg)
1230+ .getOrElse {
1231+ if (hasUnrepairedNativePartial(agg.child)) {
1232+ val reason = " Comet could not restore a native intermediate buffer producer " +
1233+ s " below Spark final aggregate ( ${agg.child.nodeName}); " +
1234+ " the remaining aggregate boundary may have incompatible buffer formats"
1235+ // AQE may reapply this rule to the same consumer. Keep the explanation, but do
1236+ // not repeat a warning that has already been attached to this plan node.
1237+ if (! agg
1238+ .getTagValue(CometExplainInfo .FALLBACK_REASONS )
1239+ .exists(_.contains(reason))) {
1240+ if (! CometConf .COMET_EXPLAIN_FALLBACK_LOG_ENABLED .get()) logWarning(reason)
1241+ withFallbackReason(agg, reason)
1242+ } else {
1243+ agg
1244+ }
1245+ } else {
1246+ agg
1247+ }
1248+ }
11901249 }
11911250 }
11921251
0 commit comments