Skip to content

Commit 6065705

Browse files
authored
fix: revert unsafe partial aggregates after final fallback (#5421)
* fix: revert unsafe partial aggregates after final fallback * fix: distinguish aggregate buffer compatibility by direction * fix: block unsafe native AVG partials before Spark final * test: cover aggregate fallback for unsupported array hash keys * test: align aggregate fallback checks across Spark versions Capture the Spark baseline inside withSQLConf so Spark 3 does not return Unit in place of the expected rows. Expect AVG to remain in Spark after Celeborn shuffle fallback, consistent with its unsafe empty partial buffer, and retain COUNT as the compatible native-partial control. * fix: clarify aggregate buffer compatibility and fallback * ci: group aggregate policy tests with aggregate suite * test: cover DISTINCT aggregate fallback boundaries Cover nondecimal AVG with SUM(DISTINCT) across global and grouped plans, shuffle modes, and selective lower-exchange fallback. Document why native-only fallback reaches the Final repair and qualify the auto/JVM boundary without changing the production predicate. * refactor: simplify aggregate fallback repair and regression tests * fix: address aggregate fallback review follow-ups Restore conditional diagnostics for known unrepaired buffer paths and focused coverage for direction defaults, immediate fallback reasons, and sticky shuffle refusal. Document the decimal SUM precision-overflow mismatch, preserve its fallback with a cancellation regression, and track remaining audits in #5975.
1 parent a51ff5c commit 6065705

9 files changed

Lines changed: 749 additions & 83 deletions

File tree

‎spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala‎

Lines changed: 113 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,16 @@ case class CometExecRule(session: SparkSession)
220220

221221
private def isCometNative(op: SparkPlan): Boolean = op.isInstanceOf[CometNativeExec]
222222

223+
/**
224+
* Restore a Spark Partial while retaining its current children. The tag prevents reconversion
225+
* when AQE replans the exchange without its Final, and records why the Partial stays in Spark.
226+
*/
227+
private def restoreSparkPartial(agg: CometHashAggregateExec, reason: String): SparkPlan = {
228+
val partial = agg.originalPlan.withNewChildren(agg.children)
229+
partial.setTagValue(CometExecRule.COMET_UNSAFE_PARTIAL, reason)
230+
withFallbackReason(partial, reason)
231+
}
232+
223233
/**
224234
* A Celeborn exchange can fall back after its child has been converted, for example because of
225235
* the partition threshold or an unsupported hash key. Keep incompatible partial aggregate
@@ -237,10 +247,8 @@ case class CometExecRule(session: SparkSession)
237247
case _: QueryStageExec | _: ShuffleExchangeLike | _: BroadcastExchangeLike => plan
238248
case agg: CometHashAggregateExec
239249
if agg.modes == Seq(Partial) &&
240-
!QueryPlanSerde.allAggsSupportMixedExecution(agg.aggregateExpressions) =>
241-
val sparkAggregate = agg.originalPlan.withNewChildren(agg.children)
242-
sparkAggregate.setTagValue(CometExecRule.COMET_UNSAFE_PARTIAL, reason)
243-
withFallbackReason(sparkAggregate, reason)
250+
!QueryPlanSerde.allAggsSupportNativePartialToSparkFinal(agg.aggregateExpressions) =>
251+
restoreSparkPartial(agg, reason)
244252
// Final output is ordinary SQL data; any partial below it belongs to another aggregate.
245253
case agg: CometHashAggregateExec if agg.modes.contains(Final) => agg
246254
case agg: BaseAggregateExec if agg.aggregateExpressions.exists(_.mode == Final) => agg
@@ -758,7 +766,7 @@ case class CometExecRule(session: SparkSession)
758766
// during the bottom-up conversion. Tags persist through AQE stage creation.
759767
tagUnsafePartialAggregates(planWithJoinRewritten)
760768

761-
var newPlan = transform(planWithJoinRewritten)
769+
var newPlan = revertUnsafePartialAggregates(transform(planWithJoinRewritten))
762770

763771
// if the plan cannot be run fully natively then explain why (when appropriate
764772
// config is enabled)
@@ -1130,7 +1138,7 @@ case class CometExecRule(session: SparkSession)
11301138
val consumerMode: AggregateMode =
11311139
if (modes.contains(PartialMerge)) PartialMerge else Final
11321140
if (consumesBuffers &&
1133-
!QueryPlanSerde.allAggsSupportMixedExecution(agg.aggregateExpressions) &&
1141+
!QueryPlanSerde.allAggsSupportNativePartialToSparkFinal(agg.aggregateExpressions) &&
11341142
!canAggregateBeConverted(agg, consumerMode)) {
11351143
findPartialAggInPlan(agg.child).foreach { partial =>
11361144
// Only tag if the Partial would otherwise have been converted. If the Partial itself
@@ -1173,6 +1181,103 @@ case class CometExecRule(session: SparkSession)
11731181
}
11741182
}
11751183

1184+
/**
1185+
* Inspect a failed repair's buffer path without rewriting it or materializing any stage. Report
1186+
* only a native Partial/PartialMerge whose emitted state is not known to be Spark-compatible.
1187+
* Spark Partials and completed aggregates establish new buffers, so stop there rather than
1188+
* finding an unrelated native producer below them. Only known aggregate and exchange wrappers
1189+
* forward the same buffer path; an arbitrary operator is not evidence of a mixed boundary.
1190+
*/
1191+
private def hasUnrepairedNativeBuffer(plan: SparkPlan): Boolean = plan match {
1192+
case agg: CometHashAggregateExec if agg.aggregateExpressions.isEmpty =>
1193+
hasUnrepairedNativeBuffer(agg.child)
1194+
case agg: CometHashAggregateExec =>
1195+
agg.modes.forall(m => m == Partial || m == PartialMerge) &&
1196+
!QueryPlanSerde.allAggsSupportNativePartialToSparkFinal(agg.aggregateExpressions)
1197+
case agg: BaseAggregateExec
1198+
if agg.aggregateExpressions.nonEmpty &&
1199+
agg.aggregateExpressions.forall(_.mode == Partial) =>
1200+
false
1201+
case agg: BaseAggregateExec =>
1202+
agg.aggregateExpressions.forall(e => e.mode == Partial || e.mode == PartialMerge) &&
1203+
hasUnrepairedNativeBuffer(agg.child)
1204+
case placeholder: CometSinkPlaceHolder => hasUnrepairedNativeBuffer(placeholder.child)
1205+
case read: AQEShuffleReadExec => hasUnrepairedNativeBuffer(read.child)
1206+
case stage: ShuffleQueryStageExec => hasUnrepairedNativeBuffer(stage.plan)
1207+
case reused: ReusedExchangeExec => hasUnrepairedNativeBuffer(reused.child)
1208+
case shuffle: CometShuffleExchangeExec => hasUnrepairedNativeBuffer(shuffle.child)
1209+
case shuffle: ShuffleExchangeExec => hasUnrepairedNativeBuffer(shuffle.child)
1210+
case _ => false
1211+
}
1212+
1213+
/**
1214+
* The early tagging pass cannot know whether a Final's child will become native. Check the
1215+
* actual conversion result before serialization or AQE stage creation, restoring the feeding
1216+
* aggregate/exchange chain while keeping native work below its Partial. Return the repaired
1217+
* plan, or preserve an unrepairable path and record one warning on its Spark Final if an unsafe
1218+
* native producer remains. Existing stages and their buffers are never rewritten by this pass.
1219+
*/
1220+
private[rules] def revertUnsafePartialAggregates(plan: SparkPlan): SparkPlan = {
1221+
def revertChain(node: SparkPlan): Option[SparkPlan] = node match {
1222+
case agg: CometHashAggregateExec if agg.modes == Seq(Partial) =>
1223+
Some(
1224+
restoreSparkPartial(
1225+
agg,
1226+
"Partial aggregate disabled: corresponding final aggregate " +
1227+
"cannot be converted to Comet and intermediate buffer formats are incompatible"))
1228+
1229+
case agg: CometHashAggregateExec
1230+
if agg.modes.forall(m => m == Partial || m == PartialMerge) =>
1231+
revertChain(agg.child).map(child => agg.originalPlan.withNewChildren(Seq(child)))
1232+
1233+
case agg: BaseAggregateExec
1234+
if agg.aggregateExpressions.nonEmpty &&
1235+
agg.aggregateExpressions.forall(_.mode == Partial) =>
1236+
// This producer already emits Spark buffers. Do not reach through it to an unrelated
1237+
// aggregate below it.
1238+
None
1239+
1240+
case agg: BaseAggregateExec
1241+
if agg.aggregateExpressions.forall(e => e.mode == Partial || e.mode == PartialMerge) =>
1242+
revertChain(agg.child).map(child => agg.withNewChildren(Seq(child)))
1243+
1244+
case CometSinkPlaceHolder(_, _, shuffle: CometShuffleExchangeExec) =>
1245+
revertChain(shuffle)
1246+
case shuffle: CometShuffleExchangeExec =>
1247+
revertChain(shuffle.child).map(child => shuffle.originalPlan.withNewChildren(Seq(child)))
1248+
case shuffle: ShuffleExchangeExec =>
1249+
revertChain(shuffle.child).map(child => shuffle.withNewChildren(Seq(child)))
1250+
1251+
// Stop at materialized stages and operators outside the feeding aggregate/exchange chain.
1252+
case _ => None
1253+
}
1254+
1255+
plan.transformUp {
1256+
case agg: BaseAggregateExec
1257+
if agg.aggregateExpressions.map(_.mode).distinct == Seq(Final) &&
1258+
!QueryPlanSerde.allAggsSupportNativePartialToSparkFinal(agg.aggregateExpressions) =>
1259+
revertChain(agg.child)
1260+
// Rebuild native consumers and shuffles from their original Spark operators. Merely
1261+
// replacing their children would leave a native protobuf reading the old buffers.
1262+
.map(child => transform(agg.withNewChildren(Seq(child))))
1263+
.getOrElse {
1264+
if (hasUnrepairedNativeBuffer(agg.child)) {
1265+
val reason = "Comet could not restore a native intermediate buffer producer " +
1266+
"below Spark final aggregate; the remaining buffer may be incompatible"
1267+
// AQE can revisit the same consumer. Record the explanation and warn once,
1268+
// regardless of whether general fallback logging is enabled.
1269+
if (!agg
1270+
.getTagValue(CometExplainInfo.FALLBACK_REASONS)
1271+
.exists(_.contains(reason))) {
1272+
if (!CometConf.COMET_EXPLAIN_FALLBACK_LOG_ENABLED.get()) logWarning(reason)
1273+
withFallbackReason(agg, reason)
1274+
}
1275+
}
1276+
agg
1277+
}
1278+
}
1279+
}
1280+
11761281
/**
11771282
* Look for the bottom Partial-mode aggregate that feeds into the given plan (the child of a
11781283
* Final). Walks through exchanges and AQE stages, and continues down through intermediate
@@ -1202,8 +1307,8 @@ case class CometExecRule(session: SparkSession)
12021307
/**
12031308
* Conservative check for whether an aggregate could be converted to Comet. Checks operator
12041309
* enablement, grouping expressions, aggregate expressions, and result expressions.
1205-
* Intentionally skips the sparkFinalMode / child-native checks since those depend on
1206-
* transformation state.
1310+
* Intentionally skips the child-native checks since those depend on transformation state;
1311+
* [[revertUnsafePartialAggregates]] checks the actual conversion result before execution.
12071312
*
12081313
* WARNING: this intentionally mirrors the predicate checks in `CometBaseAggregate.doConvert`
12091314
* (operators.scala). Any change to the convertibility rules there must be reflected here or

‎spark/src/main/scala/org/apache/comet/rules/RevertNativeForTransitionHeavyStages.scala‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,11 @@ case class RevertNativeForTransitionHeavyStages(session: SparkSession)
124124
def visit(plan: SparkPlan): Boolean = plan match {
125125
case _ if isStageBoundary(plan) => false
126126
case aggregate: CometHashAggregateExec
127-
if !QueryPlanSerde.allAggsSupportMixedExecution(aggregate.aggregateExpressions) =>
127+
if !QueryPlanSerde
128+
.allAggsSupportNativePartialToSparkFinal(aggregate.aggregateExpressions) ||
129+
QueryPlanSerde
130+
.aggsNotSupportingSparkPartialToNativeFinal(aggregate.aggregateExpressions)
131+
.nonEmpty =>
128132
val producesBuffer =
129133
aggregate.modes.exists(mode => mode == Partial || mode == PartialMerge)
130134
val consumesAcrossBoundary =

‎spark/src/main/scala/org/apache/comet/serde/CometAggregateExpressionSerde.scala‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -82,16 +82,21 @@ trait CometAggregateExpressionSerde[T <: AggregateFunction] {
8282
def getSupportLevel(expr: T): SupportLevel = Compatible(None)
8383

8484
/**
85-
* Whether this aggregate's intermediate buffer format is compatible between Spark and Comet for
86-
* the given function instance, making it safe to run the Partial in one engine and the Final in
87-
* the other. Aggregates with simple single-value buffers (MIN, MAX, bitwise) are always safe;
88-
* SUM and non-decimal AVG match Spark's buffer and are safe except where noted per instance
89-
* (e.g. TRY-mode SUM uses a Comet-internal flag column). COUNT is intentionally excluded
90-
* despite a matching buffer: mixed COUNT partial/final regressed AQE's
85+
* Whether a Comet aggregate can consume this function's Spark intermediate buffer. This covers
86+
* Spark Partial to Comet Final, including intermediate PartialMerge stages. COUNT is excluded
87+
* despite a matching buffer: a Comet Final above a Spark Partial regressed AQE's
9188
* PropagateEmptyRelationAfterAQE pattern (which matches BaseAggregateExec only) and the Spark
9289
* 4.0 count-bug decorrelation for correlated IN subqueries.
9390
*/
94-
def supportsMixedPartialFinal(fn: T): Boolean = false
91+
def supportsSparkPartialToNativeFinal(fn: T): Boolean = false
92+
93+
/**
94+
* Whether Spark can consume this function's Comet intermediate buffer. Opt in independently
95+
* from the reverse direction: consuming Spark state does not establish that Comet emits state
96+
* Spark can merge, especially from a never-updated or all-null partial accumulator. Remaining
97+
* forward-compatibility audits are tracked in issue #5975.
98+
*/
99+
def supportsNativePartialToSparkFinal(fn: T): Boolean = false
95100

96101
/**
97102
* Convert a Spark expression into a protocol buffer representation that can be passed into

‎spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -443,31 +443,35 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
443443
classOf[VarianceSamp] -> CometVarianceSamp)
444444

445445
/**
446-
* Returns true if all aggregate expressions in the list have intermediate buffer formats that
447-
* are compatible between Spark and Comet, making it safe to run Partial in one engine and Final
448-
* in the other.
446+
* Returns true if Spark can consume all the intermediate buffers produced by Comet. Used when a
447+
* Spark Final would otherwise consume a native Partial, including after shuffle fallback.
449448
*/
450-
def allAggsSupportMixedExecution(aggExprs: Seq[AggregateExpression]): Boolean = {
451-
aggExprs.forall(aggExpr => supportsMixedExecution(aggExpr.aggregateFunction))
449+
def allAggsSupportNativePartialToSparkFinal(aggExprs: Seq[AggregateExpression]): Boolean = {
450+
aggExprs.forall { aggExpr =>
451+
val fn = aggExpr.aggregateFunction
452+
aggrSerdeMap.get(fn.getClass).exists { handler =>
453+
handler
454+
.asInstanceOf[CometAggregateExpressionSerde[AggregateFunction]]
455+
.supportsNativePartialToSparkFinal(fn)
456+
}
457+
}
452458
}
453459

454460
/**
455-
* Returns the aggregate functions in the list whose intermediate buffer formats are not known
456-
* to be compatible between Spark and Comet. These are the functions that prevent a Spark Final
457-
* aggregate (without a Comet Partial) from running, since the buffer produced by one engine
458-
* cannot be safely consumed by the other.
461+
* Returns functions whose Spark intermediate buffers cannot safely be consumed by a Comet Final
462+
* or PartialMerge. This is independent of native Partial to Spark Final compatibility.
459463
*/
460-
def aggsNotSupportingMixedExecution(
464+
def aggsNotSupportingSparkPartialToNativeFinal(
461465
aggExprs: Seq[AggregateExpression]): Seq[AggregateFunction] = {
462-
aggExprs.map(_.aggregateFunction).filterNot(supportsMixedExecution)
466+
aggExprs.map(_.aggregateFunction).filterNot(supportsSparkPartialToNativeFinal)
463467
}
464468

465-
private def supportsMixedExecution(fn: AggregateFunction): Boolean = {
469+
private def supportsSparkPartialToNativeFinal(fn: AggregateFunction): Boolean = {
466470
aggrSerdeMap.get(fn.getClass) match {
467471
case Some(handler) =>
468472
handler
469473
.asInstanceOf[CometAggregateExpressionSerde[AggregateFunction]]
470-
.supportsMixedPartialFinal(fn)
474+
.supportsSparkPartialToNativeFinal(fn)
471475
case None => false
472476
}
473477
}

0 commit comments

Comments
 (0)