Skip to content

Commit 6c14796

Browse files
committed
fix: clarify aggregate buffer compatibility and fallback
1 parent 5f8aa4d commit 6c14796

7 files changed

Lines changed: 307 additions & 23 deletions

File tree

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -476,6 +476,7 @@ jobs:
476476
org.apache.comet.CometWidthBucketSuite
477477
org.apache.comet.CometUuidExpressionSuite
478478
org.apache.comet.serde.CometScalarFunctionSuite
479+
org.apache.comet.serde.CometAggregateExpressionSerdeSuite
479480
org.apache.comet.CometFallbackInvarianceSuite
480481
fail-fast: false
481482
name: ${{ matrix.profile.name }} [${{ matrix.suite.name }}]

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,7 @@ jobs:
249249
org.apache.comet.CometWidthBucketSuite
250250
org.apache.comet.CometUuidExpressionSuite
251251
org.apache.comet.serde.CometScalarFunctionSuite
252+
org.apache.comet.serde.CometAggregateExpressionSerdeSuite
252253
org.apache.comet.CometFallbackInvarianceSuite
253254
254255
fail-fast: false

‎spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala‎

Lines changed: 71 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,18 @@ case class CometExecRule(session: SparkSession)
206206

207207
private def isCometNative(op: SparkPlan): Boolean = op.isInstanceOf[CometNativeExec]
208208

209+
/**
210+
* Restore a native Partial's original Spark operator with its current children. Both repair
211+
* paths use this to preserve native work below the Partial, prevent reconversion, and record
212+
* the explanation immediately, even if a non-native child prevents another serde attempt. The
213+
* caller establishes buffer incompatibility; this does not traverse or rewrite children.
214+
*/
215+
private def restoreSparkPartial(agg: CometHashAggregateExec, reason: String): SparkPlan = {
216+
val partial = agg.originalPlan.withNewChildren(agg.children)
217+
partial.setTagValue(CometExecRule.COMET_UNSAFE_PARTIAL, reason)
218+
withFallbackReason(partial, reason)
219+
}
220+
209221
/**
210222
* A Celeborn exchange can fall back after its child has been converted, for example because of
211223
* the partition threshold or an unsupported hash key. Keep incompatible partial aggregate
@@ -224,9 +236,7 @@ case class CometExecRule(session: SparkSession)
224236
case agg: CometHashAggregateExec
225237
if agg.modes == Seq(Partial) &&
226238
!QueryPlanSerde.allAggsSupportNativePartialToSparkFinal(agg.aggregateExpressions) =>
227-
val sparkAggregate = agg.originalPlan.withNewChildren(agg.children)
228-
sparkAggregate.setTagValue(CometExecRule.COMET_UNSAFE_PARTIAL, reason)
229-
withFallbackReason(sparkAggregate, reason)
239+
restoreSparkPartial(agg, reason)
230240
// Final output is ordinary SQL data; any partial below it belongs to another aggregate.
231241
case agg: CometHashAggregateExec if agg.modes.contains(Final) => agg
232242
case agg: BaseAggregateExec if agg.aggregateExpressions.exists(_.mode == Final) => agg
@@ -1133,20 +1143,51 @@ case class CometExecRule(session: SparkSession)
11331143
}
11341144
}
11351145

1146+
/**
1147+
* Inspect an unrepaired buffer path without modifying it, including already-created stages.
1148+
* Unlike a descendant search, stop at Spark producers and completed aggregates so an unrelated
1149+
* inner native Partial cannot trigger a warning. Merge stages still consume intermediate
1150+
* buffers. Unknown unary wrappers are inspected only when they preserve all output attributes;
1151+
* changed-output and branching nodes are not assumed to forward the same buffers.
1152+
*/
1153+
private def hasUnrepairedNativePartial(plan: SparkPlan): Boolean = plan match {
1154+
case agg: CometHashAggregateExec if agg.aggregateExpressions.isEmpty =>
1155+
hasUnrepairedNativePartial(agg.child)
1156+
case agg: CometHashAggregateExec =>
1157+
agg.modes.forall(m => m == Partial || m == PartialMerge) &&
1158+
!QueryPlanSerde.allAggsSupportNativePartialToSparkFinal(agg.aggregateExpressions)
1159+
case agg: BaseAggregateExec
1160+
if agg.aggregateExpressions.nonEmpty &&
1161+
agg.aggregateExpressions.forall(_.mode == Partial) =>
1162+
false
1163+
case agg: BaseAggregateExec =>
1164+
agg.aggregateExpressions.forall(e => e.mode == Partial || e.mode == PartialMerge) &&
1165+
hasUnrepairedNativePartial(agg.child)
1166+
case stage: QueryStageExec => hasUnrepairedNativePartial(stage.plan)
1167+
case reused: ReusedExchangeExec => hasUnrepairedNativePartial(reused.child)
1168+
case read: AQEShuffleReadExec => hasUnrepairedNativePartial(read.child)
1169+
case placeholder: CometSinkPlaceHolder => hasUnrepairedNativePartial(placeholder.child)
1170+
case shuffle: CometShuffleExchangeExec => hasUnrepairedNativePartial(shuffle.child)
1171+
case other if other.children.size == 1 && other.output == other.children.head.output =>
1172+
hasUnrepairedNativePartial(other.children.head)
1173+
case _ => false
1174+
}
1175+
11361176
/**
11371177
* The early tagging pass cannot know whether a Final's child will become native. Check the
11381178
* actual conversion result as well, before native blocks are serialized or AQE launches stages.
1139-
* Restore only the feeding aggregate/exchange chain; keep native work below its Partial.
1179+
* Restore only the feeding aggregate/exchange chain; keep native work below its Partial. If a
1180+
* remaining native buffer producer cannot be restored, warn and annotate the Spark Final; do
1181+
* not rewrite materialized stages or assume unknown operators can safely be reconstructed.
11401182
*/
1141-
private def revertUnsafePartialAggregates(plan: SparkPlan): SparkPlan = {
1183+
private[rules] def revertUnsafePartialAggregates(plan: SparkPlan): SparkPlan = {
11421184
def revertChain(node: SparkPlan): Option[SparkPlan] = node match {
11431185
case agg: CometHashAggregateExec if agg.modes == Seq(Partial) =>
1144-
val partial = agg.originalPlan.withNewChildren(Seq(agg.child))
1145-
partial.setTagValue(
1146-
CometExecRule.COMET_UNSAFE_PARTIAL,
1147-
"Partial aggregate disabled: corresponding final aggregate " +
1148-
"cannot be converted to Comet and intermediate buffer formats are incompatible")
1149-
Some(partial)
1186+
Some(
1187+
restoreSparkPartial(
1188+
agg,
1189+
"Partial aggregate disabled: corresponding final aggregate " +
1190+
"cannot be converted to Comet and intermediate buffer formats are incompatible"))
11501191

11511192
case agg: CometHashAggregateExec
11521193
if agg.modes.forall(m => m == Partial || m == PartialMerge) =>
@@ -1186,7 +1227,25 @@ case class CometExecRule(session: SparkSession)
11861227
// Rebuild native consumers and shuffles from their original Spark operators. Merely
11871228
// replacing their children would leave a native protobuf reading the old buffers.
11881229
.map(child => transform(agg.withNewChildren(Seq(child))))
1189-
.getOrElse(agg)
1230+
.getOrElse {
1231+
if (hasUnrepairedNativePartial(agg.child)) {
1232+
val reason = "Comet could not restore a native intermediate buffer producer " +
1233+
s"below Spark final aggregate (${agg.child.nodeName}); " +
1234+
"the remaining aggregate boundary may have incompatible buffer formats"
1235+
// AQE may reapply this rule to the same consumer. Keep the explanation, but do
1236+
// not repeat a warning that has already been attached to this plan node.
1237+
if (!agg
1238+
.getTagValue(CometExplainInfo.FALLBACK_REASONS)
1239+
.exists(_.contains(reason))) {
1240+
if (!CometConf.COMET_EXPLAIN_FALLBACK_LOG_ENABLED.get()) logWarning(reason)
1241+
withFallbackReason(agg, reason)
1242+
} else {
1243+
agg
1244+
}
1245+
} else {
1246+
agg
1247+
}
1248+
}
11901249
}
11911250
}
11921251

‎spark/src/main/scala/org/apache/comet/serde/CometAggregateExpressionSerde.scala‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -91,14 +91,11 @@ trait CometAggregateExpressionSerde[T <: AggregateFunction] {
9191
def supportsSparkPartialToNativeFinal(fn: T): Boolean = false
9292

9393
/**
94-
* Whether Spark can consume this function's Comet intermediate buffer. Keep this separate from
95-
* the reverse direction: planner restrictions on a Comet Final do not necessarily prohibit a
96-
* Comet Partial feeding a Spark Final. Existing bidirectional implementations share the
97-
* default; handlers may admit an additional forward direction only after validating it
98-
* independently.
94+
* Whether Spark can consume this function's Comet intermediate buffer. Opt in independently
95+
* from the reverse direction: consuming Spark state does not establish that Comet emits state
96+
* Spark can merge, especially from a never-updated or all-null partial accumulator.
9997
*/
100-
def supportsNativePartialToSparkFinal(fn: T): Boolean =
101-
supportsSparkPartialToNativeFinal(fn)
98+
def supportsNativePartialToSparkFinal(fn: T): Boolean = false
10299

103100
/**
104101
* Convert a Spark expression into a protocol buffer representation that can be passed into

‎spark/src/main/scala/org/apache/comet/serde/aggregates.scala‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ object CometMin extends CometAggregateExpressionSerde[Min] {
3737

3838
override def supportsSparkPartialToNativeFinal(fn: Min): Boolean = true
3939

40+
// Native MIN emits one typed null for empty/all-null input; Spark's least merge ignores it.
41+
override def supportsNativePartialToSparkFinal(fn: Min): Boolean = true
42+
4043
override def getSupportLevel(expr: Min): SupportLevel =
4144
AggSerde.minMaxSupportLevel(expr.dataType)
4245

@@ -73,6 +76,9 @@ object CometMax extends CometAggregateExpressionSerde[Max] {
7376

7477
override def supportsSparkPartialToNativeFinal(fn: Max): Boolean = true
7578

79+
// Native MAX emits one typed null for empty/all-null input; Spark's greatest merge ignores it.
80+
override def supportsNativePartialToSparkFinal(fn: Max): Boolean = true
81+
7682
override def getSupportLevel(expr: Max): SupportLevel =
7783
AggSerde.minMaxSupportLevel(expr.dataType)
7884

@@ -198,6 +204,12 @@ object CometAverage extends CometAggregateExpressionSerde[Average] {
198204

199205
object CometSum extends CometAggregateExpressionSerde[Sum] {
200206

207+
// Non-decimal, non-TRY SUM emits one nullable sum, including null for empty/all-null input;
208+
// Spark's coalesce-based merge accepts it. Decimal and TRY state remain excluded.
209+
override def supportsNativePartialToSparkFinal(fn: Sum): Boolean =
210+
!fn.child.dataType.isInstanceOf[DecimalType] &&
211+
CometEvalModeUtil.fromSparkEvalMode(CometEvalModeUtil.sumEvalMode(fn)) != CometEvalMode.TRY
212+
201213
override def supportsSparkPartialToNativeFinal(fn: Sum): Boolean =
202214
// Decimal SUM is excluded: overflow detection (ANSI throw / Legacy null) does not survive a
203215
// Spark-partial / Comet-final split, so the required ArithmeticException is never raised.
@@ -317,6 +329,9 @@ object CometLast extends CometAggregateExpressionSerde[Last] {
317329
object CometBitAndAgg extends CometAggregateExpressionSerde[BitAndAgg] {
318330
override def supportsSparkPartialToNativeFinal(fn: BitAndAgg): Boolean = true
319331

332+
// The single native buffer is null for empty/all-null input; Spark's merge skips nulls.
333+
override def supportsNativePartialToSparkFinal(fn: BitAndAgg): Boolean = true
334+
320335
override def getSupportLevel(expr: BitAndAgg): SupportLevel =
321336
if (AggSerde.bitwiseAggTypeSupported(expr.dataType)) {
322337
Compatible()
@@ -355,6 +370,9 @@ object CometBitAndAgg extends CometAggregateExpressionSerde[BitAndAgg] {
355370
object CometBitOrAgg extends CometAggregateExpressionSerde[BitOrAgg] {
356371
override def supportsSparkPartialToNativeFinal(fn: BitOrAgg): Boolean = true
357372

373+
// The single native buffer is null for empty/all-null input; Spark's merge skips nulls.
374+
override def supportsNativePartialToSparkFinal(fn: BitOrAgg): Boolean = true
375+
358376
override def getSupportLevel(expr: BitOrAgg): SupportLevel =
359377
if (AggSerde.bitwiseAggTypeSupported(expr.dataType)) {
360378
Compatible()
@@ -393,6 +411,9 @@ object CometBitOrAgg extends CometAggregateExpressionSerde[BitOrAgg] {
393411
object CometBitXOrAgg extends CometAggregateExpressionSerde[BitXorAgg] {
394412
override def supportsSparkPartialToNativeFinal(fn: BitXorAgg): Boolean = true
395413

414+
// The single native buffer is null for empty/all-null input; Spark's merge skips nulls.
415+
override def supportsNativePartialToSparkFinal(fn: BitXorAgg): Boolean = true
416+
396417
override def getSupportLevel(expr: BitXorAgg): SupportLevel =
397418
if (AggSerde.bitwiseAggTypeSupported(expr.dataType)) {
398419
Compatible()
@@ -901,6 +922,10 @@ object CometBloomFilterAggregate extends CometAggregateExpressionSerde[BloomFilt
901922

902923
override def supportsSparkPartialToNativeFinal(fn: BloomFilterAggregate): Boolean = true
903924

925+
// Native state is Spark's serialized filter, non-null even for empty/all-null input; only
926+
// the final result may be null, so Spark's deserialize always receives a valid filter.
927+
override def supportsNativePartialToSparkFinal(fn: BloomFilterAggregate): Boolean = true
928+
904929
override def getSupportLevel(expr: BloomFilterAggregate): SupportLevel =
905930
expr.child.dataType match {
906931
case _: ByteType | _: ShortType | _: IntegerType | _: LongType | _: StringType =>
@@ -1070,6 +1095,9 @@ object CometApproxCountDistinct extends CometAggregateExpressionSerde[HyperLogLo
10701095
// be mixed in one plan.
10711096
override def supportsSparkPartialToNativeFinal(fn: HyperLogLogPlusPlus): Boolean = true
10721097

1098+
// Native empty/all-null state contains non-null zero Long words, matching Spark's registers.
1099+
override def supportsNativePartialToSparkFinal(fn: HyperLogLogPlusPlus): Boolean = true
1100+
10731101
// Types that Comet's native `xxhash64` hashes identically to Spark's `XxHash64Function`.
10741102
// `StringType` here is the default UTF8_BINARY collation; a collated `StringType(collationId)`
10751103
// falls through to the `case _` because Spark hashes those via the collation sort key.

0 commit comments

Comments
 (0)