Skip to content

Commit 6db6c1b

Browse files
committed
fix: treat the NO_CODEGEN factory mode as codegen off for the decimal SUM fallback
CollapseCodegenStages skips whole-stage codegen when spark.sql.codegen.factoryMode is NO_CODEGEN, and the interpreted projection then writes each update into the UnsafeRow buffer that latches on overflow, so the fallback reads the mode the same way.
1 parent 90cfbe6 commit 6db6c1b

3 files changed

Lines changed: 18 additions & 7 deletions

File tree

‎docs/source/user-guide/latest/compatibility/index.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -149,8 +149,9 @@ so users hunting an unexpected value have a single place to check:
149149
when a partial is written out or the sum is evaluated, which matches Spark's whole-stage codegen
150150
path. Without codegen Spark buffers the aggregate in an `UnsafeRow` and latches as soon as a
151151
running sum leaves the precision. Comet falls back at precision 38 when codegen is disabled by
152-
`spark.sql.codegen.wholeStage`, by an imperative sibling aggregate, or by the
153-
`spark.sql.codegen.maxFields` limit on the aggregate's own output and inputs, but not after a
152+
`spark.sql.codegen.wholeStage`, by `spark.sql.codegen.factoryMode=NO_CODEGEN`, by an
153+
imperative sibling aggregate, or by the `spark.sql.codegen.maxFields` limit on the aggregate's
154+
own output and inputs, but not after a
154155
codegen compile failure at runtime, where an intermediate overflow that later cancels out
155156
returns `NULL` (or raises under ANSI) in Spark but the recovered value in Comet.
156157

‎spark/src/main/scala/org/apache/spark/sql/comet/operators.scala‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import org.apache.spark.broadcast.Broadcast
3030
import org.apache.spark.internal.Logging
3131
import org.apache.spark.rdd.RDD
3232
import org.apache.spark.sql.catalyst.InternalRow
33-
import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, AttributeSeq, AttributeSet, Expression, ExpressionSet, Generator, NamedExpression, SortOrder, XXH64}
33+
import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, AttributeSeq, AttributeSet, CodegenObjectFactoryMode, Expression, ExpressionSet, Generator, NamedExpression, SortOrder, XXH64}
3434
import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateMode, CollectList, CollectSet, Final, ImperativeAggregate, Mode, Partial, PartialMerge, Percentile, Sum}
3535
import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight, BuildSide}
3636
import org.apache.spark.sql.catalyst.plans._
@@ -2230,9 +2230,12 @@ object CometHashAggregateExec
22302230
}
22312231
// Without codegen Spark buffers an ungrouped aggregate in an UnsafeRow, which latches a
22322232
// decimal sum that leaves the precision, while the native ungrouped accumulator keeps the
2233-
// unbounded intermediate. Spark turns codegen off for an imperative aggregate, by config,
2234-
// or when the operator's output or an input carries more fields than its codegen limit.
2233+
// unbounded intermediate. Spark turns codegen off for an imperative aggregate, by the
2234+
// whole-stage or factory-mode config, or when the operator's output or an input carries
2235+
// more fields than its codegen limit.
22352236
val codegenOff = !op.conf.wholeStageEnabled ||
2237+
CodegenObjectFactoryMode.withName(op.conf.codegenFactoryMode) ==
2238+
CodegenObjectFactoryMode.NO_CODEGEN ||
22362239
op.aggregateExpressions.exists(_.aggregateFunction.isInstanceOf[ImperativeAggregate]) ||
22372240
WholeStageCodegenExec.isTooManyFields(op.conf, op.schema) ||
22382241
op.children.exists(child => WholeStageCodegenExec.isTooManyFields(op.conf, child.schema))

‎spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1600,8 +1600,8 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
16001600
test("decimal sum without codegen falls back at maximum precision") {
16011601
// Without codegen Spark buffers the ungrouped sum in an UnsafeRow and latches once it
16021602
// leaves the precision; an imperative sibling such as approx_count_distinct, the whole-stage
1603-
// switch, or a field-count limit below the partial's two buffer columns turns codegen off,
1604-
// and falling back keeps Spark's answer.
1603+
// switch, the NO_CODEGEN factory mode, or a field-count limit below the partial's two buffer
1604+
// columns turns codegen off, and falling back keeps Spark's answer.
16051605
val reason = "Ungrouped decimal SUM at maximum precision without codegen cannot match " +
16061606
"Spark's latching UnsafeRow buffer"
16071607
Seq(true, false).foreach { ansiEnabled =>
@@ -1629,6 +1629,13 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
16291629
ansiEnabled,
16301630
Row(null))
16311631
}
1632+
withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> "NO_CODEGEN") {
1633+
assertDecimalSumFallsBackLikeSpark(
1634+
sql("SELECT SUM(v) FROM dec_no_codegen"),
1635+
reason,
1636+
ansiEnabled,
1637+
Row(null))
1638+
}
16321639
}
16331640
}
16341641
}

0 commit comments

Comments
 (0)