@@ -35,6 +35,7 @@ import org.apache.spark.sql.comet.CometHashAggregateExec
3535import org .apache .spark .sql .comet .execution .shuffle .CometShuffleExchangeExec
3636import org .apache .spark .sql .execution .SQLExecution
3737import org .apache .spark .sql .execution .adaptive .{AdaptiveSparkPlanExec , AdaptiveSparkPlanHelper }
38+ import org .apache .spark .sql .execution .aggregate .ObjectHashAggregateExec
3839import org .apache .spark .sql .execution .exchange .ReusedExchangeExec
3940import org .apache .spark .sql .functions .{avg , col , count_distinct , expr , sum }
4041import org .apache .spark .sql .internal .SQLConf
@@ -1654,6 +1655,49 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
16541655 }
16551656 }
16561657
1658+ test(" decimal sum with a codegen fallback expression falls back at maximum precision" ) {
1659+ // A non-leaf CodegenFallback expression such as a lambda function anywhere in a stage's
1660+ // expressions turns codegen off for that stage, and Comet runs the expression natively, so
1661+ // only the guard keeps Spark's latched answer: in the final stage the merge of the 0.6, 0.6
1662+ // and -0.6 partials latches, in the partial stage the running sum does.
1663+ val reason = " Ungrouped decimal SUM at maximum precision without codegen cannot match " +
1664+ " Spark's latching UnsafeRow buffer"
1665+ Seq (true , false ).foreach { ansiEnabled =>
1666+ withSQLConf(
1667+ SQLConf .ANSI_ENABLED .key -> ansiEnabled.toString,
1668+ CometConf .COMET_SHUFFLE_ENABLED .key -> " true" ,
1669+ CometConf .COMET_SHUFFLE_MODE .key -> " native" ) {
1670+ withMaxPrecisionDecimalTable(Seq ((1 , " 0.6" )), " dec_lambda_a" ) {
1671+ withMaxPrecisionDecimalTable(Seq ((1 , " 0.6" )), " dec_lambda_b" ) {
1672+ withMaxPrecisionDecimalTable(Seq ((1 , " -0.6" )), " dec_lambda_c" ) {
1673+ // Each partial fits on its own; the lambda in the result expressions turns
1674+ // codegen off for the final, whose merge latches.
1675+ assertDecimalSumFallsBackLikeSpark(
1676+ sql(
1677+ " SELECT SUM(v), filter(array(MAX(k)), x -> x IS NOT NULL) FROM " +
1678+ " (SELECT k, v FROM dec_lambda_a UNION ALL SELECT k, v FROM dec_lambda_b " +
1679+ " UNION ALL SELECT k, v FROM dec_lambda_c)" ),
1680+ reason,
1681+ ansiEnabled,
1682+ Row (null , Seq (1 )))
1683+ }
1684+ }
1685+ }
1686+ withMaxPrecisionDecimalTable(intermediateOverflowRows, " dec_lambda" ) {
1687+ // The lambda in the FILTER clause turns codegen off for the partial, whose running
1688+ // sum latches.
1689+ assertDecimalSumFallsBackLikeSpark(
1690+ sql(
1691+ " SELECT SUM(v) FILTER (WHERE size(filter(array(k), x -> x IS NOT NULL)) = 1) " +
1692+ " FROM dec_lambda" ),
1693+ reason,
1694+ ansiEnabled,
1695+ Row (null ))
1696+ }
1697+ }
1698+ }
1699+ }
1700+
16571701 /**
16581702 * Asserts that `df` runs the aggregate in Spark with `reason` recorded and gives Spark's
16591703 * latched result: `expected` in legacy mode, and the same failure as Spark under ANSI.
@@ -1715,6 +1759,67 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
17151759 }
17161760 }
17171761
1762+ test(" decimal sum with ungrouped object hash aggregate recovers like Spark" ) {
1763+ // Spark's object aggregation keeps the ungrouped buffer in a SpecificInternalRow, which
1764+ // holds the intermediate 1.2 unbounded like the native accumulator, so both recover 0.6.
1765+ Seq (true , false ).foreach { ansiEnabled =>
1766+ withSQLConf(
1767+ SQLConf .ANSI_ENABLED .key -> ansiEnabled.toString,
1768+ SQLConf .USE_OBJECT_HASH_AGG .key -> " true" ,
1769+ CometConf .COMET_SHUFFLE_ENABLED .key -> " true" ,
1770+ CometConf .COMET_SHUFFLE_MODE .key -> " native" ) {
1771+ withMaxPrecisionDecimalTable(intermediateOverflowRows, " dec_object_ungrouped" ) {
1772+ val df = sql(" SELECT SUM(v), collect_list(k) FROM dec_object_ungrouped" )
1773+ assert(
1774+ df.queryExecution.sparkPlan.exists(_.isInstanceOf [ObjectHashAggregateExec ]),
1775+ s " expected Spark to plan an object hash aggregate: ${df.queryExecution.sparkPlan}" )
1776+ checkSparkAnswerAndOperator(df)
1777+ val answer = df.collect().toSeq
1778+ assert(
1779+ answer == Seq (Row (recoveredSum.bigDecimal.setScale(38 ), Seq (1 , 1 , 1 ))),
1780+ s " ungrouped object hash aggregate with ansi= $ansiEnabled returned $answer, " +
1781+ s " expected the recovered sum $recoveredSum" )
1782+ }
1783+ }
1784+ }
1785+ }
1786+
1787+ test(" decimal sum distinct recovers without group by and latches with group by" ) {
1788+ // Spark's distinct rewrite sums the distinct values in its own stage. Ungrouped, that stage
1789+ // keeps the running sum unbounded in both engines; grouped, both latch once the running sum
1790+ // 1.1 leaves the precision. A single shuffle partition keeps the distinct values in scan
1791+ // order so the same running sums are formed.
1792+ val distinctSum = BigDecimal (" 0.5" )
1793+ Seq (true , false ).foreach { ansiEnabled =>
1794+ withSQLConf(
1795+ SQLConf .ANSI_ENABLED .key -> ansiEnabled.toString,
1796+ SQLConf .SHUFFLE_PARTITIONS .key -> " 1" ,
1797+ CometConf .COMET_SHUFFLE_ENABLED .key -> " true" ,
1798+ CometConf .COMET_SHUFFLE_MODE .key -> " native" ) {
1799+ withMaxPrecisionDecimalTable(Seq ((1 , " 0.6" ), (1 , " 0.5" ), (1 , " -0.6" )), " dec_distinct" ) {
1800+ val ungrouped = sql(" SELECT SUM(DISTINCT v) FROM dec_distinct" )
1801+ checkSparkAnswerAndOperator(ungrouped)
1802+ val answer = ungrouped.collect().toSeq
1803+ assert(
1804+ answer == Seq (Row (distinctSum.bigDecimal.setScale(38 ))),
1805+ s " ungrouped distinct sum with ansi= $ansiEnabled returned $answer, " +
1806+ s " expected $distinctSum" )
1807+ val grouped = sql(" SELECT k, SUM(DISTINCT v) FROM dec_distinct GROUP BY k" )
1808+ if (ansiEnabled) {
1809+ checkSparkError(grouped, " ARITHMETIC_OVERFLOW" )
1810+ } else {
1811+ checkSparkAnswerAndOperator(grouped)
1812+ val groupedAnswer = grouped.collect().toSeq
1813+ assert(
1814+ groupedAnswer == Seq (Row (1 , null )),
1815+ s " grouped distinct sum with ansi= $ansiEnabled returned $groupedAnswer, " +
1816+ " expected a null sum" )
1817+ }
1818+ }
1819+ }
1820+ }
1821+ }
1822+
17181823 test(" avg decimal with nothing to average" ) {
17191824 // There is no sum to overflow when the input is empty or entirely null, so avg must
17201825 // return null rather than raising ARITHMETIC_OVERFLOW in ANSI mode.
0 commit comments