@@ -1860,9 +1860,9 @@ class CometWindowExecSuite extends CometTestBase {
18601860 // offset -> Decimal(11,0) vs current Decimal(10,0), "Uncomparable values").
18611861 // The upstream fix (apache/datafusion#22174, closes #22113) makes
18621862 // ScalarValue::partial_cmp ignore precision for Decimal128 when scales match,
1863- // and is included in DataFusion 54.0.0 which Comet already pins . These tests
1864- // pin the behavior with `checkSparkAnswerAndOperator` so results must match
1865- // Spark AND run natively.
1863+ // and was first released in DataFusion 54.0.0. These tests pin the behavior
1864+ // with `checkSparkAnswerAndOperator` so results must match Spark AND run
1865+ // natively.
18661866 // ---------------------------------------------------------------------------
18671867
18681868 private def writeDecimalWindowFixture (dir : java.io.File , decimalType : String ): Unit = {
@@ -2107,4 +2107,71 @@ class CometWindowExecSuite extends CometTestBase {
21072107 checkSparkAnswerAndOperator(df)
21082108 }
21092109 }
2110+
2111+ private def writeDecimalOverflowFixture (dir : java.io.File , values : Seq [String ]): Unit = {
2112+ values
2113+ .map(Option (_))
2114+ .:+ (Option .empty[String ])
2115+ .toDF(" d_str" )
2116+ .selectExpr(" CAST(d_str AS DECIMAL(38, 0)) AS d" )
2117+ .repartition(1 )
2118+ .write
2119+ .mode(" overwrite" )
2120+ .parquet(dir.toString)
2121+ }
2122+
2123+ test(" window: RANGE offset on DECIMAL(38, 0) ORDER BY that overflows Decimal128 falls back" ) {
2124+ // Spark's DecimalAddNoOverflowCheck keeps the exact bound, so the bound 2N of the row N stays
2125+ // below the trailing NULL. DataFusion's i128 bound overflows and becomes the partition edge,
2126+ // which takes the NULL in, so this falls back.
2127+ // https://github.com/apache/datafusion-comet/issues/5022
2128+ val n = " 90000000000000000000000000000000000000"
2129+ withTempDir { dir =>
2130+ writeDecimalOverflowFixture(dir, Seq (" 0" , n))
2131+ spark.read.parquet(dir.toString).createOrReplaceTempView(" dec_overflow" )
2132+ for (ansi <- Seq (" true" , " false" )) {
2133+ withSQLConf(SQLConf .ANSI_ENABLED .key -> ansi) {
2134+ val df = sql(s """
2135+ SELECT d, COUNT(*) OVER (ORDER BY d ASC NULLS LAST
2136+ RANGE BETWEEN $n PRECEDING AND $n FOLLOWING) AS cnt
2137+ FROM dec_overflow
2138+ """ )
2139+ val (_, cometPlan) = checkSparkAnswerAndFallbackReason(
2140+ df,
2141+ " RANGE frame on decimal(38,0) ORDER BY with an offset whose bound can overflow " +
2142+ " Decimal128 is not supported" )
2143+ assert(collect(cometPlan) { case w : CometWindowExec => w }.isEmpty, cometPlan)
2144+ checkAnswer(df, Seq (Row (BigDecimal (0 ), 2L ), Row (BigDecimal (n), 2L ), Row (null , 1L )))
2145+ }
2146+ }
2147+ }
2148+ }
2149+
2150+ test(
2151+ " window: largest RANGE offset on DECIMAL(38, 0) ORDER BY that cannot overflow runs natively" ) {
2152+ // The limit is 2^127 - 1 minus the largest DECIMAL(38, 0), that is
2153+ // 70141183460469231731687303715884105728, so `current +/- offset` stays in an i128 for every
2154+ // value of the type, including the type's extremes next to NULLs at either edge. Spark negates
2155+ // a PRECEDING offset with 34 significant digits, which would round the limit itself past it,
2156+ // so use the largest 34-digit offset under it.
2157+ val maxValue = " 9" * 38
2158+ val offset = " 70141183460469231731687303715884100000"
2159+ withTempDir { dir =>
2160+ writeDecimalOverflowFixture(dir, Seq (s " - $maxValue" , " 0" , maxValue))
2161+ spark.read.parquet(dir.toString).createOrReplaceTempView(" dec_edge" )
2162+ for {
2163+ ansi <- Seq (" true" , " false" )
2164+ order <- Seq (" ASC NULLS LAST" , " ASC NULLS FIRST" , " DESC NULLS LAST" , " DESC NULLS FIRST" )
2165+ } {
2166+ withSQLConf(SQLConf .ANSI_ENABLED .key -> ansi) {
2167+ val df = sql(s """
2168+ SELECT d, COUNT(*) OVER (ORDER BY d $order
2169+ RANGE BETWEEN $offset PRECEDING AND $offset FOLLOWING) AS cnt
2170+ FROM dec_edge
2171+ """ )
2172+ checkSparkAnswerAndOperator(df)
2173+ }
2174+ }
2175+ }
2176+ }
21102177}
0 commit comments