@@ -29,8 +29,9 @@ import org.apache.spark.sql.{CometTestBase, Row}
2929import org .apache .spark .sql .api .java .UDF1
3030import org .apache .spark .sql .catalyst .InternalRow
3131import org .apache .spark .sql .catalyst .analysis .NoSuchFunctionException
32- import org .apache .spark .sql .catalyst .expressions .{Add , Alias , AttributeReference , BoundReference , Cast , CreateArray , CreateMap , CreateNamedStruct , Expression , Hypot , Literal , MapConcat }
32+ import org .apache .spark .sql .catalyst .expressions .{Add , Alias , AttributeReference , BoundReference , Cast , CreateArray , CreateMap , CreateNamedStruct , Expression , GenericInternalRow , Hypot , Literal , MapConcat }
3333import org .apache .spark .sql .catalyst .expressions .objects .Invoke
34+ import org .apache .spark .sql .catalyst .util .GenericArrayData
3435import org .apache .spark .sql .comet .CometProjectExec
3536import org .apache .spark .sql .connector .catalog .{FunctionCatalog , Identifier }
3637import org .apache .spark .sql .connector .catalog .functions .{BoundFunction , ScalarFunction , UnboundFunction }
@@ -2333,71 +2334,112 @@ class CometCodegenSuite
23332334 assert(runKernel(folded, 1 )(_.getUTF8String(0 ).toString) === " abab" )
23342335 }
23352336
2336- test(" decimal results of a DSv2 function are rescaled to the declared type (#6425)" ) {
2337- // Spark lowers a call to a DSv2 function with an instance `invoke` method to `Invoke`, which
2338- // the dispatcher runs. `as_money` and `as_wide_money` return `Decimal(i)` at scale 0, one
2339- // declaring `DECIMAL(10, 2)` and one `DECIMAL(20, 12)`, which covers both of the dispatcher's
2340- // decimal writers. Spark's row writer rescales the value with `changePrecision` and writes
2341- // null when it does not fit: 100000000 and -100000000 have nine integer digits and both types
2342- // allow eight. Spark adds no overflow check around the call, so that null does not depend on
2343- // ANSI mode. `map` is itself dispatched, so its value exercises the nested writer.
2344- // `mills_as_money` returns `i` thousandths, at scale 3, into `DECIMAL(10, 2)`, so the rescale
2345- // drops a digit and `changePrecision` rounds half up: -1.005 becomes -1.01 and 1.004 becomes
2346- // 1.00.
2347- def dec (s : String ) = new java.math.BigDecimal (s)
2348- val expected = Seq (
2349- Row (3 , dec(" 3.00" ), dec(" 3.000000000000" ), Map (" k" -> dec(" 3.00" )), dec(" 0.00" )),
2350- Row (- 7 , dec(" -7.00" ), dec(" -7.000000000000" ), Map (" k" -> dec(" -7.00" )), dec(" -0.01" )),
2351- Row (null , null , null , Map (" k" -> null ), null ),
2352- Row (
2353- 99999999 ,
2354- dec(" 99999999.00" ),
2355- dec(" 99999999.000000000000" ),
2356- Map (" k" -> dec(" 99999999.00" )),
2357- dec(" 100000.00" )),
2358- Row (100000000 , null , null , Map (" k" -> null ), dec(" 100000.00" )),
2359- Row (
2360- - 99999999 ,
2361- dec(" -99999999.00" ),
2362- dec(" -99999999.000000000000" ),
2363- Map (" k" -> dec(" -99999999.00" )),
2364- dec(" -100000.00" )),
2365- Row (- 100000000 , null , null , Map (" k" -> null ), dec(" -100000.00" )),
2366- Row (
2367- - 1005 ,
2368- dec(" -1005.00" ),
2369- dec(" -1005.000000000000" ),
2370- Map (" k" -> dec(" -1005.00" )),
2371- dec(" -1.01" )),
2372- Row (
2373- 1004 ,
2374- dec(" 1004.00" ),
2375- dec(" 1004.000000000000" ),
2376- Map (" k" -> dec(" 1004.00" )),
2377- dec(" 1.00" )),
2378- Row (5 , dec(" 5.00" ), dec(" 5.000000000000" ), Map (" k" -> dec(" 5.00" )), dec(" 0.01" )))
2337+ /**
2338+ * Runs `f` with [[CometCodegenSuite.DecimalFunctionCatalog ]] registered as `decfn` and `values`
2339+ * in `t (i INT)`, for the #6425 tests.
2340+ */
2341+ private def withDecimalFunctions (values : Any * )(f : => Unit ): Unit = {
23792342 withSQLConf(
23802343 " spark.sql.catalog.decfn" -> classOf [CometCodegenSuite .DecimalFunctionCatalog ].getName) {
23812344 withTable(" t" ) {
23822345 sql(" CREATE TABLE t (i INT) USING parquet" )
23832346 // One file, so the kernel sees every row in one batch.
23842347 sql(
2385- " INSERT INTO t SELECT /*+ REPARTITION(1) */ * FROM VALUES (3), (-7), (NULL), " +
2386- " (99999999), (100000000), (-99999999), (-100000000), (-1005), (1004), (5) AS v(i)" )
2387- for (ansi <- Seq (" true" , " false" )) {
2388- withSQLConf(SQLConf .ANSI_ENABLED .key -> ansi) {
2389- val df = sql(
2390- " SELECT i, decfn.ns.as_money(i), decfn.ns.as_wide_money(i), " +
2391- " map('k', decfn.ns.as_money(i)), decfn.ns.mills_as_money(i) FROM t" )
2392- assertCodegenRan {
2393- checkSparkAnswerAndOperator(df)
2394- }
2395- checkAnswer(df, expected)
2348+ " INSERT INTO t SELECT /*+ REPARTITION(1) */ * FROM VALUES " +
2349+ values.map(v => s " ( $v) " ).mkString(" , " ) + " AS v(i)" )
2350+ f
2351+ }
2352+ }
2353+ }
2354+
2355+ private def dec (s : String ) = if (s == null ) null else new java.math.BigDecimal (s)
2356+
2357+ test(" decimal results of a DSv2 function are rescaled to the declared type (#6425)" ) {
2358+ // Spark lowers a call to a DSv2 function with an instance `invoke` method to `Invoke`, and one
2359+ // with a static `invoke` to `StaticInvoke`. The dispatcher runs both. `as_money` and
2360+ // `as_wide_money` return `Decimal(i)` at scale 0, one declaring `DECIMAL(10, 2)` and one
2361+ // `DECIMAL(20, 12)`, which covers both of the dispatcher's decimal writers. `static_as_money`
2362+ // is `as_money` with a static `invoke`. Spark's row writer rescales the value with
2363+ // `changePrecision` and writes null when it does not fit: 100000000 and -100000000 have nine
2364+ // integer digits and both types allow eight. Spark adds no overflow check around the call, so
2365+ // that null does not depend on ANSI mode. `map` is itself dispatched, so its value goes
2366+ // through the kernel's map writer. `mills_as_money` returns `i` thousandths, at scale 3, into
2367+ // `DECIMAL(7, 2)`, so the rescale drops a digit and `changePrecision` rounds half up: -1.005
2368+ // becomes -1.01 and 1.004 becomes 1.00. 99999.999 has the five integer digits the type allows,
2369+ // but rounds up to 100000.00, which has six, so it is null.
2370+ //
2371+ // Each case is `(i, as_money, as_wide_money, mills_as_money)`. `static_as_money` and the `map`
2372+ // value match `as_money`.
2373+ val cases = Seq [(Any , String , String , String )](
2374+ (3 , " 3.00" , " 3.000000000000" , " 0.00" ),
2375+ (- 7 , " -7.00" , " -7.000000000000" , " -0.01" ),
2376+ (null , null , null , null ),
2377+ (99999999 , " 99999999.00" , " 99999999.000000000000" , null ),
2378+ (100000000 , null , null , null ),
2379+ (- 99999999 , " -99999999.00" , " -99999999.000000000000" , null ),
2380+ (- 100000000 , null , null , null ),
2381+ (- 1005 , " -1005.00" , " -1005.000000000000" , " -1.01" ),
2382+ (1004 , " 1004.00" , " 1004.000000000000" , " 1.00" ),
2383+ (5 , " 5.00" , " 5.000000000000" , " 0.01" ))
2384+ val expected = cases.map { case (i, money, wide, mills) =>
2385+ Row (i, dec(money), dec(money), dec(wide), Map (" k" -> dec(money)), dec(mills))
2386+ }
2387+ withDecimalFunctions(cases.map(_._1): _* ) {
2388+ for (ansi <- Seq (" true" , " false" )) {
2389+ withSQLConf(SQLConf .ANSI_ENABLED .key -> ansi) {
2390+ val df = sql(
2391+ " SELECT i, decfn.ns.as_money(i), decfn.ns.static_as_money(i), " +
2392+ " decfn.ns.as_wide_money(i), map('k', decfn.ns.as_money(i)), " +
2393+ " decfn.ns.mills_as_money(i) FROM t" )
2394+ assertCodegenRan {
2395+ checkSparkAnswerAndImpl(df, dispatched = Seq (" invoke" , " staticinvoke" ))
23962396 }
2397+ checkAnswer(df, expected)
23972398 }
23982399 }
23992400 }
24002401 }
2402+
2403+ test(" decimals in a DSv2 function's array and struct results are rescaled (#6425)" ) {
2404+ // Each function returns `Decimal(i)`, at scale 0, in every decimal of its result, so the
2405+ // kernel's array and struct writers have to rescale them, as Spark's `UnsafeArrayWriter` and
2406+ // `UnsafeRowWriter` do. `money_array`'s element and `money_struct`'s `m` field are
2407+ // `DECIMAL(10, 2)`, so both are null at 100000000. The writers skip the null check for a
2408+ // non-nullable child, so `non_null_money_array`'s element and `money_struct`'s `non_null_m`
2409+ // field cover that path. They are `DECIMAL(12, 2)`, which holds any `INT`.
2410+ //
2411+ // `array(...)` or `named_struct(...)` around a scalar call would not reach these writers:
2412+ // Comet evaluates both natively, and dispatches only the call.
2413+ withDecimalFunctions(3 , null , 100000000 ) {
2414+ val df = sql(
2415+ " SELECT i, decfn.ns.money_array(i), decfn.ns.non_null_money_array(i), " +
2416+ " decfn.ns.money_struct(i) FROM t" )
2417+ assertCodegenRan {
2418+ checkSparkAnswerAndOperator(df)
2419+ }
2420+ checkAnswer(
2421+ df,
2422+ Seq (
2423+ Row (3 , Seq (dec(" 3.00" )), Seq (dec(" 3.00" )), Row (dec(" 3.00" ), dec(" 3.00" ))),
2424+ Row (null , null , null , null ),
2425+ Row (100000000 , Seq (null ), Seq (dec(" 100000000.00" )), Row (null , dec(" 100000000.00" )))))
2426+ }
2427+ }
2428+
2429+ test(" a non-nullable DSv2 decimal result that does not fit fails, as in Spark (#6425)" ) {
2430+ // `non_null_money` declares a non-nullable `DECIMAL(10, 2)`, and `coalesce` makes its argument
2431+ // non-nullable too, so neither Spark nor the kernel checks the result for null. 100000000 does
2432+ // not fit, and both write null for it anyway. Spark then fails to decode the row, and Comet's
2433+ // native projection rejects the batch. The errors differ, but both engines fail.
2434+ withDecimalFunctions(3 , 100000000 ) {
2435+ val (sparkError, cometError) =
2436+ checkSparkAnswerMaybeThrows(sql(" SELECT decfn.ns.non_null_money(coalesce(i, 0)) FROM t" ))
2437+ assert(sparkError.isDefined, " expected Spark to fail" )
2438+ assert(
2439+ cometError.exists(_.getMessage.contains(" is declared as non-nullable but contains null" )),
2440+ s " expected Comet's native projection to reject the null, got $cometError" )
2441+ }
2442+ }
24012443}
24022444
24032445/**
@@ -2419,15 +2461,25 @@ object CometCodegenSuite {
24192461 }
24202462
24212463 /**
2422- * DSv2 function catalog for the #6425 test. `as_money` declares `DECIMAL(10, 2)` and
2423- * `as_wide_money` declares `DECIMAL(20, 12)`, and both return their argument at scale 0.
2424- * `mills_as_money` declares `DECIMAL(10, 2)` and returns its argument as thousandths .
2464+ * DSv2 function catalog for the #6425 tests. Each function returns its argument as a `Decimal`
2465+ * whose scale need not match the type it declares. `mills_as_money` returns its argument as
2466+ * thousandths, and the rest at scale 0 .
24252467 */
24262468 class DecimalFunctionCatalog extends FunctionCatalog {
2427- private val functions = Map (
2428- " as_money" -> new IntAsDecimalFunction (10 , 2 , valueScale = 0 ),
2429- " as_wide_money" -> new IntAsDecimalFunction (20 , 12 , valueScale = 0 ),
2430- " mills_as_money" -> new IntAsDecimalFunction (10 , 2 , valueScale = 3 ))
2469+ private val money = DecimalType (10 , 2 )
2470+ // Holds any `INT`.
2471+ private val intMoney = DecimalType (12 , 2 )
2472+ private val functions : Map [String , UnboundFunction ] = Map (
2473+ " as_money" -> new IntAsDecimalFunction (money),
2474+ " static_as_money" -> new StaticAsMoneyFunction ,
2475+ " as_wide_money" -> new IntAsDecimalFunction (DecimalType (20 , 12 )),
2476+ " mills_as_money" -> new IntAsDecimalFunction (DecimalType (7 , 2 ), valueScale = 3 ),
2477+ " non_null_money" -> new IntAsDecimalFunction (money, nullable = false ),
2478+ " money_array" -> new IntAsDecimalFunction (ArrayType (money, containsNull = true )),
2479+ " non_null_money_array" ->
2480+ new IntAsDecimalFunction (ArrayType (intMoney, containsNull = false )),
2481+ " money_struct" -> new IntAsDecimalFunction (
2482+ new StructType ().add(" m" , money).add(" non_null_m" , intMoney, nullable = false )))
24312483 private var catalogName : String = _
24322484
24332485 override def initialize (name : String , options : CaseInsensitiveStringMap ): Unit =
@@ -2444,24 +2496,51 @@ object CometCodegenSuite {
24442496
24452497 /**
24462498 * Returns its `INT` argument as the unscaled value of a `Decimal` at `valueScale`, whatever
2447- * scale it declares. `invoke` is an instance method, so Spark lowers a call to `Invoke`. The
2448- * function binds to itself.
2499+ * scale `declared` has. For an array or struct type, every decimal in the result holds that
2500+ * value: the array has one element, and each field of the struct has it. `invoke` is an
2501+ * instance method, so Spark lowers a call to `Invoke`. The function binds to itself.
24492502 */
2450- class IntAsDecimalFunction (precision : Int , scale : Int , valueScale : Int )
2503+ class IntAsDecimalFunction (declared : DataType , valueScale : Int = 0 , nullable : Boolean = true )
24512504 extends UnboundFunction
2452- with ScalarFunction [Decimal ] {
2505+ with ScalarFunction [Any ] {
24532506 override def name (): String = " int_as_decimal"
2454- override def description (): String =
2455- s " int -> decimal( $precision, $scale), at scale $valueScale"
2507+ override def description (): String = s " int -> ${declared.sql}, at scale $valueScale"
24562508 override def bind (inputType : StructType ): BoundFunction = this
24572509 override def inputTypes (): Array [DataType ] = Array (IntegerType )
2458- override def resultType (): DataType = DecimalType (precision, scale)
2459- // Ten digits hold any `INT`.
2460- def invoke (v : Int ): Decimal = Decimal (v.toLong, 10 , valueScale)
2461- override def produceResult (input : InternalRow ): Decimal = invoke(input.getInt(0 ))
2510+ override def resultType (): DataType = declared
2511+ override def isResultNullable (): Boolean = nullable
2512+ def invoke (v : Int ): Any = valueOf(declared, v)
2513+ override def produceResult (input : InternalRow ): Any = invoke(input.getInt(0 ))
2514+
2515+ private def valueOf (dataType : DataType , v : Int ): Any = dataType match {
2516+ // Ten digits hold any `INT`.
2517+ case _ : DecimalType => Decimal (v.toLong, 10 , valueScale)
2518+ case ArrayType (elementType, _) => new GenericArrayData (Array (valueOf(elementType, v)))
2519+ case struct : StructType =>
2520+ new GenericInternalRow (struct.fields.map(f => valueOf(f.dataType, v)))
2521+ }
24622522 }
24632523}
24642524
2525+ /**
2526+ * `as_money` for the #6425 tests, with `invoke` on the companion object. Scala also compiles a
2527+ * top-level companion's methods to static methods on the class, so Spark finds a static `invoke`
2528+ * and lowers a call to `StaticInvoke`.
2529+ */
2530+ class StaticAsMoneyFunction extends UnboundFunction with ScalarFunction [Decimal ] {
2531+ override def name (): String = " static_as_money"
2532+ override def description (): String = " int -> decimal(10, 2), at scale 0"
2533+ override def bind (inputType : StructType ): BoundFunction = this
2534+ override def inputTypes (): Array [DataType ] = Array (IntegerType )
2535+ override def resultType (): DataType = DecimalType (10 , 2 )
2536+ override def produceResult (input : InternalRow ): Decimal =
2537+ StaticAsMoneyFunction .invoke(input.getInt(0 ))
2538+ }
2539+
2540+ object StaticAsMoneyFunction {
2541+ def invoke (v : Int ): Decimal = Decimal (v)
2542+ }
2543+
24652544/**
24662545 * Case class used by the struct-input / struct-output smoke tests. Must be declared at file scope
24672546 * (not inside the test class) so Spark's TypeTag-based UDF encoder can resolve the Spark
0 commit comments