@@ -25,14 +25,19 @@ import org.scalatest.exceptions.TestFailedException
2525
2626import org .apache .arrow .vector ._
2727import org .apache .spark .{SparkConf , SparkEnv , TaskContext }
28- import org .apache .spark .sql .CometTestBase
28+ import org .apache .spark .sql .{ CometTestBase , Row }
2929import org .apache .spark .sql .api .java .UDF1
30+ import org .apache .spark .sql .catalyst .InternalRow
31+ import org .apache .spark .sql .catalyst .analysis .NoSuchFunctionException
3032import org .apache .spark .sql .catalyst .expressions .{Add , Alias , AttributeReference , BoundReference , Cast , CreateArray , CreateMap , CreateNamedStruct , Expression , Hypot , Literal , MapConcat }
3133import org .apache .spark .sql .catalyst .expressions .objects .Invoke
3234import org .apache .spark .sql .comet .CometProjectExec
35+ import org .apache .spark .sql .connector .catalog .{FunctionCatalog , Identifier }
36+ import org .apache .spark .sql .connector .catalog .functions .{BoundFunction , ScalarFunction , UnboundFunction }
3337import org .apache .spark .sql .execution .adaptive .AdaptiveSparkPlanHelper
3438import org .apache .spark .sql .internal .SQLConf
3539import org .apache .spark .sql .types ._
40+ import org .apache .spark .sql .util .CaseInsensitiveStringMap
3641import org .apache .spark .unsafe .types .UTF8String
3742
3843import org .apache .comet .CometSparkSessionExtensions .isSpark41Plus
@@ -2327,6 +2332,48 @@ class CometCodegenSuite
23272332 Invoke (target, " twice" , StringType , Seq (Literal (UTF8String .fromString(" ab" ), StringType )))
23282333 assert(runKernel(folded, 1 )(_.getUTF8String(0 ).toString) === " abab" )
23292334 }
2335+
2336+ test(" decimal results of a DSv2 function are rescaled to the declared type (#6425)" ) {
2337+ // Spark lowers a call to a DSv2 function with an instance `invoke` method to `Invoke`, which
2338+ // the dispatcher runs. Both functions return `Decimal(i)` at scale 0, one declaring
2339+ // `DECIMAL(10, 2)` and one `DECIMAL(20, 12)`, which covers both of the dispatcher's decimal
2340+ // writers. Spark's row writer rescales the value with `changePrecision` and writes null when
2341+ // it does not fit: 100000000 has nine integer digits and both types allow eight. Spark adds
2342+ // no overflow check around the call, so that null does not depend on ANSI mode. `map` is
2343+ // itself dispatched, so its value exercises the nested writer.
2344+ def dec (s : String ) = new java.math.BigDecimal (s)
2345+ val expected = Seq (
2346+ Row (3 , dec(" 3.00" ), dec(" 3.000000000000" ), Map (" k" -> dec(" 3.00" ))),
2347+ Row (- 7 , dec(" -7.00" ), dec(" -7.000000000000" ), Map (" k" -> dec(" -7.00" ))),
2348+ Row (null , null , null , Map (" k" -> null )),
2349+ Row (
2350+ 99999999 ,
2351+ dec(" 99999999.00" ),
2352+ dec(" 99999999.000000000000" ),
2353+ Map (" k" -> dec(" 99999999.00" ))),
2354+ Row (100000000 , null , null , Map (" k" -> null )))
2355+ withSQLConf(
2356+ " spark.sql.catalog.decfn" -> classOf [CometCodegenSuite .DecimalFunctionCatalog ].getName) {
2357+ withTable(" t" ) {
2358+ sql(" CREATE TABLE t (i INT) USING parquet" )
2359+ // One file, so the kernel sees every row in one batch.
2360+ sql(
2361+ " INSERT INTO t SELECT /*+ REPARTITION(1) */ * FROM " +
2362+ " VALUES (3), (-7), (NULL), (99999999), (100000000) AS v(i)" )
2363+ for (ansi <- Seq (" true" , " false" )) {
2364+ withSQLConf(SQLConf .ANSI_ENABLED .key -> ansi) {
2365+ val df = sql(
2366+ " SELECT i, decfn.ns.as_money(i), decfn.ns.as_wide_money(i), " +
2367+ " map('k', decfn.ns.as_money(i)) FROM t" )
2368+ assertCodegenRan {
2369+ checkSparkAnswerAndOperator(df)
2370+ }
2371+ checkAnswer(df, expected)
2372+ }
2373+ }
2374+ }
2375+ }
2376+ }
23302377}
23312378
23322379/**
@@ -2346,6 +2393,44 @@ object CometCodegenSuite {
23462393 class NotSerializableTarget {
23472394 def twice (s : UTF8String ): UTF8String = UTF8String .fromString(s.toString + s.toString)
23482395 }
2396+
2397+ /**
2398+ * DSv2 function catalog for the #6425 test. `as_money` declares `DECIMAL(10, 2)` and
2399+ * `as_wide_money` declares `DECIMAL(20, 12)`.
2400+ */
2401+ class DecimalFunctionCatalog extends FunctionCatalog {
2402+ private val functions = Map (
2403+ " as_money" -> new ScaleZeroDecimalFunction (10 , 2 ),
2404+ " as_wide_money" -> new ScaleZeroDecimalFunction (20 , 12 ))
2405+ private var catalogName : String = _
2406+
2407+ override def initialize (name : String , options : CaseInsensitiveStringMap ): Unit =
2408+ catalogName = name
2409+
2410+ override def name (): String = catalogName
2411+
2412+ override def listFunctions (namespace : Array [String ]): Array [Identifier ] =
2413+ functions.keys.map(Identifier .of(namespace, _)).toArray
2414+
2415+ override def loadFunction (ident : Identifier ): UnboundFunction =
2416+ functions.getOrElse(ident.name(), throw new NoSuchFunctionException (ident))
2417+ }
2418+
2419+ /**
2420+ * Returns its `INT` argument as `Decimal(v)`, at scale 0, whatever scale it declares. `invoke`
2421+ * is an instance method, so Spark lowers a call to `Invoke`. The function binds to itself.
2422+ */
2423+ class ScaleZeroDecimalFunction (precision : Int , scale : Int )
2424+ extends UnboundFunction
2425+ with ScalarFunction [Decimal ] {
2426+ override def name (): String = " scale_zero_decimal"
2427+ override def description (): String = s " int -> decimal( $precision, $scale), at scale 0 "
2428+ override def bind (inputType : StructType ): BoundFunction = this
2429+ override def inputTypes (): Array [DataType ] = Array (IntegerType )
2430+ override def resultType (): DataType = DecimalType (precision, scale)
2431+ def invoke (v : Int ): Decimal = Decimal (v)
2432+ override def produceResult (input : InternalRow ): Decimal = invoke(input.getInt(0 ))
2433+ }
23492434}
23502435
23512436/**
0 commit comments