Skip to content

Commit 4cf7c1c

Browse files
committed
test: cover rounding and the negative overflow boundary in the dispatcher decimal rescale
Add a function that returns thousandths into DECIMAL(10, 2), so the rescale has to round half up, and rows at -99999999 and -100000000 for the negative side of the overflow boundary.
1 parent ee26712 commit 4cf7c1c

1 file changed

Lines changed: 52 additions & 23 deletions

File tree

‎spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala‎

Lines changed: 52 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2335,36 +2335,60 @@ class CometCodegenSuite
23352335

23362336
test("decimal results of a DSv2 function are rescaled to the declared type (#6425)") {
23372337
// Spark lowers a call to a DSv2 function with an instance `invoke` method to `Invoke`, which
2338-
// the dispatcher runs. Both functions return `Decimal(i)` at scale 0, one declaring
2339-
// `DECIMAL(10, 2)` and one `DECIMAL(20, 12)`, which covers both of the dispatcher's decimal
2340-
// writers. Spark's row writer rescales the value with `changePrecision` and writes null when
2341-
// it does not fit: 100000000 has nine integer digits and both types allow eight. Spark adds
2342-
// no overflow check around the call, so that null does not depend on ANSI mode. `map` is
2343-
// itself dispatched, so its value exercises the nested writer.
2338+
// the dispatcher runs. `as_money` and `as_wide_money` return `Decimal(i)` at scale 0, one
2339+
// declaring `DECIMAL(10, 2)` and one `DECIMAL(20, 12)`, which covers both of the dispatcher's
2340+
// decimal writers. Spark's row writer rescales the value with `changePrecision` and writes
2341+
// null when it does not fit: 100000000 and -100000000 have nine integer digits and both types
2342+
// allow eight. Spark adds no overflow check around the call, so that null does not depend on
2343+
// ANSI mode. `map` is itself dispatched, so its value exercises the nested writer.
2344+
// `mills_as_money` returns `i` thousandths, at scale 3, into `DECIMAL(10, 2)`, so the rescale
2345+
// drops a digit and `changePrecision` rounds half up: -1.005 becomes -1.01 and 1.004 becomes
2346+
// 1.00.
23442347
def dec(s: String) = new java.math.BigDecimal(s)
23452348
val expected = Seq(
2346-
Row(3, dec("3.00"), dec("3.000000000000"), Map("k" -> dec("3.00"))),
2347-
Row(-7, dec("-7.00"), dec("-7.000000000000"), Map("k" -> dec("-7.00"))),
2348-
Row(null, null, null, Map("k" -> null)),
2349+
Row(3, dec("3.00"), dec("3.000000000000"), Map("k" -> dec("3.00")), dec("0.00")),
2350+
Row(-7, dec("-7.00"), dec("-7.000000000000"), Map("k" -> dec("-7.00")), dec("-0.01")),
2351+
Row(null, null, null, Map("k" -> null), null),
23492352
Row(
23502353
99999999,
23512354
dec("99999999.00"),
23522355
dec("99999999.000000000000"),
2353-
Map("k" -> dec("99999999.00"))),
2354-
Row(100000000, null, null, Map("k" -> null)))
2356+
Map("k" -> dec("99999999.00")),
2357+
dec("100000.00")),
2358+
Row(100000000, null, null, Map("k" -> null), dec("100000.00")),
2359+
Row(
2360+
-99999999,
2361+
dec("-99999999.00"),
2362+
dec("-99999999.000000000000"),
2363+
Map("k" -> dec("-99999999.00")),
2364+
dec("-100000.00")),
2365+
Row(-100000000, null, null, Map("k" -> null), dec("-100000.00")),
2366+
Row(
2367+
-1005,
2368+
dec("-1005.00"),
2369+
dec("-1005.000000000000"),
2370+
Map("k" -> dec("-1005.00")),
2371+
dec("-1.01")),
2372+
Row(
2373+
1004,
2374+
dec("1004.00"),
2375+
dec("1004.000000000000"),
2376+
Map("k" -> dec("1004.00")),
2377+
dec("1.00")),
2378+
Row(5, dec("5.00"), dec("5.000000000000"), Map("k" -> dec("5.00")), dec("0.01")))
23552379
withSQLConf(
23562380
"spark.sql.catalog.decfn" -> classOf[CometCodegenSuite.DecimalFunctionCatalog].getName) {
23572381
withTable("t") {
23582382
sql("CREATE TABLE t (i INT) USING parquet")
23592383
// One file, so the kernel sees every row in one batch.
23602384
sql(
2361-
"INSERT INTO t SELECT /*+ REPARTITION(1) */ * FROM " +
2362-
"VALUES (3), (-7), (NULL), (99999999), (100000000) AS v(i)")
2385+
"INSERT INTO t SELECT /*+ REPARTITION(1) */ * FROM VALUES (3), (-7), (NULL), " +
2386+
"(99999999), (100000000), (-99999999), (-100000000), (-1005), (1004), (5) AS v(i)")
23632387
for (ansi <- Seq("true", "false")) {
23642388
withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi) {
23652389
val df = sql(
23662390
"SELECT i, decfn.ns.as_money(i), decfn.ns.as_wide_money(i), " +
2367-
"map('k', decfn.ns.as_money(i)) FROM t")
2391+
"map('k', decfn.ns.as_money(i)), decfn.ns.mills_as_money(i) FROM t")
23682392
assertCodegenRan {
23692393
checkSparkAnswerAndOperator(df)
23702394
}
@@ -2396,12 +2420,14 @@ object CometCodegenSuite {
23962420

23972421
/**
23982422
* DSv2 function catalog for the #6425 test. `as_money` declares `DECIMAL(10, 2)` and
2399-
* `as_wide_money` declares `DECIMAL(20, 12)`.
2423+
* `as_wide_money` declares `DECIMAL(20, 12)`, and both return their argument at scale 0.
2424+
* `mills_as_money` declares `DECIMAL(10, 2)` and returns its argument as thousandths.
24002425
*/
24012426
class DecimalFunctionCatalog extends FunctionCatalog {
24022427
private val functions = Map(
2403-
"as_money" -> new ScaleZeroDecimalFunction(10, 2),
2404-
"as_wide_money" -> new ScaleZeroDecimalFunction(20, 12))
2428+
"as_money" -> new IntAsDecimalFunction(10, 2, valueScale = 0),
2429+
"as_wide_money" -> new IntAsDecimalFunction(20, 12, valueScale = 0),
2430+
"mills_as_money" -> new IntAsDecimalFunction(10, 2, valueScale = 3))
24052431
private var catalogName: String = _
24062432

24072433
override def initialize(name: String, options: CaseInsensitiveStringMap): Unit =
@@ -2417,18 +2443,21 @@ object CometCodegenSuite {
24172443
}
24182444

24192445
/**
2420-
* Returns its `INT` argument as `Decimal(v)`, at scale 0, whatever scale it declares. `invoke`
2421-
* is an instance method, so Spark lowers a call to `Invoke`. The function binds to itself.
2446+
* Returns its `INT` argument as the unscaled value of a `Decimal` at `valueScale`, whatever
2447+
* scale it declares. `invoke` is an instance method, so Spark lowers a call to `Invoke`. The
2448+
* function binds to itself.
24222449
*/
2423-
class ScaleZeroDecimalFunction(precision: Int, scale: Int)
2450+
class IntAsDecimalFunction(precision: Int, scale: Int, valueScale: Int)
24242451
extends UnboundFunction
24252452
with ScalarFunction[Decimal] {
2426-
override def name(): String = "scale_zero_decimal"
2427-
override def description(): String = s"int -> decimal($precision, $scale), at scale 0"
2453+
override def name(): String = "int_as_decimal"
2454+
override def description(): String =
2455+
s"int -> decimal($precision, $scale), at scale $valueScale"
24282456
override def bind(inputType: StructType): BoundFunction = this
24292457
override def inputTypes(): Array[DataType] = Array(IntegerType)
24302458
override def resultType(): DataType = DecimalType(precision, scale)
2431-
def invoke(v: Int): Decimal = Decimal(v)
2459+
// Ten digits hold any `INT`.
2460+
def invoke(v: Int): Decimal = Decimal(v.toLong, 10, valueScale)
24322461
override def produceResult(input: InternalRow): Decimal = invoke(input.getInt(0))
24332462
}
24342463
}

0 commit comments

Comments
 (0)