Skip to content

Commit 04492d8

Browse files
committed
test: cover StaticInvoke, nested writers and non-nullable overflow in the decimal rescale
Add a StaticInvoke variant of as_money, DSv2 functions that return arrays and structs of decimals with nullable and non-nullable children, and a test that a non-nullable result that does not fit fails in both Spark and Comet. Declare mills_as_money as DECIMAL(7, 2) so that 99999.999 rounds up past the precision. Note in the writer why the decimal branch has no null test.
1 parent 4cf7c1c commit 04492d8

2 files changed

Lines changed: 154 additions & 73 deletions

File tree

‎spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenOutput.scala‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,9 @@ private[codegen] object CometBatchKernelCodegenOutput extends CometTypeShim {
236236
// scale)`. A Spark expression already produces its declared precision and scale, but a
237237
// DSv2 function called through `Invoke` / `StaticInvoke` can return a `Decimal` of any
238238
// scale (#6425). Like Spark's writers, this rescales the value in place, and leaves it
239-
// untouched when it does not fit.
239+
// untouched when it does not fit. Unlike them, it does not test `source` for null: the
240+
// callers write null values themselves, and skip that test only for a type that is not
241+
// nullable.
240242
//
241243
// The precision and scale test repeats `changePrecision`'s own fast path. It keeps the call
242244
// off the common path, so the JIT can still scalar-replace the `Decimal` that an input

‎spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala‎

Lines changed: 151 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,9 @@ import org.apache.spark.sql.{CometTestBase, Row}
2929
import org.apache.spark.sql.api.java.UDF1
3030
import org.apache.spark.sql.catalyst.InternalRow
3131
import org.apache.spark.sql.catalyst.analysis.NoSuchFunctionException
32-
import org.apache.spark.sql.catalyst.expressions.{Add, Alias, AttributeReference, BoundReference, Cast, CreateArray, CreateMap, CreateNamedStruct, Expression, Hypot, Literal, MapConcat}
32+
import org.apache.spark.sql.catalyst.expressions.{Add, Alias, AttributeReference, BoundReference, Cast, CreateArray, CreateMap, CreateNamedStruct, Expression, GenericInternalRow, Hypot, Literal, MapConcat}
3333
import org.apache.spark.sql.catalyst.expressions.objects.Invoke
34+
import org.apache.spark.sql.catalyst.util.GenericArrayData
3435
import org.apache.spark.sql.comet.CometProjectExec
3536
import org.apache.spark.sql.connector.catalog.{FunctionCatalog, Identifier}
3637
import org.apache.spark.sql.connector.catalog.functions.{BoundFunction, ScalarFunction, UnboundFunction}
@@ -2333,71 +2334,112 @@ class CometCodegenSuite
23332334
assert(runKernel(folded, 1)(_.getUTF8String(0).toString) === "abab")
23342335
}
23352336

2336-
test("decimal results of a DSv2 function are rescaled to the declared type (#6425)") {
2337-
// Spark lowers a call to a DSv2 function with an instance `invoke` method to `Invoke`, which
2338-
// the dispatcher runs. `as_money` and `as_wide_money` return `Decimal(i)` at scale 0, one
2339-
// declaring `DECIMAL(10, 2)` and one `DECIMAL(20, 12)`, which covers both of the dispatcher's
2340-
// decimal writers. Spark's row writer rescales the value with `changePrecision` and writes
2341-
// null when it does not fit: 100000000 and -100000000 have nine integer digits and both types
2342-
// allow eight. Spark adds no overflow check around the call, so that null does not depend on
2343-
// ANSI mode. `map` is itself dispatched, so its value exercises the nested writer.
2344-
// `mills_as_money` returns `i` thousandths, at scale 3, into `DECIMAL(10, 2)`, so the rescale
2345-
// drops a digit and `changePrecision` rounds half up: -1.005 becomes -1.01 and 1.004 becomes
2346-
// 1.00.
2347-
def dec(s: String) = new java.math.BigDecimal(s)
2348-
val expected = Seq(
2349-
Row(3, dec("3.00"), dec("3.000000000000"), Map("k" -> dec("3.00")), dec("0.00")),
2350-
Row(-7, dec("-7.00"), dec("-7.000000000000"), Map("k" -> dec("-7.00")), dec("-0.01")),
2351-
Row(null, null, null, Map("k" -> null), null),
2352-
Row(
2353-
99999999,
2354-
dec("99999999.00"),
2355-
dec("99999999.000000000000"),
2356-
Map("k" -> dec("99999999.00")),
2357-
dec("100000.00")),
2358-
Row(100000000, null, null, Map("k" -> null), dec("100000.00")),
2359-
Row(
2360-
-99999999,
2361-
dec("-99999999.00"),
2362-
dec("-99999999.000000000000"),
2363-
Map("k" -> dec("-99999999.00")),
2364-
dec("-100000.00")),
2365-
Row(-100000000, null, null, Map("k" -> null), dec("-100000.00")),
2366-
Row(
2367-
-1005,
2368-
dec("-1005.00"),
2369-
dec("-1005.000000000000"),
2370-
Map("k" -> dec("-1005.00")),
2371-
dec("-1.01")),
2372-
Row(
2373-
1004,
2374-
dec("1004.00"),
2375-
dec("1004.000000000000"),
2376-
Map("k" -> dec("1004.00")),
2377-
dec("1.00")),
2378-
Row(5, dec("5.00"), dec("5.000000000000"), Map("k" -> dec("5.00")), dec("0.01")))
2337+
/**
2338+
* Runs `f` with [[CometCodegenSuite.DecimalFunctionCatalog]] registered as `decfn` and `values`
2339+
* in `t (i INT)`, for the #6425 tests.
2340+
*/
2341+
private def withDecimalFunctions(values: Any*)(f: => Unit): Unit = {
23792342
withSQLConf(
23802343
"spark.sql.catalog.decfn" -> classOf[CometCodegenSuite.DecimalFunctionCatalog].getName) {
23812344
withTable("t") {
23822345
sql("CREATE TABLE t (i INT) USING parquet")
23832346
// One file, so the kernel sees every row in one batch.
23842347
sql(
2385-
"INSERT INTO t SELECT /*+ REPARTITION(1) */ * FROM VALUES (3), (-7), (NULL), " +
2386-
"(99999999), (100000000), (-99999999), (-100000000), (-1005), (1004), (5) AS v(i)")
2387-
for (ansi <- Seq("true", "false")) {
2388-
withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi) {
2389-
val df = sql(
2390-
"SELECT i, decfn.ns.as_money(i), decfn.ns.as_wide_money(i), " +
2391-
"map('k', decfn.ns.as_money(i)), decfn.ns.mills_as_money(i) FROM t")
2392-
assertCodegenRan {
2393-
checkSparkAnswerAndOperator(df)
2394-
}
2395-
checkAnswer(df, expected)
2348+
"INSERT INTO t SELECT /*+ REPARTITION(1) */ * FROM VALUES " +
2349+
values.map(v => s"($v)").mkString(", ") + " AS v(i)")
2350+
f
2351+
}
2352+
}
2353+
}
2354+
2355+
private def dec(s: String) = if (s == null) null else new java.math.BigDecimal(s)
2356+
2357+
test("decimal results of a DSv2 function are rescaled to the declared type (#6425)") {
2358+
// Spark lowers a call to a DSv2 function with an instance `invoke` method to `Invoke`, and one
2359+
// with a static `invoke` to `StaticInvoke`. The dispatcher runs both. `as_money` and
2360+
// `as_wide_money` return `Decimal(i)` at scale 0, one declaring `DECIMAL(10, 2)` and one
2361+
// `DECIMAL(20, 12)`, which covers both of the dispatcher's decimal writers. `static_as_money`
2362+
// is `as_money` with a static `invoke`. Spark's row writer rescales the value with
2363+
// `changePrecision` and writes null when it does not fit: 100000000 and -100000000 have nine
2364+
// integer digits and both types allow eight. Spark adds no overflow check around the call, so
2365+
// that null does not depend on ANSI mode. `map` is itself dispatched, so its value goes
2366+
// through the kernel's map writer. `mills_as_money` returns `i` thousandths, at scale 3, into
2367+
// `DECIMAL(7, 2)`, so the rescale drops a digit and `changePrecision` rounds half up: -1.005
2368+
// becomes -1.01 and 1.004 becomes 1.00. 99999.999 has the five integer digits the type allows,
2369+
// but rounds up to 100000.00, which has six, so it is null.
2370+
//
2371+
// Each case is `(i, as_money, as_wide_money, mills_as_money)`. `static_as_money` and the `map`
2372+
// value match `as_money`.
2373+
val cases = Seq[(Any, String, String, String)](
2374+
(3, "3.00", "3.000000000000", "0.00"),
2375+
(-7, "-7.00", "-7.000000000000", "-0.01"),
2376+
(null, null, null, null),
2377+
(99999999, "99999999.00", "99999999.000000000000", null),
2378+
(100000000, null, null, null),
2379+
(-99999999, "-99999999.00", "-99999999.000000000000", null),
2380+
(-100000000, null, null, null),
2381+
(-1005, "-1005.00", "-1005.000000000000", "-1.01"),
2382+
(1004, "1004.00", "1004.000000000000", "1.00"),
2383+
(5, "5.00", "5.000000000000", "0.01"))
2384+
val expected = cases.map { case (i, money, wide, mills) =>
2385+
Row(i, dec(money), dec(money), dec(wide), Map("k" -> dec(money)), dec(mills))
2386+
}
2387+
withDecimalFunctions(cases.map(_._1): _*) {
2388+
for (ansi <- Seq("true", "false")) {
2389+
withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi) {
2390+
val df = sql(
2391+
"SELECT i, decfn.ns.as_money(i), decfn.ns.static_as_money(i), " +
2392+
"decfn.ns.as_wide_money(i), map('k', decfn.ns.as_money(i)), " +
2393+
"decfn.ns.mills_as_money(i) FROM t")
2394+
assertCodegenRan {
2395+
checkSparkAnswerAndImpl(df, dispatched = Seq("invoke", "staticinvoke"))
23962396
}
2397+
checkAnswer(df, expected)
23972398
}
23982399
}
23992400
}
24002401
}
2402+
2403+
test("decimals in a DSv2 function's array and struct results are rescaled (#6425)") {
2404+
// Each function returns `Decimal(i)`, at scale 0, in every decimal of its result, so the
2405+
// kernel's array and struct writers have to rescale them, as Spark's `UnsafeArrayWriter` and
2406+
// `UnsafeRowWriter` do. `money_array`'s element and `money_struct`'s `m` field are
2407+
// `DECIMAL(10, 2)`, so both are null at 100000000. The writers skip the null check for a
2408+
// non-nullable child, so `non_null_money_array`'s element and `money_struct`'s `non_null_m`
2409+
// field cover that path. They are `DECIMAL(12, 2)`, which holds any `INT`.
2410+
//
2411+
// `array(...)` or `named_struct(...)` around a scalar call would not reach these writers:
2412+
// Comet evaluates both natively, and dispatches only the call.
2413+
withDecimalFunctions(3, null, 100000000) {
2414+
val df = sql(
2415+
"SELECT i, decfn.ns.money_array(i), decfn.ns.non_null_money_array(i), " +
2416+
"decfn.ns.money_struct(i) FROM t")
2417+
assertCodegenRan {
2418+
checkSparkAnswerAndOperator(df)
2419+
}
2420+
checkAnswer(
2421+
df,
2422+
Seq(
2423+
Row(3, Seq(dec("3.00")), Seq(dec("3.00")), Row(dec("3.00"), dec("3.00"))),
2424+
Row(null, null, null, null),
2425+
Row(100000000, Seq(null), Seq(dec("100000000.00")), Row(null, dec("100000000.00")))))
2426+
}
2427+
}
2428+
2429+
test("a non-nullable DSv2 decimal result that does not fit fails, as in Spark (#6425)") {
2430+
// `non_null_money` declares a non-nullable `DECIMAL(10, 2)`, and `coalesce` makes its argument
2431+
// non-nullable too, so neither Spark nor the kernel checks the result for null. 100000000 does
2432+
// not fit, and both write null for it anyway. Spark then fails to decode the row, and Comet's
2433+
// native projection rejects the batch. The errors differ, but both engines fail.
2434+
withDecimalFunctions(3, 100000000) {
2435+
val (sparkError, cometError) =
2436+
checkSparkAnswerMaybeThrows(sql("SELECT decfn.ns.non_null_money(coalesce(i, 0)) FROM t"))
2437+
assert(sparkError.isDefined, "expected Spark to fail")
2438+
assert(
2439+
cometError.exists(_.getMessage.contains("is declared as non-nullable but contains null")),
2440+
s"expected Comet's native projection to reject the null, got $cometError")
2441+
}
2442+
}
24012443
}
24022444

24032445
/**
@@ -2419,15 +2461,25 @@ object CometCodegenSuite {
24192461
}
24202462

24212463
/**
2422-
* DSv2 function catalog for the #6425 test. `as_money` declares `DECIMAL(10, 2)` and
2423-
* `as_wide_money` declares `DECIMAL(20, 12)`, and both return their argument at scale 0.
2424-
* `mills_as_money` declares `DECIMAL(10, 2)` and returns its argument as thousandths.
2464+
* DSv2 function catalog for the #6425 tests. Each function returns its argument as a `Decimal`
2465+
* whose scale need not match the type it declares. `mills_as_money` returns its argument as
2466+
* thousandths, and the rest at scale 0.
24252467
*/
24262468
class DecimalFunctionCatalog extends FunctionCatalog {
2427-
private val functions = Map(
2428-
"as_money" -> new IntAsDecimalFunction(10, 2, valueScale = 0),
2429-
"as_wide_money" -> new IntAsDecimalFunction(20, 12, valueScale = 0),
2430-
"mills_as_money" -> new IntAsDecimalFunction(10, 2, valueScale = 3))
2469+
private val money = DecimalType(10, 2)
2470+
// Holds any `INT`.
2471+
private val intMoney = DecimalType(12, 2)
2472+
private val functions: Map[String, UnboundFunction] = Map(
2473+
"as_money" -> new IntAsDecimalFunction(money),
2474+
"static_as_money" -> new StaticAsMoneyFunction,
2475+
"as_wide_money" -> new IntAsDecimalFunction(DecimalType(20, 12)),
2476+
"mills_as_money" -> new IntAsDecimalFunction(DecimalType(7, 2), valueScale = 3),
2477+
"non_null_money" -> new IntAsDecimalFunction(money, nullable = false),
2478+
"money_array" -> new IntAsDecimalFunction(ArrayType(money, containsNull = true)),
2479+
"non_null_money_array" ->
2480+
new IntAsDecimalFunction(ArrayType(intMoney, containsNull = false)),
2481+
"money_struct" -> new IntAsDecimalFunction(
2482+
new StructType().add("m", money).add("non_null_m", intMoney, nullable = false)))
24312483
private var catalogName: String = _
24322484

24332485
override def initialize(name: String, options: CaseInsensitiveStringMap): Unit =
@@ -2444,24 +2496,51 @@ object CometCodegenSuite {
24442496

24452497
/**
24462498
* Returns its `INT` argument as the unscaled value of a `Decimal` at `valueScale`, whatever
2447-
* scale it declares. `invoke` is an instance method, so Spark lowers a call to `Invoke`. The
2448-
* function binds to itself.
2499+
* scale `declared` has. For an array or struct type, every decimal in the result holds that
2500+
* value: the array has one element, and each field of the struct has it. `invoke` is an
2501+
* instance method, so Spark lowers a call to `Invoke`. The function binds to itself.
24492502
*/
2450-
class IntAsDecimalFunction(precision: Int, scale: Int, valueScale: Int)
2503+
class IntAsDecimalFunction(declared: DataType, valueScale: Int = 0, nullable: Boolean = true)
24512504
extends UnboundFunction
2452-
with ScalarFunction[Decimal] {
2505+
with ScalarFunction[Any] {
24532506
override def name(): String = "int_as_decimal"
2454-
override def description(): String =
2455-
s"int -> decimal($precision, $scale), at scale $valueScale"
2507+
override def description(): String = s"int -> ${declared.sql}, at scale $valueScale"
24562508
override def bind(inputType: StructType): BoundFunction = this
24572509
override def inputTypes(): Array[DataType] = Array(IntegerType)
2458-
override def resultType(): DataType = DecimalType(precision, scale)
2459-
// Ten digits hold any `INT`.
2460-
def invoke(v: Int): Decimal = Decimal(v.toLong, 10, valueScale)
2461-
override def produceResult(input: InternalRow): Decimal = invoke(input.getInt(0))
2510+
override def resultType(): DataType = declared
2511+
override def isResultNullable(): Boolean = nullable
2512+
def invoke(v: Int): Any = valueOf(declared, v)
2513+
override def produceResult(input: InternalRow): Any = invoke(input.getInt(0))
2514+
2515+
private def valueOf(dataType: DataType, v: Int): Any = dataType match {
2516+
// Ten digits hold any `INT`.
2517+
case _: DecimalType => Decimal(v.toLong, 10, valueScale)
2518+
case ArrayType(elementType, _) => new GenericArrayData(Array(valueOf(elementType, v)))
2519+
case struct: StructType =>
2520+
new GenericInternalRow(struct.fields.map(f => valueOf(f.dataType, v)))
2521+
}
24622522
}
24632523
}
24642524

2525+
/**
2526+
* `as_money` for the #6425 tests, with `invoke` on the companion object. Scala also compiles a
2527+
* top-level companion's methods to static methods on the class, so Spark finds a static `invoke`
2528+
* and lowers a call to `StaticInvoke`.
2529+
*/
2530+
class StaticAsMoneyFunction extends UnboundFunction with ScalarFunction[Decimal] {
2531+
override def name(): String = "static_as_money"
2532+
override def description(): String = "int -> decimal(10, 2), at scale 0"
2533+
override def bind(inputType: StructType): BoundFunction = this
2534+
override def inputTypes(): Array[DataType] = Array(IntegerType)
2535+
override def resultType(): DataType = DecimalType(10, 2)
2536+
override def produceResult(input: InternalRow): Decimal =
2537+
StaticAsMoneyFunction.invoke(input.getInt(0))
2538+
}
2539+
2540+
object StaticAsMoneyFunction {
2541+
def invoke(v: Int): Decimal = Decimal(v)
2542+
}
2543+
24652544
/**
24662545
* Case class used by the struct-input / struct-output smoke tests. Must be declared at file scope
24672546
* (not inside the test class) so Spark's TypeTag-based UDF encoder can resolve the Spark

0 commit comments

Comments
 (0)