Skip to content

Commit 5d07da3

Browse files
committed
fix: preserve DSv2 decimal values through transitive consumers
1 parent d6f88f8 commit 5d07da3

2 files changed

Lines changed: 61 additions & 6 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -841,7 +841,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
841841
// declared type, and the kernel has nulled such a value at its own output: Spark counts it
842842
// and takes it as the maximum, and the row writer nulls the result afterwards. An aggregate
843843
// cannot run in the kernel, so it falls back. See [[readsDispatchedDsv2Decimal]].
844-
if (fn.children.exists(isDispatchedDsv2DecimalCall)) {
844+
if (fn.children.exists(_.exists(isDispatchedDsv2DecimalCall))) {
845845
withFallbackReason(
846846
aggExpr,
847847
s"${fn.prettyName} aggregates the decimal result of a DSv2 function, which Spark " +
@@ -1160,8 +1160,8 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
11601160
}
11611161

11621162
/**
1163-
* Whether `expr` takes the decimal result of a DSv2 scalar function call, which the codegen
1164-
* dispatcher runs, as an argument.
1163+
* Whether `expr` consumes a decimal result of a dispatched DSv2 scalar function anywhere in its
1164+
* argument trees, including through intermediate expressions and container access.
11651165
*
11661166
* Spark does not rescale such a result to the type the function declares, or write null when it
11671167
* does not fit, until it writes a row. An expression around the call reads the `Decimal` the
@@ -1170,11 +1170,13 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
11701170
* something else. `IS NULL` of a value that does not fit is false in Spark, a cast to string
11711171
* keeps the function's scale, and `hash` reads the unscaled value at that scale (#6425). So
11721172
* such an expression runs in the same kernel as the call, where Spark's own code reads the
1173-
* value the function returned. `Alias` is skipped because it computes nothing: the call under
1174-
* it is the root, and Spark writes a root as a row.
1173+
* value the function returned. Checking only immediate children would let an intermediate
1174+
* expression, such as `abs(call)` or `call[0]`, normalize the decimal before its parent reads
1175+
* it. `Alias` is skipped because it computes nothing: the call under it is the root, and Spark
1176+
* writes a root as a row.
11751177
*/
11761178
private def readsDispatchedDsv2Decimal(expr: Expression): Boolean =
1177-
!isStructuralExpr(expr) && expr.children.exists(isDispatchedDsv2DecimalCall)
1179+
!isStructuralExpr(expr) && expr.children.exists(_.exists(isDispatchedDsv2DecimalCall))
11781180

11791181
private def isDispatchedDsv2DecimalCall(expr: Expression): Boolean = {
11801182
val dispatchedDsv2Call = expr match {

‎spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2513,6 +2513,59 @@ class CometCodegenSuite
25132513
}
25142514
}
25152515

2516+
test("transitive consumers of DSv2 decimals preserve materialization timing (#6425)") {
2517+
withDecimalFunctions(3, null, 100000000, -100000000) {
2518+
for (ansi <- Seq("true", "false")) {
2519+
withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi) {
2520+
val projected = sql(
2521+
"SELECT i, abs(decfn.ns.as_money(i)) IS NULL, " +
2522+
"abs(decfn.ns.static_as_money(i)) IS NULL, " +
2523+
"decfn.ns.money_array(i)[0] IS NULL, " +
2524+
"decfn.ns.money_struct(i).m IS NULL, " +
2525+
"CAST(abs(decfn.ns.as_money(i)) AS STRING) FROM t")
2526+
assertCodegenRan {
2527+
checkSparkAnswerAndImpl(projected, dispatched = Seq("isnull", "cast"))
2528+
}
2529+
checkAnswer(
2530+
projected,
2531+
Seq(
2532+
Row(3, false, false, false, false, "3"),
2533+
Row(null, true, true, true, true, null),
2534+
Row(100000000, false, false, false, false, "100000000"),
2535+
Row(-100000000, false, false, false, false, "100000000")))
2536+
val filtered = sql("SELECT i FROM t WHERE abs(decfn.ns.as_money(i)) IS NULL")
2537+
checkSparkAnswerAndImpl(filtered, dispatched = Seq("isnull"))
2538+
checkAnswer(filtered, Seq(Row(null)))
2539+
val aggregated = sql(
2540+
"SELECT count(abs(decfn.ns.as_money(i))), " +
2541+
"count(abs(decfn.ns.static_as_money(i))), " +
2542+
"count(decfn.ns.money_array(i)[0]), " +
2543+
"count(decfn.ns.money_struct(i).m), max(abs(decfn.ns.as_money(i))), " +
2544+
"sum(abs(decfn.ns.as_money(i))) FROM t")
2545+
checkSparkAnswerAndFallbackReason(
2546+
aggregated,
2547+
"aggregates the decimal result of a DSv2 function")
2548+
checkAnswer(aggregated, Row(3L, 3L, 3L, 3L, null, dec("200000003.00")))
2549+
withSQLConf(CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "false") {
2550+
checkSparkAnswerAndFallbackReason(
2551+
"SELECT abs(decfn.ns.as_money(i)) IS NULL FROM t",
2552+
"spark.comet.exec.scalaUDF.codegen.enabled=false")
2553+
}
2554+
// A physical write is a real boundary; a view would be collapsed by Catalyst.
2555+
withTable("decimal_materialized") {
2556+
sql(
2557+
"CREATE TABLE decimal_materialized USING parquet AS " +
2558+
"SELECT i, abs(decfn.ns.as_money(i)) AS d FROM t")
2559+
checkSparkAnswerAndOperator("SELECT i, d IS NULL FROM decimal_materialized")
2560+
checkAnswer(
2561+
sql("SELECT i FROM decimal_materialized WHERE d IS NULL"),
2562+
Seq(Row(null), Row(100000000), Row(-100000000)))
2563+
}
2564+
}
2565+
}
2566+
}
2567+
}
2568+
25162569
test(
25172570
"an aggregate over a DSv2 decimal result falls back, as Spark aggregates the value (#6425)") {
25182571
// Spark aggregates the `Decimal` the function returned, and the row writer nulls the result

0 commit comments

Comments
 (0)