Skip to content

Commit 66ec696

Browse files
committed
fix: preserve DSv2 decimal values through transitive consumers
(cherry picked from commit 5d07da3)
1 parent 5c9482c commit 66ec696

2 files changed

Lines changed: 61 additions & 6 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -840,7 +840,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
840840
// declared type, and the kernel has nulled such a value at its own output: Spark counts it
841841
// and takes it as the maximum, and the row writer nulls the result afterwards. An aggregate
842842
// cannot run in the kernel, so it falls back. See [[readsDispatchedDsv2Decimal]].
843-
if (fn.children.exists(isDispatchedDsv2DecimalCall)) {
843+
if (fn.children.exists(_.exists(isDispatchedDsv2DecimalCall))) {
844844
withFallbackReason(
845845
aggExpr,
846846
s"${fn.prettyName} aggregates the decimal result of a DSv2 function, which Spark " +
@@ -1144,8 +1144,8 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
11441144
}
11451145

11461146
/**
1147-
* Whether `expr` takes the decimal result of a DSv2 scalar function call, which the codegen
1148-
* dispatcher runs, as an argument.
1147+
* Whether `expr` consumes a decimal result of a dispatched DSv2 scalar function anywhere in its
1148+
* argument trees, including through intermediate expressions and container access.
11491149
*
11501150
* Spark does not rescale such a result to the type the function declares, or write null when it
11511151
* does not fit, until it writes a row. An expression around the call reads the `Decimal` the
@@ -1154,11 +1154,13 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
11541154
* something else. `IS NULL` of a value that does not fit is false in Spark, a cast to string
11551155
* keeps the function's scale, and `hash` reads the unscaled value at that scale (#6425). So
11561156
* such an expression runs in the same kernel as the call, where Spark's own code reads the
1157-
* value the function returned. `Alias` is skipped because it computes nothing: the call under
1158-
* it is the root, and Spark writes a root as a row.
1157+
* value the function returned. Checking only immediate children would let an intermediate
1158+
* expression, such as `abs(call)` or `call[0]`, normalize the decimal before its parent reads
1159+
* it. `Alias` is skipped because it computes nothing: the call under it is the root, and Spark
1160+
* writes a root as a row.
11591161
*/
11601162
private def readsDispatchedDsv2Decimal(expr: Expression): Boolean =
1161-
!isStructuralExpr(expr) && expr.children.exists(isDispatchedDsv2DecimalCall)
1163+
!isStructuralExpr(expr) && expr.children.exists(_.exists(isDispatchedDsv2DecimalCall))
11621164

11631165
private def isDispatchedDsv2DecimalCall(expr: Expression): Boolean = {
11641166
val dispatchedDsv2Call = expr match {

‎spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2486,6 +2486,59 @@ class CometCodegenSuite
24862486
}
24872487
}
24882488

2489+
test("transitive consumers of DSv2 decimals preserve materialization timing (#6425)") {
2490+
withDecimalFunctions(3, null, 100000000, -100000000) {
2491+
for (ansi <- Seq("true", "false")) {
2492+
withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi) {
2493+
val projected = sql(
2494+
"SELECT i, abs(decfn.ns.as_money(i)) IS NULL, " +
2495+
"abs(decfn.ns.static_as_money(i)) IS NULL, " +
2496+
"decfn.ns.money_array(i)[0] IS NULL, " +
2497+
"decfn.ns.money_struct(i).m IS NULL, " +
2498+
"CAST(abs(decfn.ns.as_money(i)) AS STRING) FROM t")
2499+
assertCodegenRan {
2500+
checkSparkAnswerAndImpl(projected, dispatched = Seq("isnull", "cast"))
2501+
}
2502+
checkAnswer(
2503+
projected,
2504+
Seq(
2505+
Row(3, false, false, false, false, "3"),
2506+
Row(null, true, true, true, true, null),
2507+
Row(100000000, false, false, false, false, "100000000"),
2508+
Row(-100000000, false, false, false, false, "100000000")))
2509+
val filtered = sql("SELECT i FROM t WHERE abs(decfn.ns.as_money(i)) IS NULL")
2510+
checkSparkAnswerAndImpl(filtered, dispatched = Seq("isnull"))
2511+
checkAnswer(filtered, Seq(Row(null)))
2512+
val aggregated = sql(
2513+
"SELECT count(abs(decfn.ns.as_money(i))), " +
2514+
"count(abs(decfn.ns.static_as_money(i))), " +
2515+
"count(decfn.ns.money_array(i)[0]), " +
2516+
"count(decfn.ns.money_struct(i).m), max(abs(decfn.ns.as_money(i))), " +
2517+
"sum(abs(decfn.ns.as_money(i))) FROM t")
2518+
checkSparkAnswerAndFallbackReason(
2519+
aggregated,
2520+
"aggregates the decimal result of a DSv2 function")
2521+
checkAnswer(aggregated, Row(3L, 3L, 3L, 3L, null, dec("200000003.00")))
2522+
withSQLConf(CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "false") {
2523+
checkSparkAnswerAndFallbackReason(
2524+
"SELECT abs(decfn.ns.as_money(i)) IS NULL FROM t",
2525+
"spark.comet.exec.scalaUDF.codegen.enabled=false")
2526+
}
2527+
// A physical write is a real boundary; a view would be collapsed by Catalyst.
2528+
withTable("decimal_materialized") {
2529+
sql(
2530+
"CREATE TABLE decimal_materialized USING parquet AS " +
2531+
"SELECT i, abs(decfn.ns.as_money(i)) AS d FROM t")
2532+
checkSparkAnswerAndOperator("SELECT i, d IS NULL FROM decimal_materialized")
2533+
checkAnswer(
2534+
sql("SELECT i FROM decimal_materialized WHERE d IS NULL"),
2535+
Seq(Row(null), Row(100000000), Row(-100000000)))
2536+
}
2537+
}
2538+
}
2539+
}
2540+
}
2541+
24892542
test(
24902543
"an aggregate over a DSv2 decimal result falls back, as Spark aggregates the value (#6425)") {
24912544
// Spark aggregates the `Decimal` the function returned, and the row writer nulls the result

0 commit comments

Comments
 (0)