Skip to content

Commit 76a0e1c

Browse files
committed
fix: preserve DSv2 decimal aliases across physical operators
(cherry picked from commit c655d63)
1 parent 66ec696 commit 76a0e1c

3 files changed

Lines changed: 121 additions & 3 deletions

File tree

‎spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ package org.apache.comet.rules
2222
import scala.collection.mutable.ListBuffer
2323

2424
import org.apache.spark.sql.SparkSession
25-
import org.apache.spark.sql.catalyst.expressions.{Divide, DoubleLiteral, EqualNullSafe, EqualTo, Expression, FloatLiteral, GreaterThan, GreaterThanOrEqual, KnownFloatingPointNormalized, LessThan, LessThanOrEqual, NamedExpression, Remainder}
25+
import org.apache.spark.sql.catalyst.expressions.{Divide, DoubleLiteral, EqualNullSafe, EqualTo, Expression, ExprId, FloatLiteral, GreaterThan, GreaterThanOrEqual, KnownFloatingPointNormalized, LessThan, LessThanOrEqual, NamedExpression, Remainder}
2626
import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateMode, Final, Partial, PartialMerge}
2727
import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero
2828
import org.apache.spark.sql.catalyst.rules.Rule
@@ -70,6 +70,10 @@ object CometExecRule {
7070
val COMET_UNSAFE_PARTIAL: TreeNodeTag[String] =
7171
TreeNodeTag[String]("comet.unsafePartialAgg")
7272

73+
/** Keep a DSv2 decimal producer and its consumers within Spark's row-materialization path. */
74+
val COMET_UNMATERIALIZED_DECIMAL: TreeNodeTag[Unit] =
75+
TreeNodeTag[Unit]("comet.unmaterializedDecimal")
76+
7377
/**
7478
* Fully native operators.
7579
*/
@@ -339,6 +343,12 @@ case class CometExecRule(session: SparkSession)
339343
// spotless:on
340344
private def transform(plan: SparkPlan): SparkPlan = {
341345
def convertNode(op: SparkPlan): SparkPlan = op match {
346+
case op if op.getTagValue(CometExecRule.COMET_UNMATERIALIZED_DECIMAL).isDefined =>
347+
withFallbackReason(
348+
op,
349+
"DSv2 decimal projection and its consumers must share Spark materialization")
350+
op
351+
342352
// Scan marker produced by an optional, out-of-tree scan contrib (e.g. contrib/delta).
343353
// Matched by trait (no compile-time dependency on the contrib) and present only when that
344354
// contrib is on the classpath. The marker carries its own serde handler and typically wraps
@@ -681,6 +691,55 @@ case class CometExecRule(session: SparkSession)
681691
}
682692
}
683693

694+
/**
695+
* Spark can pass a Decimal between fused operators without writing a row. A retained Project
696+
* therefore cannot write a dispatched DSv2 decimal to Arrow before its consumers run: that
697+
* would rescale/null the value early. Follow projected attributes and keep both ends, including
698+
* intervening operators, in Spark. Declining only the consumer would leave the early write.
699+
*
700+
* Paths are propagated conservatively through decimal outputs, including row boundaries. This
701+
* may retain more Spark operators after a real materialization, but never introduces a new one
702+
* before a consumer. Unrelated inputs, and trees that consume the call within one expression,
703+
* remain eligible for native execution.
704+
*/
705+
private def tagUnmaterializedDecimals(plan: SparkPlan): Unit = {
706+
type Paths = Map[ExprId, Set[SparkPlan]]
707+
def visit(op: SparkPlan): Paths = {
708+
val inputs = op.children.flatMap(visit).groupBy(_._1).map { case (id, paths) =>
709+
id -> paths.flatMap(_._2).toSet
710+
}
711+
val consumed = op.references.toSeq.flatMap(a => inputs.getOrElse(a.exprId, Set.empty)).toSet
712+
if (consumed.nonEmpty) {
713+
(consumed + op).foreach(_.setTagValue(CometExecRule.COMET_UNMATERIALIZED_DECIMAL, ()))
714+
}
715+
val sources = op match {
716+
case p: ProjectExec =>
717+
p.projectList
718+
.filter(QueryPlanSerde.producesUnmaterializedDsv2Decimal)
719+
.map(_.exprId)
720+
.toSet
721+
case _ => Set.empty[ExprId]
722+
}
723+
op.output.flatMap { attr =>
724+
val inherited = inputs.getOrElse(attr.exprId, Set.empty)
725+
// Computed decimal outputs can retain the raw value as well (e.g. abs(d), or max(d)).
726+
val derived = if (containsDecimal(attr.dataType)) consumed else Set.empty[SparkPlan]
727+
val path = inherited ++ derived
728+
if (sources.contains(attr.exprId) || path.nonEmpty) Some(attr.exprId -> (path + op))
729+
else None
730+
}.toMap
731+
}
732+
visit(plan)
733+
}
734+
735+
private def containsDecimal(dataType: DataType): Boolean = dataType match {
736+
case _: DecimalType => true
737+
case ArrayType(elementType, _) => containsDecimal(elementType)
738+
case MapType(keyType, valueType, _) => containsDecimal(keyType) || containsDecimal(valueType)
739+
case StructType(fields) => fields.exists(f => containsDecimal(f.dataType))
740+
case _ => false
741+
}
742+
684743
private def normalizePlan(plan: SparkPlan): SparkPlan = {
685744
plan.transformUp {
686745
case p: ProjectExec =>
@@ -774,6 +833,7 @@ case class CometExecRule(session: SparkSession)
774833
// corresponding Final or PartialMerge cannot be converted and the intermediate buffer
775834
// formats are incompatible. This runs before transform() so the tags are checked
776835
// during the bottom-up conversion. Tags persist through AQE stage creation.
836+
tagUnmaterializedDecimals(planWithJoinRewritten)
777837
tagUnsafePartialAggregates(planWithJoinRewritten)
778838

779839
var newPlan = revertUnsafePartialAggregates(transform(planWithJoinRewritten))

‎spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1156,12 +1156,16 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
11561156
* such an expression runs in the same kernel as the call, where Spark's own code reads the
11571157
* value the function returned. Checking only immediate children would let an intermediate
11581158
* expression, such as `abs(call)` or `call[0]`, normalize the decimal before its parent reads
1159-
* it. `Alias` is skipped because it computes nothing: the call under it is the root, and Spark
1160-
* writes a root as a row.
1159+
* it. `Alias` is skipped because it computes nothing. A projected root is not necessarily a
1160+
* row-materialization boundary: CometExecRule keeps retained decimal aliases and their
1161+
* consumers in Spark until those consumers have read the original value.
11611162
*/
11621163
private def readsDispatchedDsv2Decimal(expr: Expression): Boolean =
11631164
!isStructuralExpr(expr) && expr.children.exists(_.exists(isDispatchedDsv2DecimalCall))
11641165

1166+
private[comet] def producesUnmaterializedDsv2Decimal(expr: Expression): Boolean =
1167+
containsDecimal(expr.dataType) && expr.exists(isDispatchedDsv2DecimalCall)
1168+
11651169
private def isDispatchedDsv2DecimalCall(expr: Expression): Boolean = {
11661170
val dispatchedDsv2Call = expr match {
11671171
case i: Invoke =>

‎spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2539,6 +2539,60 @@ class CometCodegenSuite
25392539
}
25402540
}
25412541

2542+
test("retained DSv2 decimal aliases preserve Spark materialization timing (#6425)") {
2543+
withDecimalFunctions(3, null, 100000000, -100000000) {
2544+
val reason = "DSv2 decimal projection and its consumers must share Spark materialization"
2545+
for (ansi <- Seq("true", "false"); aqe <- Seq("true", "false")) {
2546+
withSQLConf(
2547+
SQLConf.ANSI_ENABLED.key -> ansi,
2548+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> aqe) {
2549+
for (expr <- Seq(
2550+
"decfn.ns.as_money(i)",
2551+
"decfn.ns.static_as_money(i)",
2552+
"abs(decfn.ns.as_money(i))")) {
2553+
val from = s"FROM (SELECT $expr AS d FROM t)"
2554+
val projected = sql(s"SELECT d, d IS NULL $from")
2555+
checkSparkAnswerAndFallbackReason(projected, reason)
2556+
checkAnswer(
2557+
projected,
2558+
Seq(Row(dec("3.00"), false), Row(null, true), Row(null, false), Row(null, false)))
2559+
// The retained inner projection must also fall back. A Spark consumer above a
2560+
// Comet producer would still read the prematurely normalized Arrow value.
2561+
assert(
2562+
!projected.queryExecution.executedPlan.exists(_.isInstanceOf[CometProjectExec]))
2563+
val aggregated = sql(s"SELECT count(d), max(d) $from")
2564+
checkSparkAnswerAndFallbackReason(aggregated, reason)
2565+
assert(aggregated.collect().head.getLong(0) == 3L)
2566+
checkSparkAnswerAndFallbackReason(
2567+
s"SELECT d, d IS NULL $from WHERE d IS NOT NULL",
2568+
reason)
2569+
checkSparkAnswerAndFallbackReason(
2570+
s"SELECT d, e, e IS NULL FROM (SELECT d, abs(d) AS e $from)",
2571+
reason)
2572+
}
2573+
for ((call, access) <- Seq("money_array" -> "d[0]", "money_struct" -> "d.m")) {
2574+
checkSparkAnswerAndFallbackReason(
2575+
s"SELECT d, $access IS NULL FROM (SELECT decfn.ns.$call(i) AS d FROM t)",
2576+
reason)
2577+
checkSparkAnswerAndFallbackReason(
2578+
s"SELECT count($access), max($access) FROM (SELECT decfn.ns.$call(i) AS d FROM t)",
2579+
reason)
2580+
}
2581+
checkSparkAnswerAndFallbackReason(
2582+
"SELECT d, CAST(d AS STRING) FROM (SELECT decfn.ns.mills_as_money(i) AS d FROM t)",
2583+
reason)
2584+
// Without fusion, Spark materializes each Project. Retaining the same operators also
2585+
// preserves that earlier boundary, rather than always exposing the raw Decimal.
2586+
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") {
2587+
checkSparkAnswerAndFallbackReason(
2588+
"SELECT d, d IS NULL FROM (SELECT decfn.ns.as_money(i) AS d FROM t)",
2589+
reason)
2590+
}
2591+
}
2592+
}
2593+
}
2594+
}
2595+
25422596
test(
25432597
"an aggregate over a DSv2 decimal result falls back, as Spark aggregates the value (#6425)") {
25442598
// Spark aggregates the `Decimal` the function returned, and the row writer nulls the result

0 commit comments

Comments
 (0)