Skip to content

Commit c655d63

Browse files
committed
fix: preserve DSv2 decimal aliases across physical operators
1 parent 5d07da3 commit c655d63

3 files changed

Lines changed: 121 additions & 3 deletions

File tree

‎spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ package org.apache.comet.rules
2222
import scala.collection.mutable.ListBuffer
2323

2424
import org.apache.spark.sql.SparkSession
25-
import org.apache.spark.sql.catalyst.expressions.{Divide, DoubleLiteral, EqualNullSafe, EqualTo, Expression, FloatLiteral, GreaterThan, GreaterThanOrEqual, KnownFloatingPointNormalized, LessThan, LessThanOrEqual, NamedExpression, Remainder}
25+
import org.apache.spark.sql.catalyst.expressions.{Divide, DoubleLiteral, EqualNullSafe, EqualTo, Expression, ExprId, FloatLiteral, GreaterThan, GreaterThanOrEqual, KnownFloatingPointNormalized, LessThan, LessThanOrEqual, NamedExpression, Remainder}
2626
import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateMode, Final, Partial, PartialMerge}
2727
import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero
2828
import org.apache.spark.sql.catalyst.rules.Rule
@@ -79,6 +79,10 @@ object CometExecRule {
7979
val COMET_UNSAFE_PARTIAL: TreeNodeTag[String] =
8080
TreeNodeTag[String]("comet.unsafePartialAgg")
8181

82+
/** Keep a DSv2 decimal producer and its consumers within Spark's row-materialization path. */
83+
val COMET_UNMATERIALIZED_DECIMAL: TreeNodeTag[Unit] =
84+
TreeNodeTag[Unit]("comet.unmaterializedDecimal")
85+
8286
/**
8387
* Fully native operators.
8488
*/
@@ -351,6 +355,12 @@ case class CometExecRule(session: SparkSession)
351355
// spotless:on
352356
private def transform(plan: SparkPlan): SparkPlan = {
353357
def convertNode(op: SparkPlan): SparkPlan = op match {
358+
case op if op.getTagValue(CometExecRule.COMET_UNMATERIALIZED_DECIMAL).isDefined =>
359+
withFallbackReason(
360+
op,
361+
"DSv2 decimal projection and its consumers must share Spark materialization")
362+
op
363+
354364
// Scan marker produced by an optional, out-of-tree scan contrib (e.g. contrib/delta).
355365
// Matched by trait (no compile-time dependency on the contrib) and present only when that
356366
// contrib is on the classpath. The marker carries its own serde handler and typically wraps
@@ -704,6 +714,55 @@ case class CometExecRule(session: SparkSession)
704714
}
705715
}
706716

717+
/**
718+
* Spark can pass a Decimal between fused operators without writing a row. A retained Project
719+
* therefore cannot write a dispatched DSv2 decimal to Arrow before its consumers run: that
720+
* would rescale/null the value early. Follow projected attributes and keep both ends, including
721+
* intervening operators, in Spark. Declining only the consumer would leave the early write.
722+
*
723+
* Paths are propagated conservatively through decimal outputs, including row boundaries. This
724+
* may retain more Spark operators after a real materialization, but never introduces a new one
725+
* before a consumer. Unrelated inputs, and trees that consume the call within one expression,
726+
* remain eligible for native execution.
727+
*/
728+
private def tagUnmaterializedDecimals(plan: SparkPlan): Unit = {
729+
type Paths = Map[ExprId, Set[SparkPlan]]
730+
def visit(op: SparkPlan): Paths = {
731+
val inputs = op.children.flatMap(visit).groupBy(_._1).map { case (id, paths) =>
732+
id -> paths.flatMap(_._2).toSet
733+
}
734+
val consumed = op.references.toSeq.flatMap(a => inputs.getOrElse(a.exprId, Set.empty)).toSet
735+
if (consumed.nonEmpty) {
736+
(consumed + op).foreach(_.setTagValue(CometExecRule.COMET_UNMATERIALIZED_DECIMAL, ()))
737+
}
738+
val sources = op match {
739+
case p: ProjectExec =>
740+
p.projectList
741+
.filter(QueryPlanSerde.producesUnmaterializedDsv2Decimal)
742+
.map(_.exprId)
743+
.toSet
744+
case _ => Set.empty[ExprId]
745+
}
746+
op.output.flatMap { attr =>
747+
val inherited = inputs.getOrElse(attr.exprId, Set.empty)
748+
// Computed decimal outputs can retain the raw value as well (e.g. abs(d), or max(d)).
749+
val derived = if (containsDecimal(attr.dataType)) consumed else Set.empty[SparkPlan]
750+
val path = inherited ++ derived
751+
if (sources.contains(attr.exprId) || path.nonEmpty) Some(attr.exprId -> (path + op))
752+
else None
753+
}.toMap
754+
}
755+
visit(plan)
756+
}
757+
758+
private def containsDecimal(dataType: DataType): Boolean = dataType match {
759+
case _: DecimalType => true
760+
case ArrayType(elementType, _) => containsDecimal(elementType)
761+
case MapType(keyType, valueType, _) => containsDecimal(keyType) || containsDecimal(valueType)
762+
case StructType(fields) => fields.exists(f => containsDecimal(f.dataType))
763+
case _ => false
764+
}
765+
707766
private def normalizePlan(plan: SparkPlan): SparkPlan = {
708767
plan.transformUp {
709768
case p: ProjectExec =>
@@ -797,6 +856,7 @@ case class CometExecRule(session: SparkSession)
797856
// corresponding Final or PartialMerge cannot be converted and the intermediate buffer
798857
// formats are incompatible. This runs before transform() so the tags are checked
799858
// during the bottom-up conversion. Tags persist through AQE stage creation.
859+
tagUnmaterializedDecimals(planWithJoinRewritten)
800860
tagUnsafePartialAggregates(planWithJoinRewritten)
801861

802862
var newPlan = revertUnsafePartialAggregates(transform(planWithJoinRewritten))

‎spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1172,12 +1172,16 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
11721172
* such an expression runs in the same kernel as the call, where Spark's own code reads the
11731173
* value the function returned. Checking only immediate children would let an intermediate
11741174
* expression, such as `abs(call)` or `call[0]`, normalize the decimal before its parent reads
1175-
* it. `Alias` is skipped because it computes nothing: the call under it is the root, and Spark
1176-
* writes a root as a row.
1175+
* it. `Alias` is skipped because it computes nothing. A projected root is not necessarily a
1176+
* row-materialization boundary: CometExecRule keeps retained decimal aliases and their
1177+
* consumers in Spark until those consumers have read the original value.
11771178
*/
11781179
private def readsDispatchedDsv2Decimal(expr: Expression): Boolean =
11791180
!isStructuralExpr(expr) && expr.children.exists(_.exists(isDispatchedDsv2DecimalCall))
11801181

1182+
private[comet] def producesUnmaterializedDsv2Decimal(expr: Expression): Boolean =
1183+
containsDecimal(expr.dataType) && expr.exists(isDispatchedDsv2DecimalCall)
1184+
11811185
private def isDispatchedDsv2DecimalCall(expr: Expression): Boolean = {
11821186
val dispatchedDsv2Call = expr match {
11831187
case i: Invoke =>

‎spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2566,6 +2566,60 @@ class CometCodegenSuite
25662566
}
25672567
}
25682568

2569+
test("retained DSv2 decimal aliases preserve Spark materialization timing (#6425)") {
2570+
withDecimalFunctions(3, null, 100000000, -100000000) {
2571+
val reason = "DSv2 decimal projection and its consumers must share Spark materialization"
2572+
for (ansi <- Seq("true", "false"); aqe <- Seq("true", "false")) {
2573+
withSQLConf(
2574+
SQLConf.ANSI_ENABLED.key -> ansi,
2575+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> aqe) {
2576+
for (expr <- Seq(
2577+
"decfn.ns.as_money(i)",
2578+
"decfn.ns.static_as_money(i)",
2579+
"abs(decfn.ns.as_money(i))")) {
2580+
val from = s"FROM (SELECT $expr AS d FROM t)"
2581+
val projected = sql(s"SELECT d, d IS NULL $from")
2582+
checkSparkAnswerAndFallbackReason(projected, reason)
2583+
checkAnswer(
2584+
projected,
2585+
Seq(Row(dec("3.00"), false), Row(null, true), Row(null, false), Row(null, false)))
2586+
// The retained inner projection must also fall back. A Spark consumer above a
2587+
// Comet producer would still read the prematurely normalized Arrow value.
2588+
assert(
2589+
!projected.queryExecution.executedPlan.exists(_.isInstanceOf[CometProjectExec]))
2590+
val aggregated = sql(s"SELECT count(d), max(d) $from")
2591+
checkSparkAnswerAndFallbackReason(aggregated, reason)
2592+
assert(aggregated.collect().head.getLong(0) == 3L)
2593+
checkSparkAnswerAndFallbackReason(
2594+
s"SELECT d, d IS NULL $from WHERE d IS NOT NULL",
2595+
reason)
2596+
checkSparkAnswerAndFallbackReason(
2597+
s"SELECT d, e, e IS NULL FROM (SELECT d, abs(d) AS e $from)",
2598+
reason)
2599+
}
2600+
for ((call, access) <- Seq("money_array" -> "d[0]", "money_struct" -> "d.m")) {
2601+
checkSparkAnswerAndFallbackReason(
2602+
s"SELECT d, $access IS NULL FROM (SELECT decfn.ns.$call(i) AS d FROM t)",
2603+
reason)
2604+
checkSparkAnswerAndFallbackReason(
2605+
s"SELECT count($access), max($access) FROM (SELECT decfn.ns.$call(i) AS d FROM t)",
2606+
reason)
2607+
}
2608+
checkSparkAnswerAndFallbackReason(
2609+
"SELECT d, CAST(d AS STRING) FROM (SELECT decfn.ns.mills_as_money(i) AS d FROM t)",
2610+
reason)
2611+
// Without fusion, Spark materializes each Project. Retaining the same operators also
2612+
// preserves that earlier boundary, rather than always exposing the raw Decimal.
2613+
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") {
2614+
checkSparkAnswerAndFallbackReason(
2615+
"SELECT d, d IS NULL FROM (SELECT decfn.ns.as_money(i) AS d FROM t)",
2616+
reason)
2617+
}
2618+
}
2619+
}
2620+
}
2621+
}
2622+
25692623
test(
25702624
"an aggregate over a DSv2 decimal result falls back, as Spark aggregates the value (#6425)") {
25712625
// Spark aggregates the `Decimal` the function returned, and the row writer nulls the result

0 commit comments

Comments
 (0)