@@ -22,7 +22,7 @@ package org.apache.comet.rules
2222import scala .collection .mutable .ListBuffer
2323
2424import org .apache .spark .sql .SparkSession
25- import org .apache .spark .sql .catalyst .expressions .{Divide , DoubleLiteral , EqualNullSafe , EqualTo , Expression , FloatLiteral , GreaterThan , GreaterThanOrEqual , KnownFloatingPointNormalized , LessThan , LessThanOrEqual , NamedExpression , Remainder }
25+ import org .apache .spark .sql .catalyst .expressions .{Divide , DoubleLiteral , EqualNullSafe , EqualTo , Expression , ExprId , FloatLiteral , GreaterThan , GreaterThanOrEqual , KnownFloatingPointNormalized , LessThan , LessThanOrEqual , NamedExpression , Remainder }
2626import org .apache .spark .sql .catalyst .expressions .aggregate .{AggregateMode , Final , Partial , PartialMerge }
2727import org .apache .spark .sql .catalyst .optimizer .NormalizeNaNAndZero
2828import org .apache .spark .sql .catalyst .rules .Rule
@@ -79,6 +79,10 @@ object CometExecRule {
7979 val COMET_UNSAFE_PARTIAL : TreeNodeTag [String ] =
8080 TreeNodeTag [String ](" comet.unsafePartialAgg" )
8181
82+ /** Keep a DSv2 decimal producer and its consumers within Spark's row-materialization path. */
83+ val COMET_UNMATERIALIZED_DECIMAL : TreeNodeTag [Unit ] =
84+ TreeNodeTag [Unit ](" comet.unmaterializedDecimal" )
85+
8286 /**
8387 * Fully native operators.
8488 */
@@ -351,6 +355,12 @@ case class CometExecRule(session: SparkSession)
351355 // spotless:on
352356 private def transform (plan : SparkPlan ): SparkPlan = {
353357 def convertNode (op : SparkPlan ): SparkPlan = op match {
358+ case op if op.getTagValue(CometExecRule .COMET_UNMATERIALIZED_DECIMAL ).isDefined =>
359+ withFallbackReason(
360+ op,
361+ " DSv2 decimal projection and its consumers must share Spark materialization" )
362+ op
363+
354364 // Scan marker produced by an optional, out-of-tree scan contrib (e.g. contrib/delta).
355365 // Matched by trait (no compile-time dependency on the contrib) and present only when that
356366 // contrib is on the classpath. The marker carries its own serde handler and typically wraps
@@ -704,6 +714,55 @@ case class CometExecRule(session: SparkSession)
704714 }
705715 }
706716
717+ /**
718+ * Spark can pass a Decimal between fused operators without writing a row. A retained Project
719+ * therefore cannot write a dispatched DSv2 decimal to Arrow before its consumers run: that
720+ * would rescale/null the value early. Follow projected attributes and keep both ends, including
721+ * intervening operators, in Spark. Declining only the consumer would leave the early write.
722+ *
723+ * Paths are propagated conservatively through decimal outputs, including row boundaries. This
724+ * may retain more Spark operators after a real materialization, but never introduces a new one
725+ * before a consumer. Unrelated inputs, and trees that consume the call within one expression,
726+ * remain eligible for native execution.
727+ */
728+ private def tagUnmaterializedDecimals (plan : SparkPlan ): Unit = {
729+ type Paths = Map [ExprId , Set [SparkPlan ]]
730+ def visit (op : SparkPlan ): Paths = {
731+ val inputs = op.children.flatMap(visit).groupBy(_._1).map { case (id, paths) =>
732+ id -> paths.flatMap(_._2).toSet
733+ }
734+ val consumed = op.references.toSeq.flatMap(a => inputs.getOrElse(a.exprId, Set .empty)).toSet
735+ if (consumed.nonEmpty) {
736+ (consumed + op).foreach(_.setTagValue(CometExecRule .COMET_UNMATERIALIZED_DECIMAL , ()))
737+ }
738+ val sources = op match {
739+ case p : ProjectExec =>
740+ p.projectList
741+ .filter(QueryPlanSerde .producesUnmaterializedDsv2Decimal)
742+ .map(_.exprId)
743+ .toSet
744+ case _ => Set .empty[ExprId ]
745+ }
746+ op.output.flatMap { attr =>
747+ val inherited = inputs.getOrElse(attr.exprId, Set .empty)
748+ // Computed decimal outputs can retain the raw value as well (e.g. abs(d), or max(d)).
749+ val derived = if (containsDecimal(attr.dataType)) consumed else Set .empty[SparkPlan ]
750+ val path = inherited ++ derived
751+ if (sources.contains(attr.exprId) || path.nonEmpty) Some (attr.exprId -> (path + op))
752+ else None
753+ }.toMap
754+ }
755+ visit(plan)
756+ }
757+
758+ private def containsDecimal (dataType : DataType ): Boolean = dataType match {
759+ case _ : DecimalType => true
760+ case ArrayType (elementType, _) => containsDecimal(elementType)
761+ case MapType (keyType, valueType, _) => containsDecimal(keyType) || containsDecimal(valueType)
762+ case StructType (fields) => fields.exists(f => containsDecimal(f.dataType))
763+ case _ => false
764+ }
765+
707766 private def normalizePlan (plan : SparkPlan ): SparkPlan = {
708767 plan.transformUp {
709768 case p : ProjectExec =>
@@ -797,6 +856,7 @@ case class CometExecRule(session: SparkSession)
797856 // corresponding Final or PartialMerge cannot be converted and the intermediate buffer
798857 // formats are incompatible. This runs before transform() so the tags are checked
799858 // during the bottom-up conversion. Tags persist through AQE stage creation.
859+ tagUnmaterializedDecimals(planWithJoinRewritten)
800860 tagUnsafePartialAggregates(planWithJoinRewritten)
801861
802862 var newPlan = revertUnsafePartialAggregates(transform(planWithJoinRewritten))
0 commit comments