@@ -22,7 +22,7 @@ package org.apache.comet.rules
2222import scala .collection .mutable .ListBuffer
2323
2424import org .apache .spark .sql .SparkSession
25- import org .apache .spark .sql .catalyst .expressions .{Divide , DoubleLiteral , EqualNullSafe , EqualTo , Expression , FloatLiteral , GreaterThan , GreaterThanOrEqual , KnownFloatingPointNormalized , LessThan , LessThanOrEqual , NamedExpression , Remainder }
25+ import org .apache .spark .sql .catalyst .expressions .{Divide , DoubleLiteral , EqualNullSafe , EqualTo , Expression , ExprId , FloatLiteral , GreaterThan , GreaterThanOrEqual , KnownFloatingPointNormalized , LessThan , LessThanOrEqual , NamedExpression , Remainder }
2626import org .apache .spark .sql .catalyst .expressions .aggregate .{AggregateMode , Final , Partial , PartialMerge }
2727import org .apache .spark .sql .catalyst .optimizer .NormalizeNaNAndZero
2828import org .apache .spark .sql .catalyst .rules .Rule
@@ -70,6 +70,10 @@ object CometExecRule {
7070 val COMET_UNSAFE_PARTIAL : TreeNodeTag [String ] =
7171 TreeNodeTag [String ](" comet.unsafePartialAgg" )
7272
73+ /** Keep a DSv2 decimal producer and its consumers within Spark's row-materialization path. */
74+ val COMET_UNMATERIALIZED_DECIMAL : TreeNodeTag [Unit ] =
75+ TreeNodeTag [Unit ](" comet.unmaterializedDecimal" )
76+
7377 /**
7478 * Fully native operators.
7579 */
@@ -339,6 +343,12 @@ case class CometExecRule(session: SparkSession)
339343 // spotless:on
340344 private def transform (plan : SparkPlan ): SparkPlan = {
341345 def convertNode (op : SparkPlan ): SparkPlan = op match {
346+ case op if op.getTagValue(CometExecRule .COMET_UNMATERIALIZED_DECIMAL ).isDefined =>
347+ withFallbackReason(
348+ op,
349+ " DSv2 decimal projection and its consumers must share Spark materialization" )
350+ op
351+
342352 // Scan marker produced by an optional, out-of-tree scan contrib (e.g. contrib/delta).
343353 // Matched by trait (no compile-time dependency on the contrib) and present only when that
344354 // contrib is on the classpath. The marker carries its own serde handler and typically wraps
@@ -681,6 +691,55 @@ case class CometExecRule(session: SparkSession)
681691 }
682692 }
683693
694+ /**
695+ * Spark can pass a Decimal between fused operators without writing a row. A retained Project
696+ * therefore cannot write a dispatched DSv2 decimal to Arrow before its consumers run: that
697+ * would rescale/null the value early. Follow projected attributes and keep both ends, including
698+ * intervening operators, in Spark. Declining only the consumer would leave the early write.
699+ *
700+ * Paths are propagated conservatively through decimal outputs, including row boundaries. This
701+ * may retain more Spark operators after a real materialization, but never introduces a new one
702+ * before a consumer. Unrelated inputs, and trees that consume the call within one expression,
703+ * remain eligible for native execution.
704+ */
705+ private def tagUnmaterializedDecimals (plan : SparkPlan ): Unit = {
706+ type Paths = Map [ExprId , Set [SparkPlan ]]
707+ def visit (op : SparkPlan ): Paths = {
708+ val inputs = op.children.flatMap(visit).groupBy(_._1).map { case (id, paths) =>
709+ id -> paths.flatMap(_._2).toSet
710+ }
711+ val consumed = op.references.toSeq.flatMap(a => inputs.getOrElse(a.exprId, Set .empty)).toSet
712+ if (consumed.nonEmpty) {
713+ (consumed + op).foreach(_.setTagValue(CometExecRule .COMET_UNMATERIALIZED_DECIMAL , ()))
714+ }
715+ val sources = op match {
716+ case p : ProjectExec =>
717+ p.projectList
718+ .filter(QueryPlanSerde .producesUnmaterializedDsv2Decimal)
719+ .map(_.exprId)
720+ .toSet
721+ case _ => Set .empty[ExprId ]
722+ }
723+ op.output.flatMap { attr =>
724+ val inherited = inputs.getOrElse(attr.exprId, Set .empty)
725+ // Computed decimal outputs can retain the raw value as well (e.g. abs(d), or max(d)).
726+ val derived = if (containsDecimal(attr.dataType)) consumed else Set .empty[SparkPlan ]
727+ val path = inherited ++ derived
728+ if (sources.contains(attr.exprId) || path.nonEmpty) Some (attr.exprId -> (path + op))
729+ else None
730+ }.toMap
731+ }
732+ visit(plan)
733+ }
734+
735+ private def containsDecimal (dataType : DataType ): Boolean = dataType match {
736+ case _ : DecimalType => true
737+ case ArrayType (elementType, _) => containsDecimal(elementType)
738+ case MapType (keyType, valueType, _) => containsDecimal(keyType) || containsDecimal(valueType)
739+ case StructType (fields) => fields.exists(f => containsDecimal(f.dataType))
740+ case _ => false
741+ }
742+
684743 private def normalizePlan (plan : SparkPlan ): SparkPlan = {
685744 plan.transformUp {
686745 case p : ProjectExec =>
@@ -774,6 +833,7 @@ case class CometExecRule(session: SparkSession)
774833 // corresponding Final or PartialMerge cannot be converted and the intermediate buffer
775834 // formats are incompatible. This runs before transform() so the tags are checked
776835 // during the bottom-up conversion. Tags persist through AQE stage creation.
836+ tagUnmaterializedDecimals(planWithJoinRewritten)
777837 tagUnsafePartialAggregates(planWithJoinRewritten)
778838
779839 var newPlan = revertUnsafePartialAggregates(transform(planWithJoinRewritten))
0 commit comments