Skip to content

Commit 84944e9

Browse files
authored
fix: copy rows retained by identity broadcasts (#7260)
1 parent 4e54f1c commit 84944e9

2 files changed

Lines changed: 35 additions & 4 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import org.apache.spark.sql.catalyst.InternalRow
3232
import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder}
3333
import org.apache.spark.sql.catalyst.expressions.codegen._
3434
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
35-
import org.apache.spark.sql.catalyst.plans.physical.Partitioning
35+
import org.apache.spark.sql.catalyst.plans.physical.{IdentityBroadcastMode, Partitioning}
3636
import org.apache.spark.sql.comet.util.{Utils => CometUtils}
3737
import org.apache.spark.sql.errors.QueryExecutionErrors
3838
import org.apache.spark.sql.execution.{CodegenSupport, ColumnarToRowTransition, SparkPlan, SQLExecution}
@@ -137,7 +137,13 @@ case class CometColumnarToRowExec(child: SparkPlan)
137137
numOutputRows)
138138

139139
val mode = cometBroadcastExchange.get.mode
140-
val relation = mode.transform(rows, Some(numOutputRows.value))
140+
// Identity broadcasts retain rows, but both JVM conversion paths reuse the returned row.
141+
// Hashed relations copy row bytes into their own storage while consuming the iterator.
142+
val broadcastRows = mode match {
143+
case IdentityBroadcastMode => rows.map(_.copy())
144+
case _ => rows
145+
}
146+
val relation = mode.transform(broadcastRows, Some(numOutputRows.value))
141147
val broadcasted = sparkContext.broadcastInternal(relation, serializedOnly = true)
142148
val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
143149
SQLMetrics.postDriverMetricUpdates(sparkContext, executionId, metrics.values.toSeq)

‎spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,13 @@ import org.scalatest.PrivateMethodTester._
2828

2929
import org.apache.logging.log4j.Level
3030
import org.apache.spark.sql._
31-
import org.apache.spark.sql.catalyst.FunctionIdentifier
31+
import org.apache.spark.sql.catalyst.{FunctionIdentifier, InternalRow}
3232
import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, Descending, Expression, ExpressionInfo, GreaterThan, In, InSet, KnownFloatingPointNormalized, Literal, Not, RuntimeReplaceable, SortOrder}
3333
import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, BloomFilterAggregate, Final, Min, Partial, PartialMerge}
3434
import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, StaticInvoke}
3535
import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero
3636
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
37-
import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, RoundRobinPartitioning}
37+
import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, IdentityBroadcastMode, RoundRobinPartitioning}
3838
import org.apache.spark.sql.catalyst.rules.Rule
3939
import org.apache.spark.sql.comet._
4040
import org.apache.spark.sql.comet.CometExec.nativeLeaves
@@ -98,6 +98,31 @@ class CometExecRuleSuite extends CometTestBase {
9898
CometExecRule(spark).apply(stripAQEPlan(plan))
9999
}
100100

101+
test("columnar-to-row identity broadcasts retain every build row") {
102+
for (direct <- Seq(false, true)) {
103+
withSQLConf(
104+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
105+
SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false",
106+
CometConf.COMET_EXEC_RANGE_ENABLED.key -> "true",
107+
CometConf.COMET_DIRECT_COLUMNAR_TO_ROW_ENABLED.key -> direct.toString,
108+
CometConf.COMET_DIRECT_COLUMNAR_TO_ROW_MIN_BATCH_SIZE.key -> "1") {
109+
val build = applyCometExecRule(
110+
createSparkPlan(spark, "SELECT CAST(id AS INT) AS y FROM range(2024, 2028, 1, 2)"))
111+
assert(build.isInstanceOf[CometNativeExec], build.treeString)
112+
val mode = IdentityBroadcastMode
113+
val original = BroadcastExchangeExec(mode, build)
114+
val exchange = CometBroadcastExchangeExec(original, build.output, mode, build)
115+
val rows = CometColumnarToRowExec(exchange)
116+
.executeBroadcast[Array[InternalRow]]()
117+
.value
118+
// Two rows per partition expose row reuse in both JVM conversion paths.
119+
withClue(s"direct=$direct: ") {
120+
assert(rows.map(_.getInt(0)).sorted.toSeq == Seq(2024, 2025, 2026, 2027))
121+
}
122+
}
123+
}
124+
}
125+
101126
/** Create a test data frame that is used in all tests */
102127
private def createTestDataFrame = {
103128
val testSchema = new StructType(

0 commit comments

Comments
 (0)