Skip to content

Commit 16a1f73

Browse files
committed
fix: have native shuffle read a converted child's Arrow stream
A native shuffle over a CometSparkToColumnarExec wrapped the child's executeColumnar batches in a ColumnarBatchArrowReader, which closes each batch once native has it. Those batches share the vectors that the conversion reuses for the next batch, and closing a struct vector drops its children, so the next batch failed to import. Native shuffle now reads such a child's Arrow stream, as a native operator does, which also fixes the same failure with spark.comet.sparkToColumnar.enabled.
1 parent 9795b61 commit 16a1f73

2 files changed

Lines changed: 88 additions & 24 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala‎

Lines changed: 68 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import scala.collection.mutable.ListBuffer
2525
import scala.concurrent.Future
2626
import scala.jdk.CollectionConverters._
2727

28+
import org.apache.arrow.c.ArrowArrayStream
2829
import org.apache.spark._
2930
import org.apache.spark.internal.config
3031
import org.apache.spark.rdd.RDD
@@ -36,7 +37,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.LazilyGeneratedOrdering
3637
import org.apache.spark.sql.catalyst.plans.logical.Statistics
3738
import org.apache.spark.sql.catalyst.plans.physical._
3839
import org.apache.spark.sql.comet.{CometFilterExec, CometMetricNode, CometNativeExec, CometNativeScanExec, CometPlan, CometProjectExec, CometSinkPlaceHolder, CometSparkToColumnarExec, NativeExecContext}
39-
import org.apache.spark.sql.comet.execution.arrow.CometArrowStream
40+
import org.apache.spark.sql.comet.execution.arrow.{CometArrowStream, CometNativeArrowSource}
4041
import org.apache.spark.sql.execution._
4142
import org.apache.spark.sql.execution.adaptive.ShuffleQueryStageExec
4243
import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, ShuffleExchangeExec, ShuffleExchangeLike, ShuffleOrigin}
@@ -145,9 +146,16 @@ case class CometShuffleExchangeExec(
145146
ctx.perPartitionByKey,
146147
positionalRoundRobin.isDefined)
147148
case None =>
148-
// Non-native child (e.g. CometSparkToColumnarExec): no subtree to inline. The dep gets
149-
// built via the convenience overload below; we just need a real RDD of batches.
150-
child.executeColumnar()
149+
child match {
150+
// Native reads the source's Arrow stream, as it does for a native operator's input. The
151+
// batches from the source's `executeColumnar` share vectors that it reuses for the next
152+
// batch, and the convenience overload below closes each batch once native has it.
153+
// Closing a struct vector drops its children, so the next batch would lose them.
154+
case source: CometNativeArrowSource => source.doExecuteAsArrowStream()
155+
// Other non-native child: no subtree to inline. The dep gets built via the convenience
156+
// overload below; we just need a real RDD of batches.
157+
case _ => child.executeColumnar()
158+
}
151159
}
152160
} else if (shuffleType == CometColumnarShuffle) {
153161
// Row-based shuffle. CometNativeExec.doExecute wraps columnar output with
@@ -228,12 +236,27 @@ case class CometShuffleExchangeExec(
228236
ctx,
229237
positionalRoundRobin))
230238
case None =>
231-
CometShuffleExchangeExec.prepareShuffleDependency(
232-
inputRDD.asInstanceOf[RDD[ColumnarBatch]],
233-
child.output,
234-
outputPartitioning,
235-
serializer,
236-
metrics)
239+
child match {
240+
case _: CometNativeArrowSource =>
241+
CometShuffleExchangeExec.prepareArrowStreamShuffleDependency(
242+
inputRDD.asInstanceOf[RDD[ArrowArrayStream]],
243+
// The range partitioner samples rows, so it needs them as batches.
244+
outputPartitioning match {
245+
case _: RangePartitioning => Some(child.executeColumnar())
246+
case _ => None
247+
},
248+
child.output,
249+
outputPartitioning,
250+
serializer,
251+
metrics)
252+
case _ =>
253+
CometShuffleExchangeExec.prepareShuffleDependency(
254+
inputRDD.asInstanceOf[RDD[ColumnarBatch]],
255+
child.output,
256+
outputPartitioning,
257+
serializer,
258+
metrics)
259+
}
237260
}
238261
metrics("numPartitions").set(dep.partitioner.numPartitions.toLong)
239262
val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
@@ -879,6 +902,36 @@ object CometShuffleExchangeExec
879902
outputPartitioning: Partitioning,
880903
serializer: Serializer,
881904
metrics: Map[String, SQLMetric]): ShuffleDependency[Int, ColumnarBatch, ColumnarBatch] = {
905+
// Wrap the raw batches as an RDD[ArrowArrayStream] so the leaf reaches native via the Arrow C
906+
// Stream Interface, matching how CometNativeExec.buildNativeContext feeds the native-child
907+
// path.
908+
val streamRDD = CometArrowStream.wrapColumnarBatchRDD(
909+
rdd,
910+
StructType(
911+
outputAttributes.map(a => StructField(a.name, a.dataType, a.nullable, a.metadata))),
912+
CometArrowStream.NATIVE_TIMEZONE,
913+
"ShuffleWriterInput")
914+
prepareArrowStreamShuffleDependency(
915+
streamRDD,
916+
Some(rdd),
917+
outputAttributes,
918+
outputPartitioning,
919+
serializer,
920+
metrics)
921+
}
922+
923+
/**
924+
* [[prepareShuffleDependency]] for input that native reads as one Arrow stream per partition,
925+
* such as a [[CometNativeArrowSource]]'s. `samplingRDD` gives the same rows as batches, and is
926+
* only required for [[RangePartitioning]].
927+
*/
928+
private def prepareArrowStreamShuffleDependency(
929+
streamRDD: RDD[ArrowArrayStream],
930+
samplingRDD: Option[RDD[ColumnarBatch]],
931+
outputAttributes: Seq[Attribute],
932+
outputPartitioning: Partitioning,
933+
serializer: Serializer,
934+
metrics: Map[String, SQLMetric]): ShuffleDependency[Int, ColumnarBatch, ColumnarBatch] = {
882935

883936
val scanBuilder = OperatorOuterClass.Scan.newBuilder().setSource("ShuffleWriterInput")
884937
val scanTypes = outputAttributes.flatMap { attr =>
@@ -889,29 +942,20 @@ object CometShuffleExchangeExec
889942
s"$outputAttributes contains unsupported data types for CometShuffleExchangeExec.")
890943
}
891944
scanBuilder.addAllFields(scanTypes.asJava)
945+
// The native consumer of the stream.
892946
val scanOp = OperatorOuterClass.Operator.newBuilder().setScan(scanBuilder).build()
893947

894-
// Wrap the raw batches as an RDD[ArrowArrayStream] so the leaf reaches native via the Arrow C
895-
// Stream Interface, matching how CometNativeExec.buildNativeContext feeds the native-child
896-
// path. The synthetic Scan("ShuffleWriterInput") above is the native consumer.
897-
val streamRDD = CometArrowStream.wrapColumnarBatchRDD(
898-
rdd,
899-
StructType(
900-
outputAttributes.map(a => StructField(a.name, a.dataType, a.nullable, a.metadata))),
901-
CometArrowStream.NATIVE_TIMEZONE,
902-
"ShuffleWriterInput")
903-
904948
val childMetricNode = CometMetricNode(Map.empty)
905949
val thinRDD = new CometNativeShuffleInputRDD(
906-
rdd.sparkContext,
950+
streamRDD.sparkContext,
907951
Seq(streamRDD),
908-
rdd.getNumPartitions,
952+
streamRDD.getNumPartitions,
909953
shuffleScanIndices = Set.empty,
910954
spillMetricNode = CometMetricNode(metrics, Seq(childMetricNode)))
911955

912956
val ctx = NativeExecContext(
913957
inputs = Seq(streamRDD),
914-
numPartitions = rdd.getNumPartitions,
958+
numPartitions = streamRDD.getNumPartitions,
915959
subqueries = Seq.empty,
916960
broadcastedHadoopConfForEncryption = None,
917961
encryptedFilePaths = Seq.empty,
@@ -924,7 +968,7 @@ object CometShuffleExchangeExec
924968
// is `shuffleWriterMetrics` at the root with one empty leaf for the Scan child.
925969
prepareNativeShuffleDependency(
926970
thinRDD,
927-
Some(rdd),
971+
samplingRDD,
928972
outputAttributes,
929973
outputPartitioning,
930974
serializer,

‎spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,26 @@ class CometShuffleInputConversionSuite extends CometTestBase {
240240
}
241241
}
242242

243+
convertTest("struct columns over several batches") {
244+
// The rows of each input partition fill several batches. With the conversion of leaf
245+
// operators on, native shuffle reads the converted scan the same way.
246+
Seq("false", "true").foreach { leafConversion =>
247+
withSQLConf(
248+
CometConf.COMET_BATCH_SIZE.key -> "7",
249+
CometConf.COMET_SPARK_TO_ARROW_ENABLED.key -> leafConversion) {
250+
val schema = new StructType()
251+
.add("k", IntegerType)
252+
.add("payload", new StructType().add("v", LongType).add("s", StringType))
253+
val data = (0 until 24).map(i => Row(i, Row(i.toLong, s"s$i")))
254+
val df = spark.createDataFrame(spark.sparkContext.parallelize(data, 1), schema)
255+
Seq(df.repartition(3, col("k")), df.repartitionByRange(3, col("k"))).foreach { shuffled =>
256+
val (_, plan) = checkSparkAnswer(shuffled)
257+
assert(convertedShuffles(plan).length == 1, s"leaf conversion $leafConversion:\n$plan")
258+
}
259+
}
260+
}
261+
}
262+
243263
convertTest("columns the conversion does not support keep the JVM columnar shuffle") {
244264
val df = arraysDf().repartition(5, col("k")).groupBy("k").agg(sum(size(col("xs"))))
245265
val (_, plan) = checkSparkAnswer(df)

0 commit comments

Comments
 (0)