@@ -25,6 +25,7 @@ import scala.collection.mutable.ListBuffer
2525import scala .concurrent .Future
2626import scala .jdk .CollectionConverters ._
2727
28+ import org .apache .arrow .c .ArrowArrayStream
2829import org .apache .spark ._
2930import org .apache .spark .internal .config
3031import org .apache .spark .rdd .RDD
@@ -36,7 +37,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.LazilyGeneratedOrdering
3637import org .apache .spark .sql .catalyst .plans .logical .Statistics
3738import org .apache .spark .sql .catalyst .plans .physical ._
3839import org .apache .spark .sql .comet .{CometFilterExec , CometMetricNode , CometNativeExec , CometNativeScanExec , CometPlan , CometProjectExec , CometSinkPlaceHolder , CometSparkToColumnarExec , NativeExecContext }
39- import org .apache .spark .sql .comet .execution .arrow .CometArrowStream
40+ import org .apache .spark .sql .comet .execution .arrow .{ CometArrowStream , CometNativeArrowSource }
4041import org .apache .spark .sql .execution ._
4142import org .apache .spark .sql .execution .adaptive .ShuffleQueryStageExec
4243import org .apache .spark .sql .execution .exchange .{ENSURE_REQUIREMENTS , ShuffleExchangeExec , ShuffleExchangeLike , ShuffleOrigin }
@@ -145,9 +146,16 @@ case class CometShuffleExchangeExec(
145146 ctx.perPartitionByKey,
146147 positionalRoundRobin.isDefined)
147148 case None =>
148- // Non-native child (e.g. CometSparkToColumnarExec): no subtree to inline. The dep gets
149- // built via the convenience overload below; we just need a real RDD of batches.
150- child.executeColumnar()
149+ child match {
150+ // Native reads the source's Arrow stream, as it does for a native operator's input. The
151+ // batches from the source's `executeColumnar` share vectors that it reuses for the next
152+ // batch, and the convenience overload below closes each batch once native has it.
153+ // Closing a struct vector drops its children, so the next batch would lose them.
154+ case source : CometNativeArrowSource => source.doExecuteAsArrowStream()
155+ // Other non-native child: no subtree to inline. The dep gets built via the convenience
156+ // overload below; we just need a real RDD of batches.
157+ case _ => child.executeColumnar()
158+ }
151159 }
152160 } else if (shuffleType == CometColumnarShuffle ) {
153161 // Row-based shuffle. CometNativeExec.doExecute wraps columnar output with
@@ -228,12 +236,27 @@ case class CometShuffleExchangeExec(
228236 ctx,
229237 positionalRoundRobin))
230238 case None =>
231- CometShuffleExchangeExec .prepareShuffleDependency(
232- inputRDD.asInstanceOf [RDD [ColumnarBatch ]],
233- child.output,
234- outputPartitioning,
235- serializer,
236- metrics)
239+ child match {
240+ case _ : CometNativeArrowSource =>
241+ CometShuffleExchangeExec .prepareArrowStreamShuffleDependency(
242+ inputRDD.asInstanceOf [RDD [ArrowArrayStream ]],
243+ // The range partitioner samples rows, so it needs them as batches.
244+ outputPartitioning match {
245+ case _ : RangePartitioning => Some (child.executeColumnar())
246+ case _ => None
247+ },
248+ child.output,
249+ outputPartitioning,
250+ serializer,
251+ metrics)
252+ case _ =>
253+ CometShuffleExchangeExec .prepareShuffleDependency(
254+ inputRDD.asInstanceOf [RDD [ColumnarBatch ]],
255+ child.output,
256+ outputPartitioning,
257+ serializer,
258+ metrics)
259+ }
237260 }
238261 metrics(" numPartitions" ).set(dep.partitioner.numPartitions.toLong)
239262 val executionId = sparkContext.getLocalProperty(SQLExecution .EXECUTION_ID_KEY )
@@ -879,6 +902,36 @@ object CometShuffleExchangeExec
879902 outputPartitioning : Partitioning ,
880903 serializer : Serializer ,
881904 metrics : Map [String , SQLMetric ]): ShuffleDependency [Int , ColumnarBatch , ColumnarBatch ] = {
905+ // Wrap the raw batches as an RDD[ArrowArrayStream] so the leaf reaches native via the Arrow C
906+ // Stream Interface, matching how CometNativeExec.buildNativeContext feeds the native-child
907+ // path.
908+ val streamRDD = CometArrowStream .wrapColumnarBatchRDD(
909+ rdd,
910+ StructType (
911+ outputAttributes.map(a => StructField (a.name, a.dataType, a.nullable, a.metadata))),
912+ CometArrowStream .NATIVE_TIMEZONE ,
913+ " ShuffleWriterInput" )
914+ prepareArrowStreamShuffleDependency(
915+ streamRDD,
916+ Some (rdd),
917+ outputAttributes,
918+ outputPartitioning,
919+ serializer,
920+ metrics)
921+ }
922+
923+ /**
924+ * [[prepareShuffleDependency ]] for input that native reads as one Arrow stream per partition,
925+ * such as a [[CometNativeArrowSource ]]'s. `samplingRDD` gives the same rows as batches, and is
926+ * only required for [[RangePartitioning ]].
927+ */
928+ private def prepareArrowStreamShuffleDependency (
929+ streamRDD : RDD [ArrowArrayStream ],
930+ samplingRDD : Option [RDD [ColumnarBatch ]],
931+ outputAttributes : Seq [Attribute ],
932+ outputPartitioning : Partitioning ,
933+ serializer : Serializer ,
934+ metrics : Map [String , SQLMetric ]): ShuffleDependency [Int , ColumnarBatch , ColumnarBatch ] = {
882935
883936 val scanBuilder = OperatorOuterClass .Scan .newBuilder().setSource(" ShuffleWriterInput" )
884937 val scanTypes = outputAttributes.flatMap { attr =>
@@ -889,29 +942,20 @@ object CometShuffleExchangeExec
889942 s " $outputAttributes contains unsupported data types for CometShuffleExchangeExec. " )
890943 }
891944 scanBuilder.addAllFields(scanTypes.asJava)
945+ // The native consumer of the stream.
892946 val scanOp = OperatorOuterClass .Operator .newBuilder().setScan(scanBuilder).build()
893947
894- // Wrap the raw batches as an RDD[ArrowArrayStream] so the leaf reaches native via the Arrow C
895- // Stream Interface, matching how CometNativeExec.buildNativeContext feeds the native-child
896- // path. The synthetic Scan("ShuffleWriterInput") above is the native consumer.
897- val streamRDD = CometArrowStream .wrapColumnarBatchRDD(
898- rdd,
899- StructType (
900- outputAttributes.map(a => StructField (a.name, a.dataType, a.nullable, a.metadata))),
901- CometArrowStream .NATIVE_TIMEZONE ,
902- " ShuffleWriterInput" )
903-
904948 val childMetricNode = CometMetricNode (Map .empty)
905949 val thinRDD = new CometNativeShuffleInputRDD (
906- rdd .sparkContext,
950+ streamRDD .sparkContext,
907951 Seq (streamRDD),
908- rdd .getNumPartitions,
952+ streamRDD .getNumPartitions,
909953 shuffleScanIndices = Set .empty,
910954 spillMetricNode = CometMetricNode (metrics, Seq (childMetricNode)))
911955
912956 val ctx = NativeExecContext (
913957 inputs = Seq (streamRDD),
914- numPartitions = rdd .getNumPartitions,
958+ numPartitions = streamRDD .getNumPartitions,
915959 subqueries = Seq .empty,
916960 broadcastedHadoopConfForEncryption = None ,
917961 encryptedFilePaths = Seq .empty,
@@ -924,7 +968,7 @@ object CometShuffleExchangeExec
924968 // is `shuffleWriterMetrics` at the root with one empty leaf for the Scan child.
925969 prepareNativeShuffleDependency(
926970 thinRDD,
927- Some (rdd) ,
971+ samplingRDD ,
928972 outputAttributes,
929973 outputPartitioning,
930974 serializer,
0 commit comments