@@ -21,22 +21,14 @@ package org.apache.spark.sql.comet.execution.shuffle
2121
2222import java .nio .channels .ReadableByteChannel
2323
24- import org .apache .spark .sql .internal .SQLConf
2524import org .apache .spark .sql .vectorized .ColumnarBatch
2625
27- import org .apache .comet .CometConf
28- import org .apache .comet .vector .{NativeUtil , StreamReader }
26+ import org .apache .comet .vector .StreamReader
2927
3028class ArrowReaderIterator (channel : ReadableByteChannel ) extends Iterator [ColumnarBatch ] {
3129
32- private val nativeUtil = new NativeUtil
33-
34- private val maxBatchSize = CometConf .COMET_BATCH_SIZE .get(SQLConf .get)
35-
3630 private val reader = StreamReader (channel)
37- private var currentIdx = - 1
3831 private var batch = nextBatch()
39- private var previousBatch : ColumnarBatch = null
4032 private var currentBatch : ColumnarBatch = null
4133
4234 override def hasNext : Boolean = {
@@ -57,47 +49,28 @@ class ArrowReaderIterator(channel: ReadableByteChannel) extends Iterator[Columna
5749 }
5850
5951 val nextBatch = batch.get
60- val batchRows = nextBatch.numRows()
61- val numRows = Math .min(batchRows - currentIdx, maxBatchSize)
6252
63- // Release the previous sliced batch.
53+ // Release the previous batch.
6454 // If it is not released, when closing the reader, arrow library will complain about
6555 // memory leak.
6656 if (currentBatch != null ) {
67- // Close plain arrays in the previous sliced batch.
68- // The dictionary arrays will be closed when closing the entire batch.
6957 currentBatch.close()
7058 }
7159
72- currentBatch = nativeUtil.takeRows(nextBatch, currentIdx, numRows)
73- currentIdx += numRows
74-
75- if (currentIdx == batchRows) {
76- // We cannot close the batch here, because if there is dictionary array in the batch,
77- // the dictionary array will be closed immediately, and the returned sliced batch will
78- // be invalid.
79- previousBatch = batch.get
80-
81- batch = None
82- currentIdx = - 1
83- }
84-
60+ currentBatch = nextBatch
61+ batch = None
8562 currentBatch
8663 }
8764
8865 private def nextBatch (): Option [ColumnarBatch ] = {
89- if (previousBatch != null ) {
90- previousBatch.close()
91- previousBatch = null
92- }
93- currentIdx = 0
9466 reader.nextBatch()
9567 }
9668
9769 def close (): Unit =
9870 synchronized {
9971 if (currentBatch != null ) {
10072 currentBatch.close()
73+ currentBatch = null
10174 }
10275 reader.close()
10376 }
0 commit comments