Skip to content

Commit 4fec40e

Browse files
authored
refactor: Skipping slicing on shuffle arrays in shuffle reader (apache#189)
* refactor: Skipping slicing on shuffle arrays * Add note for columnar shuffle batch size.
1 parent 7ba69d8 commit 4fec40e

2 files changed

Lines changed: 8 additions & 33 deletions

File tree

‎common/src/main/scala/org/apache/comet/CometConf.scala‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,9 @@ object CometConf {
227227
val COMET_COLUMNAR_SHUFFLE_BATCH_SIZE: ConfigEntry[Int] =
228228
conf("spark.comet.columnar.shuffle.batch.size")
229229
.internal()
230-
.doc("Batch size when writing out sorted spill files on the native side.")
230+
.doc("Batch size when writing out sorted spill files on the native side. Note that " +
231+
"this should not be larger than batch size (i.e., `spark.comet.batchSize`). Otherwise " +
232+
"it will produce larger batches than expected in the native operator after shuffle.")
231233
.intConf
232234
.createWithDefault(8192)
233235

‎common/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/ArrowReaderIterator.scala‎

Lines changed: 5 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -21,22 +21,14 @@ package org.apache.spark.sql.comet.execution.shuffle
2121

2222
import java.nio.channels.ReadableByteChannel
2323

24-
import org.apache.spark.sql.internal.SQLConf
2524
import org.apache.spark.sql.vectorized.ColumnarBatch
2625

27-
import org.apache.comet.CometConf
28-
import org.apache.comet.vector.{NativeUtil, StreamReader}
26+
import org.apache.comet.vector.StreamReader
2927

3028
class ArrowReaderIterator(channel: ReadableByteChannel) extends Iterator[ColumnarBatch] {
3129

32-
private val nativeUtil = new NativeUtil
33-
34-
private val maxBatchSize = CometConf.COMET_BATCH_SIZE.get(SQLConf.get)
35-
3630
private val reader = StreamReader(channel)
37-
private var currentIdx = -1
3831
private var batch = nextBatch()
39-
private var previousBatch: ColumnarBatch = null
4032
private var currentBatch: ColumnarBatch = null
4133

4234
override def hasNext: Boolean = {
@@ -57,47 +49,28 @@ class ArrowReaderIterator(channel: ReadableByteChannel) extends Iterator[Columna
5749
}
5850

5951
val nextBatch = batch.get
60-
val batchRows = nextBatch.numRows()
61-
val numRows = Math.min(batchRows - currentIdx, maxBatchSize)
6252

63-
// Release the previous sliced batch.
53+
// Release the previous batch.
6454
// If it is not released, when closing the reader, arrow library will complain about
6555
// memory leak.
6656
if (currentBatch != null) {
67-
// Close plain arrays in the previous sliced batch.
68-
// The dictionary arrays will be closed when closing the entire batch.
6957
currentBatch.close()
7058
}
7159

72-
currentBatch = nativeUtil.takeRows(nextBatch, currentIdx, numRows)
73-
currentIdx += numRows
74-
75-
if (currentIdx == batchRows) {
76-
// We cannot close the batch here, because if there is dictionary array in the batch,
77-
// the dictionary array will be closed immediately, and the returned sliced batch will
78-
// be invalid.
79-
previousBatch = batch.get
80-
81-
batch = None
82-
currentIdx = -1
83-
}
84-
60+
currentBatch = nextBatch
61+
batch = None
8562
currentBatch
8663
}
8764

8865
private def nextBatch(): Option[ColumnarBatch] = {
89-
if (previousBatch != null) {
90-
previousBatch.close()
91-
previousBatch = null
92-
}
93-
currentIdx = 0
9466
reader.nextBatch()
9567
}
9668

9769
def close(): Unit =
9870
synchronized {
9971
if (currentBatch != null) {
10072
currentBatch.close()
73+
currentBatch = null
10174
}
10275
reader.close()
10376
}

0 commit comments

Comments
 (0)