1919
2020package org .apache .spark .sql .comet .execution .arrow
2121
22- import java .io .{ ByteArrayInputStream , ByteArrayOutputStream }
22+ import java .nio . ByteBuffer
2323import java .nio .channels .Channels
2424
2525import scala .collection .mutable
@@ -39,6 +39,7 @@ import org.apache.arrow.vector.util.DataSizeRoundingUtil
3939import org .apache .spark .SparkException
4040import org .apache .spark .sql .comet .util .Utils
4141import org .apache .spark .sql .vectorized .ColumnarBatch
42+ import org .apache .spark .util .io .{ChunkedByteBuffer , ChunkedByteBufferOutputStream }
4243
4344import org .apache .comet .vector .CometVector
4445
@@ -83,11 +84,6 @@ private[comet] object CachedBatchIpc {
8384 " Supported values: none, zstd" )
8485 }
8586
86- // Room for the encapsulated metadata message that precedes the body. The message is a small
87- // flatbuffer whose size grows with the field count, not the data, so this is a starting size for
88- // the output buffer rather than a bound -- it grows if a very wide schema needs more.
89- private val METADATA_SIZE_HINT = 8 * 1024
90-
9187 // Decompressors are stateless and shared. Resolving one per cached batch would allocate a codec
9288 // per batch on every scan, and the enum lookup walks the CodecType values each time.
9389 private val readCodecs : Map [CompressionUtil .CodecType , CompressionCodec ] =
@@ -184,7 +180,13 @@ private[comet] object CachedBatchIpc {
184180 /**
185181 * Serialize `batch` into one encapsulated IPC RecordBatch message.
186182 *
187- * Returns the message bytes and the on-body compressed size of each top-level column, which the
183+ * The message is written into heap chunks of `chunkSize` bytes rather than one array, so a
184+ * batch is not bounded by the 2 GiB a single JVM array can hold. Nothing bounds a cached
185+ * batch's bytes upstream: the row path cuts batches by row count alone, and a columnar input
186+ * batch arrives at whatever size the plan above produced. Chunks are appended rather than grown
187+ * and recopied, so the write also never holds the payload twice.
188+ *
189+ * Returns the message and the on-body compressed size of each top-level column, which the
188190 * caller records in the statistics row. The sizes come from the message's own buffer layout, so
189191 * they are the real stored sizes rather than an estimate.
190192 *
@@ -200,7 +202,8 @@ private[comet] object CachedBatchIpc {
200202 def serialize (
201203 batch : ColumnarBatch ,
202204 codec : CompressionCodec ,
203- allocator : BufferAllocator ): (Array [Byte ], Array [Long ]) = {
205+ allocator : BufferAllocator ,
206+ chunkSize : Int ): (ChunkedByteBuffer , Array [Long ]) = {
204207 val (vectors, decoded) = decodeDictionaries(batch, allocator)
205208 try {
206209 val root = new VectorSchemaRoot (vectors.asJava)
@@ -224,15 +227,14 @@ private[comet] object CachedBatchIpc {
224227 // clearing releases buffers, not the schema.
225228 root.clear()
226229
227- // Sized up front from the body length the record batch already knows, plus room for the
228- // metadata message. An unsized ByteArrayOutputStream starts at 32 bytes and doubles, so a
229- // multi-MiB payload would be reallocated and recopied a dozen-odd times per batch.
230- val sizeHint = recordBatch.computeBodyLength() + METADATA_SIZE_HINT
231- val out = new ByteArrayOutputStream (
232- math.min(math.max(sizeHint, METADATA_SIZE_HINT ), Int .MaxValue .toLong).toInt)
233- val channel = new WriteChannel (Channels .newChannel(out))
234- MessageSerializer .serialize(channel, recordBatch)
235- (out.toByteArray, columnSizes(fields, recordBatch))
230+ val out = new ChunkedByteBufferOutputStream (chunkSize, ByteBuffer .allocate)
231+ try {
232+ val channel = new WriteChannel (Channels .newChannel(out))
233+ MessageSerializer .serialize(channel, recordBatch)
234+ } finally {
235+ out.close()
236+ }
237+ (out.toChunkedByteBuffer, columnSizes(fields, recordBatch))
236238 } finally {
237239 recordBatch.close()
238240 }
@@ -278,8 +280,8 @@ private[comet] object CachedBatchIpc {
278280 * selected are never read, let alone inflated. The windows are then decompressed in one pass;
279281 * see [[decompressed ]] for why that is not left to `VectorLoader`.
280282 */
281- def load (data : Array [ Byte ] , allocator : BufferAllocator ): VectorSchemaRoot = {
282- val readChannel = new ReadChannel (Channels .newChannel(new ByteArrayInputStream ( data)))
283+ def load (data : ChunkedByteBuffer , allocator : BufferAllocator ): VectorSchemaRoot = {
284+ val readChannel = new ReadChannel (Channels .newChannel(data.toInputStream( )))
283285 // Reads the message metadata only. The body stays in `data` and is copied selectively.
284286 val metadata = MessageSerializer .readMessage(readChannel)
285287 if (metadata == null ) {
@@ -302,7 +304,8 @@ private[comet] object CachedBatchIpc {
302304
303305 // serialize writes exactly [encapsulated message][body] and nothing after it, so the body is
304306 // the tail of `data`.
305- val bodyStart = data.length - metadata.getMessageBodyLength.toInt
307+ val bodyStart = data.size - metadata.getMessageBodyLength
308+ val chunks = new PayloadChunks (data)
306309
307310 val compression =
308311 if (batch.compression() == null ) NoCompressionCodec .DEFAULT_BODY_COMPRESSION
@@ -337,7 +340,7 @@ private[comet] object CachedBatchIpc {
337340 while (i < bufferIndices.length) {
338341 val length = lengths(i)
339342 if (length > 0 ) {
340- body.setBytes(position, data , bodyStart + offsets(i).toInt , length.toInt )
343+ chunks.copyTo(body, position , bodyStart + offsets(i), length)
341344 }
342345 val window = body.slice(position, length)
343346 window.writerIndex(length)
@@ -376,6 +379,50 @@ private[comet] object CachedBatchIpc {
376379 }
377380 }
378381
382+ /**
383+ * Random access to a chunked payload by absolute position.
384+ *
385+ * A buffer's window within the body is wherever the writer's chunking happened to put it, so it
386+ * can start in one chunk and end several chunks later. The chunk starts are summed from the
387+ * chunks themselves rather than assumed to be a multiple of the writer's chunk size, which
388+ * keeps the reader independent of how the payload was cut.
389+ */
390+ private final class PayloadChunks (data : ChunkedByteBuffer ) {
391+ // Duplicates, so the positions set while copying do not touch the cached payload's buffers.
392+ private val chunks : Array [ByteBuffer ] = data.getChunks()
393+ private val starts : Array [Long ] = chunks.scanLeft(0L )(_ + _.remaining()).toArray
394+
395+ /** Copy `length` bytes starting at `from` in the payload into `dst` at `dstIndex`. */
396+ def copyTo (dst : ArrowBuf , dstIndex : Long , from : Long , length : Long ): Unit = {
397+ // The flatbuffer accessors behind `from` and `length` are unchecked, so a window reaching
398+ // past the payload is a corrupt or mismatched batch, and is reported as one rather than as
399+ // an index error from inside the copy.
400+ if (from < 0 || length < 0 || from + length > starts.last) {
401+ throw new SparkException (
402+ s " Comet cached batch records a buffer at [ $from, ${from + length}) outside its " +
403+ s " ${starts.last}-byte payload " )
404+ }
405+ // The last chunk whose start is at or before `from`. Empty chunks share a start with the
406+ // chunk after them and are skipped by the loop below.
407+ val found = java.util.Arrays .binarySearch(starts, 0 , chunks.length, from)
408+ var c = if (found >= 0 ) found else - found - 2
409+ var copied = 0L
410+ while (copied < length) {
411+ val chunk = chunks(c)
412+ val offsetInChunk = (from + copied - starts(c)).toInt
413+ val n = math.min(length - copied, chunk.remaining().toLong - offsetInChunk).toInt
414+ if (n > 0 ) {
415+ val window = chunk.duplicate()
416+ window.position(chunk.position() + offsetInChunk)
417+ window.limit(chunk.position() + offsetInChunk + n)
418+ dst.setBytes(dstIndex + copied, window)
419+ copied += n
420+ }
421+ c += 1
422+ }
423+ }
424+ }
425+
379426 /**
380427 * The indices, within a record batch's flat depth-first sequence, that the selected columns
381428 * own, paired with the length of the whole sequence.
0 commit comments