Skip to content

Commit 14bf845

Browse files
committed
fix: store cached batch payloads in chunks so a batch can exceed 2 GiB
A single Array[Byte] capped each cached batch at the 2 GiB one JVM array can hold, a regression against the chunked format this replaced. Nothing bounds a batch's bytes upstream, so wide rows or a large columnar input batch could reach it. Write the IPC message into ChunkedByteBuffer heap chunks and copy each projected buffer window out chunk by chunk. An internal spark.comet.exec.inMemoryCache.chunkSize config lets the projection, corruption and leak tests run with 61-byte chunks so every window crosses a chunk boundary.
1 parent 6b8ded1 commit 14bf845

6 files changed

Lines changed: 298 additions & 132 deletions

File tree

‎spark/src/main/scala/org/apache/comet/CometConf.scala‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,18 @@ object CometConf extends ShimCometConf {
307307
.intConf
308308
.createWithDefault(1)
309309

310+
val COMET_EXEC_IN_MEMORY_CACHE_CHUNK_SIZE: ConfigEntry[Long] =
311+
conf("spark.comet.exec.inMemoryCache.chunkSize")
312+
.category(CATEGORY_TESTING)
313+
.doc(
314+
"The size of the heap chunks a cached batch payload is written into. Chunking is what " +
315+
"lets a single cached batch exceed the 2 GiB limit of one JVM array. This is an " +
316+
"internal config for testing purposes.")
317+
.internal()
318+
.bytesConf(ByteUnit.BYTE)
319+
.checkValue(v => v > 0 && v <= Int.MaxValue, "Must be positive and fit in an Int.")
320+
.createWithDefault(1024 * 1024)
321+
310322
val COMET_NATIVE_COLUMNAR_TO_ROW_ENABLED: ConfigEntry[Boolean] =
311323
conf(s"$COMET_EXEC_CONFIG_PREFIX.columnarToRow.native.enabled")
312324
.category(CATEGORY_EXEC)

‎spark/src/main/scala/org/apache/comet/CometKryoRegistrator.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,8 @@ import com.esotericsoftware.kryo.Kryo
4242
* `CometDriverPlugin` logs a warning instead when the combination looks unsafe.
4343
*
4444
* Two payloads need it: the `Array[ChunkedByteBuffer]` a native broadcast broadcasts, and
45-
* `CometCachedBatch`. The first applies whether or not the in-memory cache feature is enabled.
45+
* `CometCachedBatch`, whose payload is a `ChunkedByteBuffer` too. The first applies whether or
46+
* not the in-memory cache feature is enabled.
4647
*/
4748
class CometKryoRegistrator extends KryoRegistrator {
4849
override def registerClasses(kryo: Kryo): Unit = {

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowCachedBatchSerializer.scala‎

Lines changed: 27 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import org.apache.spark.sql.types._
3636
import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector}
3737
import org.apache.spark.storage.StorageLevel
3838
import org.apache.spark.unsafe.types.UTF8String
39+
import org.apache.spark.util.io.ChunkedByteBuffer
3940

4041
import org.apache.comet.{CometArrowAllocator, CometConf, DataTypeSupport}
4142
import org.apache.comet.vector.NativeUtil
@@ -44,27 +45,30 @@ import org.apache.comet.vector.NativeUtil
4445
* Cached batch format used when Comet writes Spark in-memory cache data.
4546
*
4647
* `bytes` is one encapsulated Arrow IPC RecordBatch message and its body, with no Schema message
47-
* and no end-of-stream marker, produced by `CachedBatchIpc.serialize`. Compression is applied per
48-
* Arrow buffer rather than over the payload as a whole, which is what lets a scan decompress only
49-
* the columns it projected: the message records every buffer's offset and length, so
50-
* `CachedBatchIpc.Projection.load` copies out just the selected columns' byte ranges. The cache
51-
* manager still owns storage and eviction; this class only changes the cached payload.
48+
* and no end-of-stream marker, produced by `CachedBatchIpc.serialize`. It is held in heap chunks
49+
* rather than one array so that a batch is not capped at the 2 GiB a JVM array can address.
50+
* Compression is applied per Arrow buffer rather than over the payload as a whole, which is what
51+
* lets a scan decompress only the columns it projected: the message records every buffer's offset
52+
* and length, so `CachedBatchIpc.Projection.load` copies out just the selected columns' byte
53+
* ranges. The cache manager still owns storage and eviction; this class only changes the cached
54+
* payload.
5255
*/
5356
private case class CometCachedBatch(
5457
override val numRows: Int,
5558
override val sizeInBytes: Long,
5659
override val stats: InternalRow,
57-
bytes: Array[Byte])
60+
bytes: ChunkedByteBuffer)
5861
extends SimpleMetricsCachedBatch
5962

6063
/**
61-
* The write codec, resolved on the driver and shipped to the executors in the write closure.
64+
* The write codec and payload chunk size, resolved on the driver and shipped to the executors in
65+
* the write closure.
6266
*
6367
* Both write paths resolve it there rather than inside their `mapPartitions` closure: on an
6468
* executor `CometConf` would resolve against whatever `SQLConf` happens to be current on that
6569
* thread rather than against this session's.
6670
*/
67-
private case class CacheCodecSettings(name: String, zstdLevel: Int)
71+
private case class CacheWriteSettings(codecName: String, zstdLevel: Int, chunkSize: Int)
6872

6973
/**
7074
* Cache serializer that stores Comet-compatible Arrow batches in Spark's in-memory cache.
@@ -372,10 +376,11 @@ class ArrowCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer {
372376
case _ => false
373377
}
374378

375-
private def codecSettings(conf: SQLConf): CacheCodecSettings =
376-
CacheCodecSettings(
379+
private def writeSettings(conf: SQLConf): CacheWriteSettings =
380+
CacheWriteSettings(
377381
CometConf.COMET_EXEC_IN_MEMORY_CACHE_COMPRESSION_CODEC.get(conf),
378-
CometConf.COMET_EXEC_IN_MEMORY_CACHE_COMPRESSION_ZSTD_LEVEL.get(conf))
382+
CometConf.COMET_EXEC_IN_MEMORY_CACHE_COMPRESSION_ZSTD_LEVEL.get(conf),
383+
CometConf.COMET_EXEC_IN_MEMORY_CACHE_CHUNK_SIZE.get(conf).toInt)
379384

380385
// Serialize each batch to Arrow, gathering the Spark-compatible cache stats first. The stats are
381386
// stored beside the Arrow bytes so Spark's cache filter can prune a CometCachedBatch without
@@ -387,11 +392,11 @@ class ArrowCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer {
387392
private def encodeBatches(
388393
batches: Iterator[ColumnarBatch],
389394
attrs: Seq[Attribute],
390-
codecSetting: CacheCodecSettings): Iterator[CachedBatch] = {
395+
settings: CacheWriteSettings): Iterator[CachedBatch] = {
391396
val arrowSchema =
392397
Utils.toArrowSchema(Utils.fromAttributes(attrs), CometArrowStream.NATIVE_TIMEZONE)
393398
val readerFields = arrowSchema.getFields.asScala.toIndexedSeq
394-
val codec = CachedBatchIpc.compressionCodec(codecSetting.name, codecSetting.zstdLevel)
399+
val codec = CachedBatchIpc.compressionCodec(settings.codecName, settings.zstdLevel)
395400
val orderings = boundsOrderings(attrs)
396401

397402
batches.map { batch =>
@@ -408,17 +413,17 @@ class ArrowCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer {
408413
Utils.isArrowBacked(batch) && CachedBatchIpc.matchesReaderLayout(batch, readerFields)
409414

410415
val (bytes, columnSizes) = if (writeDirectly) {
411-
CachedBatchIpc.serialize(batch, codec, CometArrowAllocator)
416+
CachedBatchIpc.serialize(batch, codec, CometArrowAllocator, settings.chunkSize)
412417
} else {
413418
val arrowBatch =
414419
CometArrowConverters.columnarBatchToArrowBatch(batch, arrowSchema, CometArrowAllocator)
415-
try CachedBatchIpc.serialize(arrowBatch, codec, CometArrowAllocator)
420+
try CachedBatchIpc.serialize(arrowBatch, codec, CometArrowAllocator, settings.chunkSize)
416421
finally arrowBatch.close()
417422
}
418423

419424
CometCachedBatch(
420425
numRows = numRows,
421-
sizeInBytes = bytes.length.toLong,
426+
sizeInBytes = bytes.size,
422427
stats = statsRow(lower, upper, nulls, numRows, columnSizes),
423428
bytes = bytes)
424429
}
@@ -498,10 +503,10 @@ class ArrowCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer {
498503
storageLevel: StorageLevel,
499504
conf: SQLConf): RDD[CachedBatch] = {
500505

501-
val codec = codecSettings(conf)
506+
val settings = writeSettings(conf)
502507

503508
input.mapPartitions { batches =>
504-
encodeBatches(batches, schema, codec)
509+
encodeBatches(batches, schema, settings)
505510
}
506511
}
507512

@@ -638,7 +643,7 @@ class ArrowCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer {
638643
fallback.convertInternalRowToCachedBatch(input, schema, storageLevel, conf)
639644
} else {
640645
val batchSize = conf.columnBatchSize
641-
val codec = codecSettings(conf)
646+
val settings = writeSettings(conf)
642647

643648
input.mapPartitions { rows =>
644649
val iter = CometArrowConverters.rowToArrowBatchIter(
@@ -656,7 +661,7 @@ class ArrowCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer {
656661
CometArrowStream.NATIVE_TIMEZONE,
657662
CometArrowAllocator)
658663

659-
encodeBatches(iter, schema, codec)
664+
encodeBatches(iter, schema, settings)
660665
}
661666
}
662667
}
@@ -728,10 +733,9 @@ object ArrowCachedBatchSerializer {
728733
* `CometKryoRegistrator` registers these instead.
729734
*/
730735
def kryoClasses: Seq[Class[_]] = Seq(
736+
// The payload itself is a ChunkedByteBuffer of heap chunks, which Utils.arrowBytesKryoClasses
737+
// already registers for the native broadcast path.
731738
classOf[CometCachedBatch],
732-
// The payload itself. Kryo registers Array[Byte] by default, but registering it here is what
733-
// keeps that true if the payload type ever changes again.
734-
classOf[Array[Byte]],
735739
// The statistics row, whose values are bounds in Spark's internal representation: boxed
736740
// primitives, which Kryo registers by default, plus UTF8String and Decimal, which it does not.
737741
// A Decimal above Long precision holds a scala.math.BigDecimal, which Chill's Scala registrar

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchIpc.scala‎

Lines changed: 68 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
package org.apache.spark.sql.comet.execution.arrow
2121

22-
import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
22+
import java.nio.ByteBuffer
2323
import java.nio.channels.Channels
2424

2525
import scala.collection.mutable
@@ -39,6 +39,7 @@ import org.apache.arrow.vector.util.DataSizeRoundingUtil
3939
import org.apache.spark.SparkException
4040
import org.apache.spark.sql.comet.util.Utils
4141
import org.apache.spark.sql.vectorized.ColumnarBatch
42+
import org.apache.spark.util.io.{ChunkedByteBuffer, ChunkedByteBufferOutputStream}
4243

4344
import org.apache.comet.vector.CometVector
4445

@@ -83,11 +84,6 @@ private[comet] object CachedBatchIpc {
8384
"Supported values: none, zstd")
8485
}
8586

86-
// Room for the encapsulated metadata message that precedes the body. The message is a small
87-
// flatbuffer whose size grows with the field count, not the data, so this is a starting size for
88-
// the output buffer rather than a bound -- it grows if a very wide schema needs more.
89-
private val METADATA_SIZE_HINT = 8 * 1024
90-
9187
// Decompressors are stateless and shared. Resolving one per cached batch would allocate a codec
9288
// per batch on every scan, and the enum lookup walks the CodecType values each time.
9389
private val readCodecs: Map[CompressionUtil.CodecType, CompressionCodec] =
@@ -184,7 +180,13 @@ private[comet] object CachedBatchIpc {
184180
/**
185181
* Serialize `batch` into one encapsulated IPC RecordBatch message.
186182
*
187-
* Returns the message bytes and the on-body compressed size of each top-level column, which the
183+
* The message is written into heap chunks of `chunkSize` bytes rather than one array, so a
184+
* batch is not bounded by the 2 GiB a single JVM array can hold. Nothing bounds a cached
185+
* batch's bytes upstream: the row path cuts batches by row count alone, and a columnar input
186+
* batch arrives at whatever size the plan above produced. Chunks are appended rather than grown
187+
* and recopied, so the write also never holds the payload twice.
188+
*
189+
* Returns the message and the on-body compressed size of each top-level column, which the
188190
* caller records in the statistics row. The sizes come from the message's own buffer layout, so
189191
* they are the real stored sizes rather than an estimate.
190192
*
@@ -200,7 +202,8 @@ private[comet] object CachedBatchIpc {
200202
def serialize(
201203
batch: ColumnarBatch,
202204
codec: CompressionCodec,
203-
allocator: BufferAllocator): (Array[Byte], Array[Long]) = {
205+
allocator: BufferAllocator,
206+
chunkSize: Int): (ChunkedByteBuffer, Array[Long]) = {
204207
val (vectors, decoded) = decodeDictionaries(batch, allocator)
205208
try {
206209
val root = new VectorSchemaRoot(vectors.asJava)
@@ -224,15 +227,14 @@ private[comet] object CachedBatchIpc {
224227
// clearing releases buffers, not the schema.
225228
root.clear()
226229

227-
// Sized up front from the body length the record batch already knows, plus room for the
228-
// metadata message. An unsized ByteArrayOutputStream starts at 32 bytes and doubles, so a
229-
// multi-MiB payload would be reallocated and recopied a dozen-odd times per batch.
230-
val sizeHint = recordBatch.computeBodyLength() + METADATA_SIZE_HINT
231-
val out = new ByteArrayOutputStream(
232-
math.min(math.max(sizeHint, METADATA_SIZE_HINT), Int.MaxValue.toLong).toInt)
233-
val channel = new WriteChannel(Channels.newChannel(out))
234-
MessageSerializer.serialize(channel, recordBatch)
235-
(out.toByteArray, columnSizes(fields, recordBatch))
230+
val out = new ChunkedByteBufferOutputStream(chunkSize, ByteBuffer.allocate)
231+
try {
232+
val channel = new WriteChannel(Channels.newChannel(out))
233+
MessageSerializer.serialize(channel, recordBatch)
234+
} finally {
235+
out.close()
236+
}
237+
(out.toChunkedByteBuffer, columnSizes(fields, recordBatch))
236238
} finally {
237239
recordBatch.close()
238240
}
@@ -278,8 +280,8 @@ private[comet] object CachedBatchIpc {
278280
* selected are never read, let alone inflated. The windows are then decompressed in one pass;
279281
* see [[decompressed]] for why that is not left to `VectorLoader`.
280282
*/
281-
def load(data: Array[Byte], allocator: BufferAllocator): VectorSchemaRoot = {
282-
val readChannel = new ReadChannel(Channels.newChannel(new ByteArrayInputStream(data)))
283+
def load(data: ChunkedByteBuffer, allocator: BufferAllocator): VectorSchemaRoot = {
284+
val readChannel = new ReadChannel(Channels.newChannel(data.toInputStream()))
283285
// Reads the message metadata only. The body stays in `data` and is copied selectively.
284286
val metadata = MessageSerializer.readMessage(readChannel)
285287
if (metadata == null) {
@@ -302,7 +304,8 @@ private[comet] object CachedBatchIpc {
302304

303305
// serialize writes exactly [encapsulated message][body] and nothing after it, so the body is
304306
// the tail of `data`.
305-
val bodyStart = data.length - metadata.getMessageBodyLength.toInt
307+
val bodyStart = data.size - metadata.getMessageBodyLength
308+
val chunks = new PayloadChunks(data)
306309

307310
val compression =
308311
if (batch.compression() == null) NoCompressionCodec.DEFAULT_BODY_COMPRESSION
@@ -337,7 +340,7 @@ private[comet] object CachedBatchIpc {
337340
while (i < bufferIndices.length) {
338341
val length = lengths(i)
339342
if (length > 0) {
340-
body.setBytes(position, data, bodyStart + offsets(i).toInt, length.toInt)
343+
chunks.copyTo(body, position, bodyStart + offsets(i), length)
341344
}
342345
val window = body.slice(position, length)
343346
window.writerIndex(length)
@@ -376,6 +379,50 @@ private[comet] object CachedBatchIpc {
376379
}
377380
}
378381

382+
/**
383+
* Random access to a chunked payload by absolute position.
384+
*
385+
* A buffer's window within the body is wherever the writer's chunking happened to put it, so it
386+
* can start in one chunk and end several chunks later. The chunk starts are summed from the
387+
* chunks themselves rather than assumed to be a multiple of the writer's chunk size, which
388+
* keeps the reader independent of how the payload was cut.
389+
*/
390+
private final class PayloadChunks(data: ChunkedByteBuffer) {
391+
// Duplicates, so the positions set while copying do not touch the cached payload's buffers.
392+
private val chunks: Array[ByteBuffer] = data.getChunks()
393+
private val starts: Array[Long] = chunks.scanLeft(0L)(_ + _.remaining()).toArray
394+
395+
/** Copy `length` bytes starting at `from` in the payload into `dst` at `dstIndex`. */
396+
def copyTo(dst: ArrowBuf, dstIndex: Long, from: Long, length: Long): Unit = {
397+
// The flatbuffer accessors behind `from` and `length` are unchecked, so a window reaching
398+
// past the payload is a corrupt or mismatched batch, and is reported as one rather than as
399+
// an index error from inside the copy.
400+
if (from < 0 || length < 0 || from + length > starts.last) {
401+
throw new SparkException(
402+
s"Comet cached batch records a buffer at [$from, ${from + length}) outside its " +
403+
s"${starts.last}-byte payload")
404+
}
405+
// The last chunk whose start is at or before `from`. Empty chunks share a start with the
406+
// chunk after them and are skipped by the loop below.
407+
val found = java.util.Arrays.binarySearch(starts, 0, chunks.length, from)
408+
var c = if (found >= 0) found else -found - 2
409+
var copied = 0L
410+
while (copied < length) {
411+
val chunk = chunks(c)
412+
val offsetInChunk = (from + copied - starts(c)).toInt
413+
val n = math.min(length - copied, chunk.remaining().toLong - offsetInChunk).toInt
414+
if (n > 0) {
415+
val window = chunk.duplicate()
416+
window.position(chunk.position() + offsetInChunk)
417+
window.limit(chunk.position() + offsetInChunk + n)
418+
dst.setBytes(dstIndex + copied, window)
419+
copied += n
420+
}
421+
c += 1
422+
}
423+
}
424+
}
425+
379426
/**
380427
* The indices, within a record batch's flat depth-first sequence, that the selected columns
381428
* own, paired with the length of the whole sequence.

0 commit comments

Comments
 (0)