Skip to content

Commit c7bc49e

Browse files
committed
refactor: dedup Spark-columnar-to-Arrow copy and tidy cache serializer
Extract the element-wise Spark `ColumnVector` -> Arrow copy loop that `columnarBatchToArrowBatch` had duplicated from `SparkColumnarArrowReader` into a shared `CometArrowConverters.writeColumns`, so the zero-column rowCount workaround lives in one place. Also: - hoist the Arrow schema out of the per-batch path - replace the `(batch, ownsBatch)` tuple and `.toList` with a `serializeBatch` helper that takes the single element eagerly - move `isArrowBacked` to `Utils`, next to the `getBatchFieldVectors` precondition it describes - drop `toStructType` in favour of the existing `Utils.fromAttributes` - document at `supportsColumnarInput` why its schema-only answer means the conversion in `encodeBatches` is load bearing - route the new tests through the existing `withNativeCache` helper and assert both of them really cached in Comet's format
1 parent 9dce0d6 commit c7bc49e

5 files changed

Lines changed: 137 additions & 151 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowCachedBatchSerializer.scala‎

Lines changed: 32 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -70,13 +70,6 @@ class ArrowCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer {
7070

7171
private val fallback = new DefaultCachedBatchSerializer()
7272

73-
// Row-to-Arrow conversion needs a StructType, while cache APIs pass attributes.
74-
private def toStructType(schema: Seq[Attribute]): StructType = {
75-
StructType(schema.map { attr =>
76-
StructField(attr.name, attr.dataType, attr.nullable, attr.metadata)
77-
})
78-
}
79-
8073
// Build the statistics row expected by SimpleMetricsCachedBatchSerializer.
8174
// For each cached column Spark expects five values in this order:
8275
// lower bound, upper bound, null count, row count, and size in bytes.
@@ -181,52 +174,40 @@ class ArrowCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer {
181174
// The stats are stored beside the Arrow bytes so Spark's cache filter can prune
182175
// CometCachedBatch without decoding the batch first.
183176
//
184-
// Spark decides the input path from `supportsColumnarInput`, which only sees the schema, so a
185-
// columnar input batch is not guaranteed to be Arrow-backed: any Spark or third-party columnar
186-
// leaf (Spark's vectorized Parquet/ORC reader, a connector's own vectors) reaches
187-
// `convertColumnarBatchToCachedBatch` with `On/OffHeapColumnVector`-style columns, which
188-
// `Utils.serializeBatches` cannot write. Copy those into Arrow first.
177+
// A columnar input batch is not guaranteed to be Arrow-backed; see supportsColumnarInput for
178+
// why. Batches that are not get copied into Arrow first, since Utils.serializeBatches only
179+
// writes CometVector columns.
189180
private def encodeBatches(
190181
batches: Iterator[ColumnarBatch],
191182
attrs: Seq[Attribute]): Iterator[CachedBatch] = {
192-
lazy val structType = toStructType(attrs)
183+
val arrowSchema =
184+
Utils.toArrowSchema(Utils.fromAttributes(attrs), CometArrowStream.NATIVE_TIMEZONE)
193185

194-
batches.flatMap { batch =>
186+
batches.map { batch =>
195187
val stats = computeStats(batch, attrs)
196188

197-
val (arrowBatch, ownsBatch) =
198-
if (CometArrowConverters.isArrowBacked(batch)) {
199-
(batch, false)
200-
} else {
201-
val converted = CometArrowConverters.columnarBatchToArrowBatch(
202-
batch,
203-
structType,
204-
CometArrowStream.NATIVE_TIMEZONE,
205-
CometArrowAllocator)
206-
(converted, true)
207-
}
208-
209-
try {
210-
// `Utils.serializeBatches` is lazy and clears the batch's vectors as it writes, so the
211-
// result has to be materialized before a converted batch is closed.
212-
Utils
213-
.serializeBatches(Iterator.single(arrowBatch))
214-
.map { case (rows, buffer) =>
215-
CometCachedBatch(
216-
numRows = rows.toInt,
217-
sizeInBytes = buffer.size,
218-
stats = stats,
219-
bytes = buffer)
220-
}
221-
.toList
222-
} finally {
223-
if (ownsBatch) {
224-
arrowBatch.close()
225-
}
189+
if (Utils.isArrowBacked(batch)) {
190+
serializeBatch(batch, stats)
191+
} else {
192+
val arrowBatch =
193+
CometArrowConverters.columnarBatchToArrowBatch(batch, arrowSchema, CometArrowAllocator)
194+
try serializeBatch(arrowBatch, stats)
195+
finally arrowBatch.close()
226196
}
227197
}
228198
}
229199

200+
// Utils.serializeBatches is one-in/one-out, so take the single element eagerly: the write has to
201+
// happen before a converted batch is closed.
202+
private def serializeBatch(batch: ColumnarBatch, stats: InternalRow): CachedBatch = {
203+
val (rows, buffer) = Utils.serializeBatches(Iterator.single(batch)).next()
204+
CometCachedBatch(
205+
numRows = rows.toInt,
206+
sizeInBytes = buffer.size,
207+
stats = stats,
208+
bytes = buffer)
209+
}
210+
230211
// Resolve requested columns by exprId, not by name, because aliases may reuse names.
231212
private def selectedIndices(
232213
cacheAttributes: Seq[Attribute],
@@ -285,6 +266,13 @@ class ArrowCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer {
285266
// Comet's Arrow writer only handles the types listed in supportsSchema. Reporting false here
286267
// sends the relation down the row path, where it is delegated to Spark's default serializer,
287268
// instead of failing at cache materialization inside Utils.serializeBatches.
269+
//
270+
// This answer is schema-only, because attributes are all Spark gives us; it says nothing about
271+
// the vectors. Returning true also makes InMemoryRelation strip the ColumnarToRow above the
272+
// cached plan, so convertColumnarBatchToCachedBatch then receives whatever that plan produces:
273+
// a Comet scan's CometVectors, but equally Spark's vectorized Parquet/ORC reader or a
274+
// connector's own vectors. encodeBatches converts the non-Arrow ones; that conversion is load
275+
// bearing, not defensive.
288276
override def supportsColumnarInput(schema: Seq[Attribute]): Boolean = supportsSchema(schema)
289277

290278
// A relation Comet stores is always readable as columnar Arrow. Anything else holds
@@ -356,7 +344,7 @@ class ArrowCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer {
356344
input.mapPartitions { rows =>
357345
val iter = CometArrowConverters.rowToArrowBatchIter(
358346
rows,
359-
toStructType(schema),
347+
Utils.fromAttributes(schema),
360348
batchSize,
361349
sessionTz,
362350
CometArrowAllocator)

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowConverters.scala‎

Lines changed: 35 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,19 @@ import org.apache.spark.sql.comet.util.Utils
2828
import org.apache.spark.sql.types.StructType
2929
import org.apache.spark.sql.vectorized.{ColumnarArray, ColumnarBatch}
3030

31-
import org.apache.comet.vector.{CometVector, NativeUtil}
31+
import org.apache.comet.vector.NativeUtil
3232

3333
/**
34-
* Convert a stream of Spark `InternalRow`s to a stream of independently-owned Arrow
35-
* `ColumnarBatch`es: each emitted batch owns a fresh `VectorSchemaRoot` with newly allocated
36-
* buffers and the consumer is responsible for closing it.
34+
* Convert Spark data that is not Arrow-backed (`InternalRow`s, or `ColumnarBatch`es whose columns
35+
* are Spark/third-party `ColumnVector`s) into independently-owned Arrow `ColumnarBatch`es: each
36+
* emitted batch owns a fresh `VectorSchemaRoot` with newly allocated buffers and the consumer is
37+
* responsible for closing it.
3738
*
38-
* This differs from [[RowArrowReader]], which reuses one stable `VectorSchemaRoot`
39-
* (release-and-replace) so only one batch is valid at a time. Use this when multiple emitted
40-
* batches must be alive simultaneously (e.g. tests that buffer several batches before consuming).
41-
* Buffers come from the caller-provided `BufferAllocator`, whose lifecycle the caller owns.
39+
* This differs from [[RowArrowReader]] and [[SparkColumnarArrowReader]], which reuse one stable
40+
* `VectorSchemaRoot` (release-and-replace) so only one batch is valid at a time. Use this when
41+
* multiple emitted batches must be alive simultaneously (e.g. tests that buffer several batches
42+
* before consuming). Buffers come from the caller-provided `BufferAllocator`, whose lifecycle the
43+
* caller owns.
4244
*/
4345
object CometArrowConverters extends Logging {
4446

@@ -76,27 +78,22 @@ object CometArrowConverters extends Logging {
7678
}
7779

7880
/**
79-
* Copy a Spark `ColumnarBatch` whose columns are not Arrow-backed (e.g.
80-
* `On/OffHeapColumnVector` from Spark's vectorized Parquet reader, or a third-party connector's
81-
* vectors) into a freshly allocated Arrow `ColumnarBatch` of `CometVector`s.
81+
* Copy `numRows` rows starting at `startRow` from a Spark `ColumnarBatch` into `root`.
8282
*
83-
* The input batch is not consumed or closed; the caller owns the returned batch and must close
84-
* it. Values are copied element-wise, since Spark's `ColumnVector` implementations do not
85-
* expose Arrow buffers.
83+
* Spark's `ColumnVector` implementations do not expose Arrow buffers, so values are necessarily
84+
* copied element-wise. Shared by [[SparkColumnarArrowReader]], which slices into a stable root,
85+
* and [[columnarBatchToArrowBatch]], which fills a fresh one.
8686
*/
87-
def columnarBatchToArrowBatch(
87+
private[arrow] def writeColumns(
88+
root: VectorSchemaRoot,
8889
batch: ColumnarBatch,
89-
schema: StructType,
90-
timeZoneId: String,
91-
allocator: BufferAllocator): ColumnarBatch = {
92-
val arrowSchema: Schema = Utils.toArrowSchema(schema, timeZoneId)
93-
val root = VectorSchemaRoot.create(arrowSchema, allocator)
90+
startRow: Int,
91+
numRows: Int): Unit = {
9492
val writer = ArrowWriter.create(root)
95-
val numRows = batch.numRows()
9693
var col = 0
9794
while (col < batch.numCols()) {
9895
val column = batch.column(col)
99-
val columnArray = new ColumnarArray(column, 0, numRows)
96+
val columnArray = new ColumnarArray(column, startRow, numRows)
10097
if (column.hasNull) {
10198
writer.writeCol(columnArray, col)
10299
} else {
@@ -109,10 +106,22 @@ object CometArrowConverters extends Logging {
109106
// batch (Spark's count-from-metadata scan: numRows > 0, numCols == 0) would otherwise produce
110107
// a root with rowCount == 0 and silently drop the rows.
111108
root.setRowCount(numRows)
112-
NativeUtil.rootAsBatch(root)
113109
}
114110

115-
/** Whether every column in `batch` is already an Arrow-backed `CometVector`. */
116-
def isArrowBacked(batch: ColumnarBatch): Boolean =
117-
(0 until batch.numCols()).forall(i => batch.column(i).isInstanceOf[CometVector])
111+
/**
112+
* Copy a Spark `ColumnarBatch` whose columns are not Arrow-backed (e.g.
113+
* `On/OffHeapColumnVector` from Spark's vectorized Parquet reader, or a third-party connector's
114+
* vectors) into a freshly allocated Arrow `ColumnarBatch` of `CometVector`s.
115+
*
116+
* The input batch is not consumed or closed; the caller owns the returned batch and must close
117+
* it.
118+
*/
119+
def columnarBatchToArrowBatch(
120+
batch: ColumnarBatch,
121+
arrowSchema: Schema,
122+
allocator: BufferAllocator): ColumnarBatch = {
123+
val root = VectorSchemaRoot.create(arrowSchema, allocator)
124+
writeColumns(root, batch, 0, batch.numRows())
125+
NativeUtil.rootAsBatch(root)
126+
}
118127
}

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/SparkColumnarArrowReader.scala‎

Lines changed: 8 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,13 @@ package org.apache.spark.sql.comet.execution.arrow
2222
import org.apache.arrow.memory.BufferAllocator
2323
import org.apache.arrow.vector.ipc.ArrowReader
2424
import org.apache.arrow.vector.types.pojo.Schema
25-
import org.apache.spark.sql.vectorized.{ColumnarArray, ColumnarBatch}
25+
import org.apache.spark.sql.vectorized.ColumnarBatch
2626

2727
/**
2828
* `ArrowReader` over an iterator of Spark-side `ColumnarBatch`es (not Arrow-backed). Slices up to
2929
* `maxRecordsPerBatch` rows per `loadNextBatch` from the current Spark batch into the reader's
30-
* stable VSR via `ArrowWriter.writeCol`. Spark's `ColumnVector` implementations aren't Arrow
31-
* buffers, so this reader necessarily copies element values into Arrow format.
30+
* stable VSR via `CometArrowConverters.writeColumns`. Spark's `ColumnVector` implementations
31+
* aren't Arrow buffers, so this reader necessarily copies element values into Arrow format.
3232
*/
3333
private[comet] class SparkColumnarArrowReader(
3434
allocator: BufferAllocator,
@@ -76,26 +76,13 @@ private[comet] class SparkColumnarArrowReader(
7676
if (maxRecordsPerBatch <= 0) rowsRemaining
7777
else math.min(maxRecordsPerBatch, rowsRemaining)
7878

79-
val writer = ArrowWriter.create(getVectorSchemaRoot)
80-
var col = 0
81-
while (col < current.numCols()) {
82-
val column = current.column(col)
83-
val columnArray = new ColumnarArray(column, rowsConsumedInCurrent, rowsToProduce)
84-
if (column.hasNull) {
85-
writer.writeCol(columnArray, col)
86-
} else {
87-
writer.writeColNoNull(columnArray, col)
88-
}
89-
col += 1
90-
}
79+
CometArrowConverters.writeColumns(
80+
getVectorSchemaRoot,
81+
current,
82+
rowsConsumedInCurrent,
83+
rowsToProduce)
9184
rowsConsumedInCurrent += rowsToProduce
9285

93-
writer.finish()
94-
// ArrowWriter derives the root row count from its per-column writes, so a zero-column
95-
// input batch (Spark's count-from-metadata scan: numRows > 0, numCols == 0) would otherwise
96-
// produce a root with rowCount == 0 and silently drop the rows. Set rowCount explicitly so
97-
// downstream aggregations (e.g. df.count()) see the correct value.
98-
getVectorSchemaRoot.setRowCount(rowsToProduce)
9986
onConversionNs(System.nanoTime() - startNs)
10087
true
10188
}

‎spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,15 @@ object Utils extends CometTypeShim with Logging {
397397
}
398398
}
399399

400+
/**
401+
* Whether every column in `batch` satisfies [[getBatchFieldVectors]]'s precondition, i.e. is an
402+
* Arrow-backed `CometVector`. Callers that may receive batches from a plan they did not build
403+
* (e.g. Comet's cache serializer, which Spark hands the cached plan's columnar output) use this
404+
* to convert foreign vectors to Arrow instead of tripping the exception below.
405+
*/
406+
def isArrowBacked(batch: ColumnarBatch): Boolean =
407+
(0 until batch.numCols()).forall(i => batch.column(i).isInstanceOf[CometVector])
408+
400409
def getBatchFieldVectors(
401410
batch: ColumnarBatch): (Seq[FieldVector], Option[DictionaryProvider]) = {
402411
var provider: Option[DictionaryProvider] = None

0 commit comments

Comments
 (0)