Skip to content

Commit 9dce0d6

Browse files
committed
fix: convert non-Arrow columnar input in Comet cache serializer
`ArrowCachedBatchSerializer.supportsColumnarInput` decides the cache input format from the schema alone, which is all Spark gives it. Returning true makes `InMemoryRelation` strip the `ColumnarToRow` above the cached plan and feed `cachedPlan.executeColumnar()` directly to `convertColumnarBatchToCachedBatch`. Any Spark or third-party columnar leaf under that transition (Spark's vectorized Parquet/ORC reader, a connector's own vectors) therefore reaches the serializer with `On/OffHeapColumnVector`-style columns, and `Utils.serializeBatches` fails with "Comet execution only takes Arrow Arrays". The serializer cannot detect this from the schema, so copy non-Arrow batches into Arrow at write time. Comet-native cached plans keep the existing zero-copy path.
1 parent 63c5a91 commit 9dce0d6

3 files changed

Lines changed: 170 additions & 8 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowCachedBatchSerializer.scala‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -180,18 +180,49 @@ class ArrowCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer {
180180
// Compute Spark-compatible cache stats before serializing each batch to Arrow.
181181
// The stats are stored beside the Arrow bytes so Spark's cache filter can prune
182182
// CometCachedBatch without decoding the batch first.
183+
//
184+
// Spark decides the input path from `supportsColumnarInput`, which only sees the schema, so a
185+
// columnar input batch is not guaranteed to be Arrow-backed: any Spark or third-party columnar
186+
// leaf (Spark's vectorized Parquet/ORC reader, a connector's own vectors) reaches
187+
// `convertColumnarBatchToCachedBatch` with `On/OffHeapColumnVector`-style columns, which
188+
// `Utils.serializeBatches` cannot write. Copy those into Arrow first.
183189
private def encodeBatches(
184190
batches: Iterator[ColumnarBatch],
185191
attrs: Seq[Attribute]): Iterator[CachedBatch] = {
192+
lazy val structType = toStructType(attrs)
193+
186194
batches.flatMap { batch =>
187195
val stats = computeStats(batch, attrs)
188196

189-
Utils.serializeBatches(Iterator.single(batch)).map { case (rows, buffer) =>
190-
CometCachedBatch(
191-
numRows = rows.toInt,
192-
sizeInBytes = buffer.size,
193-
stats = stats,
194-
bytes = buffer)
197+
val (arrowBatch, ownsBatch) =
198+
if (CometArrowConverters.isArrowBacked(batch)) {
199+
(batch, false)
200+
} else {
201+
val converted = CometArrowConverters.columnarBatchToArrowBatch(
202+
batch,
203+
structType,
204+
CometArrowStream.NATIVE_TIMEZONE,
205+
CometArrowAllocator)
206+
(converted, true)
207+
}
208+
209+
try {
210+
// `Utils.serializeBatches` is lazy and clears the batch's vectors as it writes, so the
211+
// result has to be materialized before a converted batch is closed.
212+
Utils
213+
.serializeBatches(Iterator.single(arrowBatch))
214+
.map { case (rows, buffer) =>
215+
CometCachedBatch(
216+
numRows = rows.toInt,
217+
sizeInBytes = buffer.size,
218+
stats = stats,
219+
bytes = buffer)
220+
}
221+
.toList
222+
} finally {
223+
if (ownsBatch) {
224+
arrowBatch.close()
225+
}
195226
}
196227
}
197228
}

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CometArrowConverters.scala‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ import org.apache.spark.internal.Logging
2626
import org.apache.spark.sql.catalyst.InternalRow
2727
import org.apache.spark.sql.comet.util.Utils
2828
import org.apache.spark.sql.types.StructType
29-
import org.apache.spark.sql.vectorized.ColumnarBatch
29+
import org.apache.spark.sql.vectorized.{ColumnarArray, ColumnarBatch}
3030

31-
import org.apache.comet.vector.NativeUtil
31+
import org.apache.comet.vector.{CometVector, NativeUtil}
3232

3333
/**
3434
* Convert a stream of Spark `InternalRow`s to a stream of independently-owned Arrow
@@ -74,4 +74,45 @@ object CometArrowConverters extends Logging {
7474
}
7575
}
7676
}
77+
78+
/**
79+
* Copy a Spark `ColumnarBatch` whose columns are not Arrow-backed (e.g.
80+
* `On/OffHeapColumnVector` from Spark's vectorized Parquet reader, or a third-party connector's
81+
* vectors) into a freshly allocated Arrow `ColumnarBatch` of `CometVector`s.
82+
*
83+
* The input batch is not consumed or closed; the caller owns the returned batch and must close
84+
* it. Values are copied element-wise, since Spark's `ColumnVector` implementations do not
85+
* expose Arrow buffers.
86+
*/
87+
def columnarBatchToArrowBatch(
88+
batch: ColumnarBatch,
89+
schema: StructType,
90+
timeZoneId: String,
91+
allocator: BufferAllocator): ColumnarBatch = {
92+
val arrowSchema: Schema = Utils.toArrowSchema(schema, timeZoneId)
93+
val root = VectorSchemaRoot.create(arrowSchema, allocator)
94+
val writer = ArrowWriter.create(root)
95+
val numRows = batch.numRows()
96+
var col = 0
97+
while (col < batch.numCols()) {
98+
val column = batch.column(col)
99+
val columnArray = new ColumnarArray(column, 0, numRows)
100+
if (column.hasNull) {
101+
writer.writeCol(columnArray, col)
102+
} else {
103+
writer.writeColNoNull(columnArray, col)
104+
}
105+
col += 1
106+
}
107+
writer.finish()
108+
// ArrowWriter derives the root row count from its per-column writes, so a zero-column input
109+
// batch (Spark's count-from-metadata scan: numRows > 0, numCols == 0) would otherwise produce
110+
// a root with rowCount == 0 and silently drop the rows.
111+
root.setRowCount(numRows)
112+
NativeUtil.rootAsBatch(root)
113+
}
114+
115+
/** Whether every column in `batch` is already an Arrow-backed `CometVector`. */
116+
def isArrowBacked(batch: ColumnarBatch): Boolean =
117+
(0 until batch.numCols()).forall(i => batch.column(i).isInstanceOf[CometVector])
77118
}

‎spark/src/test/scala/org/apache/comet/exec/CometInMemoryCacheSuite.scala‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -688,4 +688,94 @@ class CometInMemoryCacheSuite extends CometTestBase {
688688
spark.catalog.clearCache()
689689
}
690690
}
691+
692+
test("cache a Spark columnar plan whose vectors are not Arrow-backed") {
693+
withTempPath { path =>
694+
spark
695+
.range(1000)
696+
.selectExpr(
697+
"id as key",
698+
"id % 8 as value",
699+
"cast(id as string) as s",
700+
"cast(id as double) as d",
701+
"cast(null as int) as n",
702+
"cast(id as decimal(20,3)) as dec",
703+
"date_add(date'2020-01-01', cast(id as int)) as dt",
704+
"timestamp_micros(id * 1000000) as ts")
705+
.write
706+
.parquet(path.toString)
707+
708+
withSQLConf(
709+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
710+
CometConf.COMET_SHUFFLE_MODE.key -> "jvm",
711+
SQLConf.CACHE_VECTORIZED_READER_ENABLED.key -> "true",
712+
CometConf.COMET_EXEC_IN_MEMORY_CACHE_ENABLED.key -> "true",
713+
// Force the cached plan to be Spark's own vectorized Parquet reader, which produces
714+
// On/OffHeapColumnVector rather than CometVector. Spark's InMemoryRelation strips the
715+
// ColumnarToRow above it because supportsColumnarInput is true for this schema, so the
716+
// serializer receives non-Arrow columnar batches.
717+
CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "false",
718+
CometConf.COMET_SPARK_TO_ARROW_ENABLED.key -> "false",
719+
SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
720+
SQLConf.SESSION_LOCAL_TIMEZONE.key -> "America/Denver") {
721+
722+
spark.catalog.clearCache()
723+
724+
spark.read.parquet(path.toString).createOrReplaceTempView("spark_columnar_cache")
725+
726+
spark.catalog.cacheTable("spark_columnar_cache")
727+
assert(spark.table("spark_columnar_cache").count() == 1000)
728+
729+
assert(
730+
cachedBatchTypes("spark_columnar_cache").sameElements(
731+
Array("org.apache.spark.sql.comet.execution.arrow.CometCachedBatch")))
732+
733+
checkSparkAnswer(
734+
spark.sql(
735+
"SELECT * FROM spark_columnar_cache WHERE key >= 10 AND key < 20 ORDER BY key"))
736+
checkSparkAnswer(
737+
spark.sql("SELECT sum(key), sum(d), sum(dec), count(s), count(n), max(dt), max(ts) " +
738+
"FROM spark_columnar_cache"))
739+
740+
spark.catalog.clearCache()
741+
}
742+
}
743+
}
744+
745+
test("cache a non-Arrow-backed Spark columnar plan with complex types") {
746+
withTempPath { path =>
747+
spark
748+
.range(200)
749+
.selectExpr(
750+
"id as key",
751+
"if(id % 5 = 0, null, array(id, id + 1)) as a",
752+
"named_struct('x', id, 'y', cast(id as string)) as st",
753+
"if(id % 7 = 0, null, map(cast(id as string), id)) as m",
754+
"cast(id as binary) as b")
755+
.write
756+
.parquet(path.toString)
757+
758+
withSQLConf(
759+
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
760+
CometConf.COMET_SHUFFLE_MODE.key -> "jvm",
761+
SQLConf.CACHE_VECTORIZED_READER_ENABLED.key -> "true",
762+
CometConf.COMET_EXEC_IN_MEMORY_CACHE_ENABLED.key -> "true",
763+
CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "false",
764+
CometConf.COMET_SPARK_TO_ARROW_ENABLED.key -> "false",
765+
SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
766+
SQLConf.PARQUET_VECTORIZED_READER_NESTED_COLUMN_ENABLED.key -> "true") {
767+
768+
spark.catalog.clearCache()
769+
770+
spark.read.parquet(path.toString).createOrReplaceTempView("spark_columnar_complex")
771+
772+
spark.catalog.cacheTable("spark_columnar_complex")
773+
assert(spark.table("spark_columnar_complex").count() == 200)
774+
775+
checkSparkAnswer(spark.sql("SELECT key, a, st, m, b FROM spark_columnar_complex"))
776+
777+
spark.catalog.clearCache()
778+
}
779+
}
780+
}
691781
}

0 commit comments

Comments
 (0)