Skip to content

Commit cb07053

Browse files
committed
fix: address review findings on the cache scan and serializer
Five issues from review of 86a91ea. Emit zero-column batches for an empty-output cache scan. The scan widened an empty projection to a placeholder column so the native plan would have a non-empty scan schema, but that claim was untrue: a zero-column scan plans and runs. Meanwhile the emitted batches disagreed with the declared output, so a join over a cached relation read the wrong column and silently returned a wrong sum rather than failing. Removing the widening fixes the join and makes count(*) decode nothing at all. Serialize each column with the dictionary provider it was decoded with. Columns decoded from separate streams have independent dictionary ID namespaces, so re-encoding a decoded batch with the first column's provider could not resolve the rest. Release readers opened before a later column fails to decode. The holder is published to the task-completion listener only after its constructor returns, so a partial failure leaked off-heap for the life of the executor. Build the cached RDD in doExecuteColumnar rather than at planning time. CachedRDDBuilder.cachedColumnBuffers executes the cached plan, so planning a query over an adaptively cached relation ran a job and finalized that plan during EXPLAIN. Report large-offset Arrow vectors as not directly writable. isArrowBacked accepted any CometVector, including one wrapping LargeVarCharVector or LargeVarBinaryVector, which getFieldVector rejects; such batches now take the conversion path instead of failing at cache materialization.
1 parent 86a91ea commit cb07053

5 files changed

Lines changed: 272 additions & 83 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/CometInMemoryTableScanExec.scala‎

Lines changed: 18 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,10 @@ import scala.collection.JavaConverters._
2323

2424
import org.apache.spark.rdd.RDD
2525
import org.apache.spark.sql.catalyst.expressions.Attribute
26-
import org.apache.spark.sql.columnar.{CachedBatch, CachedBatchSerializer}
26+
import org.apache.spark.sql.columnar.CachedBatchSerializer
2727
import org.apache.spark.sql.execution.LeafExecNode
28-
import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
28+
import org.apache.spark.sql.execution.columnar.{CachedRDDBuilder, InMemoryTableScanExec}
2929
import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
30-
import org.apache.spark.sql.types._
3130
import org.apache.spark.sql.vectorized.ColumnarBatch
3231

3332
import org.apache.comet.CometConf
@@ -49,7 +48,7 @@ import org.apache.comet.serde.QueryPlanSerde.serializeDataType
4948
case class CometInMemoryTableScanExec(
5049
originalPlan: InMemoryTableScanExec,
5150
serializer: CachedBatchSerializer,
52-
cachedBuffers: RDD[CachedBatch],
51+
cacheBuilder: CachedRDDBuilder,
5352
relationOutput: Seq[Attribute],
5453
scanOutput: Seq[Attribute])
5554
extends CometExec
@@ -58,10 +57,11 @@ case class CometInMemoryTableScanExec(
5857
override lazy val metrics: Map[String, SQLMetric] = Map(
5958
"numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of output rows"))
6059

61-
// For an empty-projection scan (`SELECT count(*)`) this is empty while `scanOutput` holds one
62-
// placeholder column, so the emitted batches are wider than the declared output. That is safe
63-
// because the only consumer of an empty-output scan is a count-style aggregate, which reads the
64-
// row count rather than any column; see `scanOutputFor` for why the scan cannot simply be empty.
60+
// `scanOutput` always equals this, including when it is empty. An empty-output scan
61+
// (`SELECT count(*)`) emits genuinely zero-column batches carrying only a row count: widening it
62+
// to a placeholder column, or to the whole cache schema, makes the emitted batches disagree with
63+
// the declared output, and a consumer that reads by ordinal rather than by row count -- a join,
64+
// for instance -- then reads the wrong column.
6565
override def output: Seq[Attribute] = originalPlan.output
6666

6767
// Use the serializer's vector types because the cached batch layout is owned by the serializer.
@@ -80,6 +80,13 @@ case class CometInMemoryTableScanExec(
8080
override def doExecuteColumnar(): RDD[ColumnarBatch] = {
8181
val numOutputRows = longMetric("numOutputRows")
8282

83+
// Resolved here rather than at planning time. CachedRDDBuilder.cachedColumnBuffers is not a
84+
// metadata lookup: it builds the RDD by calling execute/executeColumnar on the cached plan,
85+
// so touching it while Comet is still planning the outer query runs jobs during planning --
86+
// visibly, an EXPLAIN of a query over an adaptively-cached relation would launch a job and
87+
// finalize that plan.
88+
val cachedBuffers = cacheBuilder.cachedColumnBuffers
89+
8390
val filteredBuffers =
8491
if (originalPlan.predicates.nonEmpty && conf.inMemoryPartitionPruning) {
8592
val filter = serializer.buildFilter(originalPlan.predicates, relationOutput)
@@ -107,7 +114,7 @@ object CometInMemoryTableScanExec extends CometOperatorSerde[InMemoryTableScanEx
107114
builder: OperatorOuterClass.Operator.Builder,
108115
childOp: Operator*): Option[Operator] = {
109116

110-
val scanTypes = scanOutputFor(op).flatMap(attr => serializeDataType(attr.dataType))
117+
val scanTypes = op.output.flatMap(attr => serializeDataType(attr.dataType))
111118

112119
val scanBuilder = OperatorOuterClass.Scan
113120
.newBuilder()
@@ -127,43 +134,9 @@ object CometInMemoryTableScanExec extends CometOperatorSerde[InMemoryTableScanEx
127134
CometInMemoryTableScanExec(
128135
op,
129136
relation.cacheBuilder.serializer,
130-
relation.cacheBuilder.cachedColumnBuffers,
137+
relation.cacheBuilder,
131138
relation.output,
132-
scanOutputFor(op)))
133-
}
134-
135-
/**
136-
* Columns the cache scan asks the serializer to decode.
137-
*
138-
* An empty-output scan (`SELECT count(*)`) still needs a non-empty schema for native planning,
139-
* and the batches the node emits have to match that schema. Falling back to the whole cache
140-
* schema satisfies both, but the serializer decodes exactly what it is asked for, so the
141-
* cheapest query in the workload would decode every cached column. One column is enough: the
142-
* aggregate above an empty-output scan reads the row count and never a value, so pick the
143-
* cheapest to decode rather than all of them.
144-
*
145-
* `convert` and `createExec` must choose identically, or the native scan's declared schema and
146-
* the batches fed to it disagree.
147-
*/
148-
private def scanOutputFor(op: InMemoryTableScanExec): Seq[Attribute] = {
149-
if (op.output.nonEmpty) {
150-
op.output
151-
} else if (op.relation.output.isEmpty) {
152-
Nil
153-
} else {
154-
Seq(op.relation.output.minBy(a => decodeCostRank(a.dataType)))
155-
}
139+
op.output))
156140
}
157141

158-
// Rank by how much work decoding a column of this type costs, cheapest first. Fixed-width types
159-
// decode to a flat buffer; variable-width and nested ones carry offsets, children and possibly
160-
// dictionaries.
161-
private def decodeCostRank(dt: DataType): Int = dt match {
162-
case BooleanType | ByteType => 0
163-
case ShortType => 1
164-
case IntegerType | FloatType | DateType => 2
165-
case LongType | DoubleType | TimestampType | TimestampNTZType => 3
166-
case _: DecimalType => 4
167-
case _ => 5
168-
}
169142
}

‎spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/ArrowCachedBatchSerializer.scala‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.apache.spark.sql.comet.execution.arrow
2121

2222
import scala.collection.JavaConverters._
23+
import scala.util.control.NonFatal
2324

2425
import org.apache.spark.TaskContext
2526
import org.apache.spark.rdd.RDD
@@ -366,8 +367,34 @@ class ArrowCachedBatchSerializer extends SimpleMetricsCachedBatchSerializer {
366367
// this yields a single-element iterator that closes on exhaustion, matching what
367368
// ArrowReaderIterator did when the payload was one stream.
368369
private class ColumnReaders(buffers: Array[ChunkedByteBuffer], numRows: Int) {
369-
private val readers: Array[Iterator[ColumnarBatch]] =
370-
buffers.map(Utils.decodeBatches(_, "CometCache"))
370+
// decodeBatches opens a reader and eagerly decodes its first batch, so it allocates. If a
371+
// later column throws, the readers already opened here are unreachable: the task-completion
372+
// listener cannot release them because `current` is only assigned once this constructor
373+
// returns, so they would leak off-heap for the life of the executor.
374+
private val readers: Array[Iterator[ColumnarBatch]] = {
375+
val opened = new Array[Iterator[ColumnarBatch]](buffers.length)
376+
var i = 0
377+
try {
378+
while (i < buffers.length) {
379+
opened(i) = Utils.decodeBatches(buffers(i), "CometCache")
380+
i += 1
381+
}
382+
} catch {
383+
case NonFatal(e) =>
384+
var j = 0
385+
while (j < i) {
386+
opened(j) match {
387+
case reader: ArrowReaderIterator =>
388+
try reader.close()
389+
catch { case NonFatal(closeError) => e.addSuppressed(closeError) }
390+
case _ => ()
391+
}
392+
j += 1
393+
}
394+
throw e
395+
}
396+
opened
397+
}
371398
private var closed = false
372399

373400
def close(): Unit = synchronized {

‎spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala‎

Lines changed: 52 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -280,11 +280,12 @@ object Utils extends CometTypeShim with Logging {
280280
* it alongside. As with [[serializeBatches]], the batch's vectors are cleared once written.
281281
*/
282282
def serializeBatchColumns(batch: ColumnarBatch): Array[ChunkedByteBuffer] = {
283-
val (fieldVectors, batchProviderOpt) = getBatchFieldVectors(batch)
284-
val provider = batchProviderOpt.getOrElse(new CDataDictionaryProvider)
285283
val codec = CompressionCodec.createCodec(SparkEnv.get.conf)
286284

287-
fieldVectors.map { fieldVector =>
285+
// Each column is written with the provider it was decoded with, not the batch's first one:
286+
// columns decoded from separate streams have independent dictionary ID namespaces.
287+
getBatchFieldVectorsWithProviders(batch).map { case (fieldVector, providerOpt) =>
288+
val provider = providerOpt.getOrElse(new CDataDictionaryProvider)
288289
val cbbos = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate)
289290
val out = new DataOutputStream(codec.compressedOutputStream(cbbos))
290291

@@ -440,35 +441,58 @@ object Utils extends CometTypeShim with Logging {
440441
* rather than being materialized column by column.
441442
*/
442443
def isArrowBacked(batch: ColumnarBatch): Boolean =
443-
(0 until batch.numCols()).forall(i => batch.column(i).isInstanceOf[CometVector])
444+
(0 until batch.numCols()).forall { i =>
445+
batch.column(i) match {
446+
// Not every CometVector can be handed to getFieldVector: a CometPlainVector can wrap a
447+
// LargeVarCharVector or LargeVarBinaryVector (an accelerated mapInArrow returning
448+
// pa.large_string(), for instance), which it rejects. Answering true for those would
449+
// send a batch down the direct write path that then fails, so check the vector itself
450+
// and let the caller convert instead.
451+
case v: CometVector => isSupportedFieldVector(v.getValueVector)
452+
case _ => false
453+
}
454+
}
444455

445456
def getBatchFieldVectors(
446457
batch: ColumnarBatch): (Seq[FieldVector], Option[DictionaryProvider]) = {
447-
var provider: Option[DictionaryProvider] = None
458+
val columns = getBatchFieldVectorsWithProviders(batch)
459+
(columns.map(_._1), columns.flatMap(_._2).headOption)
460+
}
461+
462+
/**
463+
* Field vectors of `batch` paired with the dictionary provider each column was decoded with.
464+
*
465+
* [[getBatchFieldVectors]] collapses these to the first provider, which is right when every
466+
* column came from the same reader. Comet's cache decodes each column from its own stream, so a
467+
* batch's dictionary-backed columns can carry independent providers whose IDs collide; writing
468+
* such a batch back out with one column's provider cannot resolve the others. Callers that
469+
* serialize columns individually use this instead.
470+
*/
471+
def getBatchFieldVectorsWithProviders(
472+
batch: ColumnarBatch): Seq[(FieldVector, Option[DictionaryProvider])] = {
448473
val rows = batch.numRows()
449-
val fieldVectors = (0 until batch.numCols()).map { index =>
474+
(0 until batch.numCols()).map { index =>
450475
batch.column(index) match {
451476
case a: CometVector =>
452477
val valueVector = a.getValueVector
453-
if (valueVector.getField.getDictionary != null) {
454-
if (provider.isEmpty) {
455-
provider = Some(a.getDictionaryProvider)
456-
}
457-
}
478+
val provider =
479+
if (valueVector.getField.getDictionary != null) Some(a.getDictionaryProvider)
480+
else None
458481

459-
getFieldVector(valueVector, "serialize")
482+
(getFieldVector(valueVector, "serialize"), provider)
460483

461484
case cv: ConstantColumnVector =>
462485
// Spark wraps file-source partition columns and other per-batch constants in
463486
// `ConstantColumnVector`. Materialise to an Arrow vector so the serialisation path
464487
// doesn't reject the batch. "UTC" is intentional -- see `ConstantColumnVectors`.
465-
ConstantColumnVectors.materialize(
488+
val materialized = ConstantColumnVectors.materialize(
466489
cv,
467490
cv.dataType(),
468491
rows,
469492
s"_const_$index",
470493
org.apache.comet.CometArrowAllocator,
471494
"UTC")
495+
(materialized, None)
472496

473497
case c =>
474498
throw new SparkException(
@@ -482,19 +506,24 @@ object Utils extends CometTypeShim with Logging {
482506
"data to Arrow format automatically.")
483507
}
484508
}
485-
(fieldVectors, provider)
509+
}
510+
511+
/** Whether [[getFieldVector]] accepts this vector, without throwing to find out. */
512+
def isSupportedFieldVector(valueVector: ValueVector): Boolean = valueVector match {
513+
case _: BitVector | _: TinyIntVector | _: SmallIntVector | _: IntVector | _: BigIntVector |
514+
_: Float4Vector | _: Float8Vector | _: VarCharVector | _: DecimalVector |
515+
_: DateDayVector | _: TimeStampMicroTZVector | _: VarBinaryVector |
516+
_: FixedSizeBinaryVector | _: TimeStampMicroVector | _: StructVector | _: ListVector |
517+
_: MapVector | _: NullVector | _: TimeNanoVector =>
518+
true
519+
case _ => false
486520
}
487521

488522
def getFieldVector(valueVector: ValueVector, reason: String): FieldVector = {
489-
valueVector match {
490-
case v @ (_: BitVector | _: TinyIntVector | _: SmallIntVector | _: IntVector |
491-
_: BigIntVector | _: Float4Vector | _: Float8Vector | _: VarCharVector |
492-
_: DecimalVector | _: DateDayVector | _: TimeStampMicroTZVector | _: VarBinaryVector |
493-
_: FixedSizeBinaryVector | _: TimeStampMicroVector | _: StructVector | _: ListVector |
494-
_: MapVector | _: NullVector | _: TimeNanoVector) =>
495-
v.asInstanceOf[FieldVector]
496-
case _ =>
497-
throw new SparkException(s"Unsupported Arrow Vector for $reason: ${valueVector.getClass}")
523+
if (isSupportedFieldVector(valueVector)) {
524+
valueVector.asInstanceOf[FieldVector]
525+
} else {
526+
throw new SparkException(s"Unsupported Arrow Vector for $reason: ${valueVector.getClass}")
498527
}
499528
}
500529
}

0 commit comments

Comments
 (0)