@@ -26,8 +26,10 @@ import scala.jdk.CollectionConverters._
2626
2727import org .apache .arrow .c .CDataDictionaryProvider
2828import org .apache .arrow .memory .BufferAllocator
29- import org .apache .arrow .vector .{BaseLargeVariableWidthVector , BitVectorHelper , LargeVarBinaryVector , LargeVarCharVector , ValueVector , VarBinaryVector , VarCharVector , VariableWidthFieldVector }
29+ import org .apache .arrow .vector .{BaseLargeVariableWidthVector , BitVectorHelper , IntVector , LargeVarBinaryVector , LargeVarCharVector , ValueVector , VarBinaryVector , VarCharVector , VariableWidthFieldVector }
30+ import org .apache .arrow .vector .dictionary .{Dictionary , DictionaryProvider }
3031import org .apache .arrow .vector .ipc .message .ArrowFieldNode
32+ import org .apache .arrow .vector .types .pojo .{ArrowType , DictionaryEncoding , Field , FieldType }
3133import org .apache .spark .SparkException
3234import org .apache .spark .sql .CometTestBase
3335import org .apache .spark .sql .execution .vectorized .ConstantColumnVector
@@ -237,7 +239,9 @@ class UtilsSuite extends CometTestBase {
237239 // mapInArrow returning pa.large_string() produces one -- but getFieldVector rejects both, and
238240 // getBatchFieldVectors writes a 32-bit copy of them rather than the vector itself. A caller
239241 // taking the direct write path expects the batch's own vectors, so isArrowBacked must report
240- // false and let the caller convert the batch instead.
242+ // false and let the caller convert the batch instead. The cache's direct write path also asks
243+ // CachedBatchIpc.matchesReaderLayout, which refuses LargeUtf8 against the reader's Utf8, so
244+ // either check alone keeps such a batch off it.
241245 val values = Seq (" hello" , " world" )
242246 Seq [ValueVector ](
243247 fill(new LargeVarCharVector (" s" , CometArrowAllocator ), values),
@@ -314,7 +318,7 @@ class UtilsSuite extends CometTestBase {
314318 }
315319 }
316320
317- test(" coalesceBroadcastBatches appends a large-offset batch to a 32-bit one " ) {
321+ test(" coalesceBroadcastBatches coalesces batches of mixed offset widths " ) {
318322 // The coalescer appends every batch to a root built from the first one's schema, and leaves
319323 // the broadcast uncoalesced when a batch does not match it. A producer can hand back
320324 // large_string for one batch and string for the next, so both must serialize alike.
@@ -339,19 +343,6 @@ class UtilsSuite extends CometTestBase {
339343 }
340344 }
341345
342- test(" serializeBatches refuses a large-offset column too big for 32-bit offsets" ) {
343- // The offsets alone claim 2 GiB of data. The size is checked before anything is copied, so
344- // the data itself never has to exist.
345- val strings = tooBigForIntOffsets(" s" )
346- try {
347- val batch = new ColumnarBatch (Array (cometVector(strings)), 1 )
348- val e = intercept[SparkException ](Utils .serializeBatches(Iterator (batch)).next())
349- assert(e.getMessage.contains(" more than 32-bit offsets can address" ), e.getMessage)
350- } finally {
351- strings.close()
352- }
353- }
354-
355346 test(" serializeBatches releases the vectors it made when a later column fails" ) {
356347 // A materialized constant and a 32-bit copy of a large-offset column are both made before the
357348 // last column is refused. Nothing outside serializeBatches ever holds them, so it has to
@@ -374,6 +365,49 @@ class UtilsSuite extends CometTestBase {
374365 assert(CometArrowAllocator .getAllocatedMemory == before)
375366 }
376367
368+ test(" serializeBatches releases the vectors it made when the dictionaries of a batch clash" ) {
369+ // The vectors are all made before the dictionaries of the columns are combined, which is where
370+ // two columns sharing an ID without sharing a dictionary are refused. By then the caller has
371+ // not got them either, and a copy of a large-offset column can be close to 2 GiB.
372+ val before = CometArrowAllocator .getAllocatedMemory
373+ val large = fill(new LargeVarCharVector (" large" , CometArrowAllocator ), Seq (" a" ))
374+ val (first, firstProvider) = dictionaryEncoded(" first" , 7L , " x" )
375+ val (second, secondProvider) = dictionaryEncoded(" second" , 7L , " y" )
376+ try {
377+ val batch = new ColumnarBatch (
378+ Array (
379+ cometVector(large),
380+ CometVector .getVector(first, firstProvider),
381+ CometVector .getVector(second, secondProvider)),
382+ 1 )
383+ val e = intercept[SparkException ](Utils .serializeBatches(Iterator (batch)).next())
384+ assert(e.getMessage.contains(" different dictionaries under ID 7" ), e.getMessage)
385+ assert(large.getValueCount == 1 && large.getObject(0 ).toString == " a" )
386+ } finally {
387+ large.close()
388+ Seq (first, second).foreach(_.close())
389+ Seq (firstProvider, secondProvider).foreach(_.lookup(7L ).getVector.close())
390+ }
391+ assert(CometArrowAllocator .getAllocatedMemory == before)
392+ }
393+
394+ /** A dictionary-encoded index column holding the one entry of a dictionary holding `value`. */
395+ private def dictionaryEncoded (
396+ name : String ,
397+ id : Long ,
398+ value : String ): (IntVector , DictionaryProvider ) = {
399+ val encoding = new DictionaryEncoding (id, false , null )
400+ val dictionary =
401+ new Dictionary (fill(new VarCharVector (" dict" , CometArrowAllocator ), Seq (value)), encoding)
402+ val indices = new IntVector (
403+ new Field (name, new FieldType (true , new ArrowType .Int (32 , true ), encoding), null ),
404+ CometArrowAllocator )
405+ indices.allocateNew(1 )
406+ indices.set(0 , 0 )
407+ indices.setValueCount(1 )
408+ (indices, new DictionaryProvider .MapDictionaryProvider (dictionary))
409+ }
410+
377411 /** Writes `values` into `vector`, leaving a null entry unset so that it reads back as null. */
378412 private def fill [V <: VariableWidthFieldVector ](vector : V , values : Seq [String ]): V = {
379413 vector.allocateNew(values.length)
0 commit comments