@@ -638,6 +638,52 @@ class CometArrowWriterSuite extends AnyFunSuite with Matchers {
638638 }
639639 }
640640
641+ test(" a dictionary-encoded field appends as a column after its struct took the row path" ) {
642+ // A struct with a collection field takes the row path in a batch that holds null structs,
643+ // which leaves the offsets of its string field's trailing nulls for the next value to fill. A
644+ // dictionary-encoded field has to fill them, and grow its buffers, before it appends a column.
645+ val st = new StructType ().add(" a" , ArrayType (IntegerType )).add(" s" , StringType )
646+ val schema = new StructType ().add(" st" , st)
647+ val rnd = new Random (5 )
648+ val batches = Seq ((0.5 , 100 ), (0.0 , 5000 ), (0.5 , 100 ), (0.0 , 5000 )).map {
649+ case (nullFraction, n) =>
650+ val v = newVector(n, st, offHeap = false )
651+ (0 until n).foreach { i =>
652+ if (rnd.nextDouble() < nullFraction || (nullFraction > 0 && i == n - 1 )) {
653+ v.putNull(i)
654+ }
655+ }
656+ fill(v.getChild(0 ), ArrayType (IntegerType ), n, rnd, nullFraction = 0.0 , reversed = false )
657+ fillDictionary(v.getChild(1 ), StringType , n, rnd, nullFraction = 0.0 )
658+ // Spark's Parquet reader nulls the fields of a null struct.
659+ (0 until n).foreach { i =>
660+ if (v.isNullAt(i)) {
661+ v.getChild(0 ).putNull(i)
662+ v.getChild(1 ).putNull(i)
663+ }
664+ }
665+ new ColumnarBatch (Array [ColumnVector ](v), n)
666+ }
667+ val allocator = new RootAllocator (Long .MaxValue )
668+ val arrowSchema = Utils .toArrowSchema(schema, " UTC" )
669+ val columnar = VectorSchemaRoot .create(arrowSchema, allocator)
670+ val rows = VectorSchemaRoot .create(arrowSchema, allocator)
671+ try {
672+ val columnarWriter = ArrowWriter .create(columnar, 1 )
673+ batches.foreach(b => columnarWriter.writeColumns(b, 0 , b.numRows()))
674+ columnarWriter.finish()
675+ val rowWriter = ArrowWriter .create(rows, 1 )
676+ batches.foreach(b => (0 until b.numRows()).foreach(i => rowWriter.write(b.getRow(i))))
677+ rowWriter.finish()
678+ assertSameRoots(rows, columnar)
679+ } finally {
680+ columnar.close()
681+ rows.close()
682+ allocator.close()
683+ batches.foreach(_.close())
684+ }
685+ }
686+
641687 test(" a struct that switches between the columnar and row paths across appended batches" ) {
642688 // A struct with an array or a map field takes the row path only in batches that hold null
643689 // structs, so its fields' writers keep appending where the other path left off.
@@ -688,18 +734,19 @@ class CometArrowWriterSuite extends AnyFunSuite with Matchers {
688734 }
689735 }
690736
691- // Nested shapes whose unsafe forms take paths of their own: an array of each primitive type,
692- // whose elements are either copied in one block or converted one at a time, collections inside
693- // collections, and a struct wider than one word of null bits.
694- private val moreNestedTypes : Seq [DataType ] = primitiveTypes.map(ArrayType (_)) ++ Seq (
695- ArrayType (MapType (StringType , IntegerType )),
696- MapType (StringType , ArrayType (StringType )),
697- MapType (LongType , DecimalType (9 , 2 )),
698- new StructType ()
699- .add(" m" , MapType (IntegerType , StringType ))
700- .add(" s" , new StructType ().add(" x" , BinaryType ).add(" y" , DecimalType (18 , 4 ))),
701- StructType (
702- (0 until 70 ).map(i => StructField (s " f $i" , if (i % 7 == 3 ) StringType else LongType ))))
737+ // Nested shapes whose unsafe forms take paths of their own: an array of each primitive type not
738+ // in `nestedTypes`, whose elements are either copied in one block or converted one at a time,
739+ // collections inside collections, and a struct wider than one word of null bits.
740+ private val moreNestedTypes : Seq [DataType ] =
741+ primitiveTypes.map(ArrayType (_)).filterNot(nestedTypes.contains) ++ Seq (
742+ ArrayType (MapType (StringType , IntegerType )),
743+ MapType (StringType , ArrayType (StringType )),
744+ MapType (LongType , DecimalType (9 , 2 )),
745+ new StructType ()
746+ .add(" m" , MapType (IntegerType , StringType ))
747+ .add(" s" , new StructType ().add(" x" , BinaryType ).add(" y" , DecimalType (18 , 4 ))),
748+ StructType (
749+ (0 until 70 ).map(i => StructField (s " f $i" , if (i % 7 == 3 ) StringType else LongType ))))
703750
704751 /**
705752 * Writes `numRows` rows, `row(i)` through the generic row path and its unsafe projection
0 commit comments