@@ -755,9 +755,11 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar
755755 }
756756
757757 test(" columnar shuffle stores offsets as lengths only with zstd" ) {
758- withParquetTable(( 0 until 40000 ).map(i => (i, s " str $i " , Seq ( s " a $i " , s " bb $i " )) ), " tbl" ) {
758+ withParquetTable(rowsWithOffsets( 40000 ), " tbl" ) {
759759 // 10 partitions take the bypass-merge writer, 201 the sort-based one
760760 Seq (10 , 201 ).foreach { numPartitions =>
761+ def shuffled : DataFrame = sql(" SELECT * FROM tbl" ).repartition(numPartitions, $" _1" )
762+
761763 def bytesWritten (codec : String , offsetsAsLengths : Boolean ): Long = {
762764 var bytes = 0L
763765 withSQLConf(
@@ -766,9 +768,9 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar
766768 CometConf .COMET_SHUFFLE_COMPRESSION_CODEC .key -> codec,
767769 CometConf .COMET_SHUFFLE_COMPRESSION_OFFSETS_AS_LENGTHS .key ->
768770 offsetsAsLengths.toString) {
769- val shuffled = sql( " SELECT * FROM tbl " ).repartition(numPartitions, $ " _1 " )
770- checkShuffleAnswer(shuffled , 1 )
771- bytes = cometShuffleMetric(shuffled , " shuffleBytesWritten" )
771+ val df = shuffled
772+ checkShuffleAnswer(df , 1 )
773+ bytes = cometShuffleMetric(df , " shuffleBytesWritten" )
772774 }
773775 bytes
774776 }
@@ -781,6 +783,14 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar
781783 " offsets" )
782784 // LZ4 does not take lengths yet, so the setting must not change what it writes
783785 assert(bytesWritten(" lz4" , offsetsAsLengths = true ) == bytesWritten(" lz4" , false ))
786+
787+ // At the suite's own threshold, every partition gets many blocks of a few rows, which
788+ // the sort-based writer spills to files and merges
789+ withSQLConf(
790+ CometConf .COMET_SHUFFLE_COMPRESSION_CODEC .key -> " zstd" ,
791+ CometConf .COMET_SHUFFLE_COMPRESSION_OFFSETS_AS_LENGTHS .key -> " true" ) {
792+ checkShuffleAnswer(shuffled, 1 )
793+ }
784794 }
785795 }
786796 }
0 commit comments