@@ -615,6 +615,63 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper
615615 }
616616 }
617617
618+ test(" native shuffle on struct data column" ) {
619+ // The native shuffle type gate (CometShuffleExchangeExec.supportedSerializableDataType)
620+ // allows struct data columns, but nothing exercised them: struct columns can only reach
621+ // native shuffle as DATA, since supportedHashPartitioningDataType rejects them as a
622+ // hash key. So partition on a primitive and carry the struct along.
623+ Seq (10 , 201 ).foreach { numPartitions =>
624+ withParquetTable((0 until 50 ).map(i => (i, (i + 1 , (i + 2 ).toString), i + 3 )), " tbl" ) {
625+ val df = sql(" SELECT * FROM tbl" )
626+ .filter($" _3" > 10 )
627+ .repartition(numPartitions, $" _1" )
628+ .sortWithinPartitions($" _1" )
629+
630+ checkShuffleAnswer(df, 1 )
631+ }
632+ }
633+ }
634+
635+ test(" native shuffle on struct data column including nulls" ) {
636+ Seq (10 , 201 ).foreach { numPartitions =>
637+ val data : Seq [(Int , (Int , String ))] =
638+ Seq ((1 , (0 , " 1" )), (2 , (3 , " 3" )), (3 , null ), (4 , (5 , null )))
639+ withParquetTable(data, " tbl" ) {
640+ val df = sql(" SELECT * FROM tbl" )
641+ .repartition(numPartitions, $" _1" )
642+ .sortWithinPartitions($" _1" )
643+
644+ checkShuffleAnswer(df, 1 )
645+ }
646+ }
647+ }
648+
649+ test(" native shuffle on deeply nested data columns" ) {
650+ // struct<array<...>> and array<struct<...>>: the recursive branches of the type gate.
651+ Seq (10 , 201 ).foreach { numPartitions =>
652+ withParquetTable(
653+ (0 until 50 ).map(i =>
654+ (i, (Seq (i + 1 , i + 2 ), (i + 3 ).toString), Seq ((i + 4 , (i + 5 ).toString)))),
655+ " tbl" ) {
656+ val df = sql(" SELECT * FROM tbl" )
657+ .repartition(numPartitions, $" _1" )
658+ .sortWithinPartitions($" _1" )
659+
660+ checkShuffleAnswer(df, 1 )
661+ }
662+ }
663+ }
664+
665+ test(" native shuffle on struct data column with map field" ) {
666+ withSQLConf(CometConf .COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED .key -> " true" ) {
667+ val df = spark.sql(
668+ " SELECT id, named_struct('m', map(id, id + 1), 's', cast(id AS STRING)) AS st " +
669+ " FROM VALUES (1), (2), (3) AS t(id)" )
670+ val shuffled = df.repartition(2 , $" id" )
671+ checkShuffleAnswer(shuffled, 1 )
672+ }
673+ }
674+
618675 test(" fix: Comet native shuffle with binary data" ) {
619676 withParquetTable((0 until 5 ).map(i => (i, (i + 1 ).toLong)), " tbl" ) {
620677 val df = sql(" SELECT cast(cast(_1 as STRING) as BINARY) as binary, _2 FROM tbl" )
0 commit comments