Skip to content

Commit 9afe4eb

Browse files
authored
Merge branch 'main' into perf/shuffle-compressor-reuse
2 parents abf5807 + 199a910 commit 9afe4eb

1 file changed

Lines changed: 57 additions & 0 deletions

File tree

‎spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,6 +615,63 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper
615615
}
616616
}
617617

618+
test("native shuffle on struct data column") {
619+
// The native shuffle type gate (CometShuffleExchangeExec.supportedSerializableDataType)
620+
// allows struct data columns, but nothing exercised them: struct columns can only reach
621+
// native shuffle as DATA, since supportedHashPartitioningDataType rejects them as a
622+
// hash key. So partition on a primitive and carry the struct along.
623+
Seq(10, 201).foreach { numPartitions =>
624+
withParquetTable((0 until 50).map(i => (i, (i + 1, (i + 2).toString), i + 3)), "tbl") {
625+
val df = sql("SELECT * FROM tbl")
626+
.filter($"_3" > 10)
627+
.repartition(numPartitions, $"_1")
628+
.sortWithinPartitions($"_1")
629+
630+
checkShuffleAnswer(df, 1)
631+
}
632+
}
633+
}
634+
635+
test("native shuffle on struct data column including nulls") {
636+
Seq(10, 201).foreach { numPartitions =>
637+
val data: Seq[(Int, (Int, String))] =
638+
Seq((1, (0, "1")), (2, (3, "3")), (3, null), (4, (5, null)))
639+
withParquetTable(data, "tbl") {
640+
val df = sql("SELECT * FROM tbl")
641+
.repartition(numPartitions, $"_1")
642+
.sortWithinPartitions($"_1")
643+
644+
checkShuffleAnswer(df, 1)
645+
}
646+
}
647+
}
648+
649+
test("native shuffle on deeply nested data columns") {
650+
// struct<array<...>> and array<struct<...>>: the recursive branches of the type gate.
651+
Seq(10, 201).foreach { numPartitions =>
652+
withParquetTable(
653+
(0 until 50).map(i =>
654+
(i, (Seq(i + 1, i + 2), (i + 3).toString), Seq((i + 4, (i + 5).toString)))),
655+
"tbl") {
656+
val df = sql("SELECT * FROM tbl")
657+
.repartition(numPartitions, $"_1")
658+
.sortWithinPartitions($"_1")
659+
660+
checkShuffleAnswer(df, 1)
661+
}
662+
}
663+
}
664+
665+
test("native shuffle on struct data column with map field") {
666+
withSQLConf(CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true") {
667+
val df = spark.sql(
668+
"SELECT id, named_struct('m', map(id, id + 1), 's', cast(id AS STRING)) AS st " +
669+
"FROM VALUES (1), (2), (3) AS t(id)")
670+
val shuffled = df.repartition(2, $"id")
671+
checkShuffleAnswer(shuffled, 1)
672+
}
673+
}
674+
618675
test("fix: Comet native shuffle with binary data") {
619676
withParquetTable((0 until 5).map(i => (i, (i + 1).toLong)), "tbl") {
620677
val df = sql("SELECT cast(cast(_1 as STRING) as BINARY) as binary, _2 FROM tbl")

0 commit comments

Comments
 (0)