Skip to content

Commit 199a910

Browse files
viiryaclaude
andauthored
test: cover struct data columns in native shuffle (#5564)
`CometShuffleExchangeExec.supportedSerializableDataType` allows struct data columns in native shuffle, but `CometNativeShuffleSuite` never exercised one: the only nested coverage was `array<array<int>>` and `map<int, null>`, and the word "struct" did not appear in the suite at all. The columnar suite has extensive nested coverage, so the gap was specific to native shuffle. Struct columns can only reach native shuffle as DATA columns, since `supportedHashPartitioningDataType` rejects nested types as a hash key, so these tests partition on a primitive and carry the struct along. Adds four tests: - a plain `struct<int, string>` data column - a struct data column with a null struct and a null field inside a struct - `struct<array<int>, string>` and `array<struct<int, string>>`, covering the recursive branches of the type gate - a struct data column with a map field Co-authored-by: Claude Code <[email protected]>
1 parent 918bc7d commit 199a910

1 file changed

Lines changed: 57 additions & 0 deletions

File tree

‎spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,6 +615,63 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper
615615
}
616616
}
617617

618+
test("native shuffle on struct data column") {
619+
// The native shuffle type gate (CometShuffleExchangeExec.supportedSerializableDataType)
620+
// allows struct data columns, but nothing exercised them: struct columns can only reach
621+
// native shuffle as DATA, since supportedHashPartitioningDataType rejects them as a
622+
// hash key. So partition on a primitive and carry the struct along.
623+
Seq(10, 201).foreach { numPartitions =>
624+
withParquetTable((0 until 50).map(i => (i, (i + 1, (i + 2).toString), i + 3)), "tbl") {
625+
val df = sql("SELECT * FROM tbl")
626+
.filter($"_3" > 10)
627+
.repartition(numPartitions, $"_1")
628+
.sortWithinPartitions($"_1")
629+
630+
checkShuffleAnswer(df, 1)
631+
}
632+
}
633+
}
634+
635+
test("native shuffle on struct data column including nulls") {
636+
Seq(10, 201).foreach { numPartitions =>
637+
val data: Seq[(Int, (Int, String))] =
638+
Seq((1, (0, "1")), (2, (3, "3")), (3, null), (4, (5, null)))
639+
withParquetTable(data, "tbl") {
640+
val df = sql("SELECT * FROM tbl")
641+
.repartition(numPartitions, $"_1")
642+
.sortWithinPartitions($"_1")
643+
644+
checkShuffleAnswer(df, 1)
645+
}
646+
}
647+
}
648+
649+
test("native shuffle on deeply nested data columns") {
650+
// struct<array<...>> and array<struct<...>>: the recursive branches of the type gate.
651+
Seq(10, 201).foreach { numPartitions =>
652+
withParquetTable(
653+
(0 until 50).map(i =>
654+
(i, (Seq(i + 1, i + 2), (i + 3).toString), Seq((i + 4, (i + 5).toString)))),
655+
"tbl") {
656+
val df = sql("SELECT * FROM tbl")
657+
.repartition(numPartitions, $"_1")
658+
.sortWithinPartitions($"_1")
659+
660+
checkShuffleAnswer(df, 1)
661+
}
662+
}
663+
}
664+
665+
test("native shuffle on struct data column with map field") {
666+
withSQLConf(CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true") {
667+
val df = spark.sql(
668+
"SELECT id, named_struct('m', map(id, id + 1), 's', cast(id AS STRING)) AS st " +
669+
"FROM VALUES (1), (2), (3) AS t(id)")
670+
val shuffled = df.repartition(2, $"id")
671+
checkShuffleAnswer(shuffled, 1)
672+
}
673+
}
674+
618675
test("fix: Comet native shuffle with binary data") {
619676
withParquetTable((0 until 5).map(i => (i, (i + 1).toLong)), "tbl") {
620677
val df = sql("SELECT cast(cast(_1 as STRING) as BINARY) as binary, _2 FROM tbl")

0 commit comments

Comments
 (0)