Skip to content

Commit 057f7a5

Browse files
committed
test: cover nulls, empty values, binary and struct columns and spilling in the offsets-as-lengths tests
1 parent b41b08d commit 057f7a5

3 files changed

Lines changed: 53 additions & 10 deletions

File tree

‎spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -755,9 +755,11 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar
755755
}
756756

757757
test("columnar shuffle stores offsets as lengths only with zstd") {
758-
withParquetTable((0 until 40000).map(i => (i, s"str$i", Seq(s"a$i", s"bb$i"))), "tbl") {
758+
withParquetTable(rowsWithOffsets(40000), "tbl") {
759759
// 10 partitions take the bypass-merge writer, 201 the sort-based one
760760
Seq(10, 201).foreach { numPartitions =>
761+
def shuffled: DataFrame = sql("SELECT * FROM tbl").repartition(numPartitions, $"_1")
762+
761763
def bytesWritten(codec: String, offsetsAsLengths: Boolean): Long = {
762764
var bytes = 0L
763765
withSQLConf(
@@ -766,9 +768,9 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar
766768
CometConf.COMET_SHUFFLE_COMPRESSION_CODEC.key -> codec,
767769
CometConf.COMET_SHUFFLE_COMPRESSION_OFFSETS_AS_LENGTHS.key ->
768770
offsetsAsLengths.toString) {
769-
val shuffled = sql("SELECT * FROM tbl").repartition(numPartitions, $"_1")
770-
checkShuffleAnswer(shuffled, 1)
771-
bytes = cometShuffleMetric(shuffled, "shuffleBytesWritten")
771+
val df = shuffled
772+
checkShuffleAnswer(df, 1)
773+
bytes = cometShuffleMetric(df, "shuffleBytesWritten")
772774
}
773775
bytes
774776
}
@@ -781,6 +783,14 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar
781783
"offsets")
782784
// LZ4 does not take lengths yet, so the setting must not change what it writes
783785
assert(bytesWritten("lz4", offsetsAsLengths = true) == bytesWritten("lz4", false))
786+
787+
// At the suite's own threshold, every partition gets many blocks of a few rows, which
788+
// the sort-based writer spills to files and merges
789+
withSQLConf(
790+
CometConf.COMET_SHUFFLE_COMPRESSION_CODEC.key -> "zstd",
791+
CometConf.COMET_SHUFFLE_COMPRESSION_OFFSETS_AS_LENGTHS.key -> "true") {
792+
checkShuffleAnswer(shuffled, 1)
793+
}
784794
}
785795
}
786796
}

‎spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1564,17 +1564,17 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper
15641564
}
15651565

15661566
test("native shuffle stores offsets as lengths only with zstd") {
1567-
withParquetTable(
1568-
(0 until 20000).map(i => (i, s"str$i", Seq(s"a$i", s"bb$i"), Map(s"k$i" -> i))),
1569-
"tbl") {
1567+
withParquetTable(rowsWithOffsets(20000), "tbl") {
1568+
def shuffled: DataFrame = sql("SELECT * FROM tbl").repartition(10, $"_1")
1569+
15701570
def bytesWritten(codec: String, offsetsAsLengths: Boolean): Long = {
15711571
var bytes = 0L
15721572
withSQLConf(
15731573
CometConf.COMET_SHUFFLE_COMPRESSION_CODEC.key -> codec,
15741574
CometConf.COMET_SHUFFLE_COMPRESSION_OFFSETS_AS_LENGTHS.key -> offsetsAsLengths.toString) {
1575-
val shuffled = sql("SELECT * FROM tbl").repartition(10, $"_1")
1576-
checkShuffleAnswer(shuffled, 1)
1577-
bytes = cometShuffleMetric(shuffled, "shuffleBytesWritten")
1575+
val df = shuffled
1576+
checkShuffleAnswer(df, 1)
1577+
bytes = cometShuffleMetric(df, "shuffleBytesWritten")
15781578
}
15791579
bytes
15801580
}
@@ -1584,6 +1584,16 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper
15841584
assert(lengths < offsets, s"zstd wrote $lengths bytes with lengths, $offsets with offsets")
15851585
// LZ4 does not take lengths yet, so the setting must not change what it writes
15861586
assert(bytesWritten("lz4", offsetsAsLengths = true) == bytesWritten("lz4", false))
1587+
1588+
// A small buffer limit makes the writer spill, so blocks also pass through the spill file
1589+
withSQLConf(
1590+
CometConf.COMET_SHUFFLE_COMPRESSION_CODEC.key -> "zstd",
1591+
CometConf.COMET_SHUFFLE_COMPRESSION_OFFSETS_AS_LENGTHS.key -> "true",
1592+
CometConf.COMET_SHUFFLE_NATIVE_MAX_BUFFER_BYTES.key -> "64k") {
1593+
val df = shuffled
1594+
checkShuffleAnswer(df, 1)
1595+
assert(cometShuffleMetric(df, "spill_count") > 0)
1596+
}
15871597
}
15881598
}
15891599

‎spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1357,6 +1357,29 @@ abstract class CometTestBase
13571357
}.get
13581358
}
13591359

1360+
/**
1361+
* Rows for shuffle tests of offsets stored as lengths, with offsets in a string, a binary, an
1362+
* array, a map and a struct column. Every tenth string is null, and so are the binary, array
1363+
* and map values next to it. The string after it is empty, and every array holds a null.
1364+
*/
1365+
protected def rowsWithOffsets(
1366+
n: Int): Seq[(Int, String, Array[Byte], Seq[String], Map[String, String], (String, Int))] =
1367+
(0 until n).map { i =>
1368+
val s = i % 10 match {
1369+
case 0 => null
1370+
case 1 => ""
1371+
case _ => s"str$i"
1372+
}
1373+
val value = Option(s)
1374+
(
1375+
i,
1376+
s,
1377+
value.map(_.getBytes("UTF-8")).orNull,
1378+
value.map(v => Seq(v, null, s"a$i")).orNull,
1379+
value.map(v => Map(s"k$i" -> v)).orNull,
1380+
(s, i))
1381+
}
1382+
13601383
protected def checkCometExchange(
13611384
df: DataFrame,
13621385
cometExchangeNum: Int,

0 commit comments

Comments
 (0)