Skip to content

Commit 2d5e9cf

Browse files
committed
test: count a range-partitioned conversion's rows once, and range-partition every column in the benchmark
Before the sampling change, the range partitioner's sampling job ran through the conversion and added its rows to the conversion's metrics, so a range-partitioned shuffle reported twice the rows it wrote.
1 parent ef1d8ec commit 2d5e9cf

2 files changed

Lines changed: 6 additions & 3 deletions

File tree

‎spark/src/test/scala/org/apache/comet/exec/CometShuffleInputConversionSuite.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -396,6 +396,9 @@ class CometShuffleInputConversionSuite extends CometTestBase {
396396
convertTest("range partitioning") {
397397
val (_, plan) = checkSparkAnswer(rowsDf().orderBy(col("l").desc))
398398
assert(convertedShuffles(plan).length == 1, plan)
399+
// The range partitioner samples the rows that the conversion reads, without converting them,
400+
// so the conversion only counts the rows that the shuffle writes.
401+
assert(conversions(plan).head.metrics("numInputRows").value == 1000, plan)
399402
}
400403

401404
convertTest("the conversion only takes over from the JVM columnar shuffle") {

‎spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleInputConversionBenchmark.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -131,12 +131,12 @@ object CometShuffleInputConversionBenchmark extends CometBenchmarkBase {
131131
.repartition(numPartitions, col("k"))
132132
.groupBy("k")
133133
.agg((sum("l") + sum(length(col("s"))) + sum("d") + sum("m")).as("s")))),
134-
"RDD rows: int, long, double, range partitioned" -> (() =>
134+
"RDD rows: int, long, string, double, decimal(18,2), range partitioned" -> (() =>
135135
total(
136136
rddRows()
137-
.select("k", "l", "d")
138137
.repartitionByRange(numPartitions, col("l"))
139-
.select((col("l") + col("d")).as("s")))),
138+
.select((col("k") + col("l") + length(col("s")) + col("d").cast("long") + col("m"))
139+
.as("s")))),
140140
"map over a Parquet scan" -> (() =>
141141
total(
142142
spark

0 commit comments

Comments
 (0)