Skip to content

Commit 175fb30

Browse files
committed
test: cover growing pages under memory pressure; update manyPartitions comment
1 parent b523cf1 commit 175fb30

2 files changed

Lines changed: 71 additions & 3 deletions

File tree

‎spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,10 @@ object CometShuffleBenchmark extends CometBenchmarkBase {
5757
/**
5858
* High partition count for the groups that measure both a low and a high fan-out. This must
5959
* stay above Spark's `spark.shuffle.sort.bypassMergeThreshold` (200). Below that threshold the
60-
* write path switches to `CometBypassMergeSortShuffleWriter`, which holds a page per partition
61-
* from a JVM-wide pool, so a lower value both raises the shuffle memory requirement several
62-
* fold and hides the JVM shuffle's degradation at high fan-out.
60+
* write path switches to `CometBypassMergeSortShuffleWriter`, which writes a separate file per
61+
* partition instead of sorting. The high fan-out group is meant to measure the sort-based
62+
* writer, so a lower value would measure the bypass writer and hide the sort-based writer's
63+
* degradation at high fan-out.
6364
*/
6465
private val manyPartitions = 201
6566

‎spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriterSuite.scala‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,73 @@ class CometDiskBlockWriterSuite extends AnyFunSuite {
342342
}
343343
}
344344

345+
test("growing pages under real memory pressure spill but still write full batches") {
346+
// The partitions' batches together do not fit, so writers must spill, and a growing writer
347+
// whose next page is refused spills itself and restarts at the small page. Spilling is
348+
// legitimate here, but it must not degrade to writing about one row per batch (#7058).
349+
val conf = new SparkConf()
350+
.set("spark.memory.offHeap.enabled", "true")
351+
.set("spark.memory.offHeap.size", "1g")
352+
val memoryManager = new TestMemoryManager(conf)
353+
val kib = 1024L
354+
val initialPageSize = 64 * kib
355+
val maxPageSize = 1024 * kib
356+
// 32 partitions fill about 8 MiB below, so this holds well under half of the batches.
357+
memoryManager.limit(3 * 1024 * kib)
358+
val tmm = new TaskMemoryManager(memoryManager, 0L)
359+
val taskContext = newTaskContext(tmm, 0L)
360+
val allocator = CometShuffleMemoryAllocator.getInstance(tmm, initialPageSize)
361+
val tempDir = Utils.createTempDir()
362+
try {
363+
val partitions = 32
364+
val rowsPerPartition = 2000
365+
val currentWriters = new JLinkedList[CometDiskBlockWriter]()
366+
val serializer = new UnsafeRowSerializer(1).newInstance()
367+
val writers = (0 until partitions).map { partition =>
368+
new CometDiskBlockWriter(
369+
new File(tempDir, s"partition$partition"),
370+
allocator,
371+
initialPageSize,
372+
maxPageSize,
373+
taskContext,
374+
serializer,
375+
schema,
376+
new ShuffleWriteMetrics,
377+
conf,
378+
false,
379+
currentWriters)
380+
}
381+
382+
val toUnsafe = UnsafeProjection.create(schema)
383+
// Round-robin, so that each partition gets a row before any gets a second one. A write
384+
// under memory pressure can be of any writer, so each insert checks all of them.
385+
val written = Array.fill(partitions)(0L)
386+
var batches = 0L
387+
var rowsInBatches = 0L
388+
for (_ <- 0 until rowsPerPartition; partition <- 0 until partitions) {
389+
writers(partition).insertRow(toUnsafe(InternalRow(new Array[Byte](100))), 0)
390+
for (i <- 0 until partitions if writers(i).getOutputRecords > written(i)) {
391+
batches += 1
392+
rowsInBatches += writers(i).getOutputRecords - written(i)
393+
written(i) = writers(i).getOutputRecords
394+
}
395+
}
396+
writers.foreach(_.close())
397+
398+
assert(writers.map(_.getOutputRecords).sum == partitions * rowsPerPartition)
399+
assert(taskContext.taskMetrics.memoryBytesSpilled > 0)
400+
assert(taskContext.taskMetrics.diskBytesSpilled > 0)
401+
assert(tmm.getMemoryConsumptionForThisTask == 0)
402+
assert(allocator.getUsed == 0)
403+
// Writing a page's worth of rows is far more than one row. Pages of Spark's size, when
404+
// every writer held one from its first row, gave about one row per batch here.
405+
assert(batches > 0 && rowsInBatches / batches >= 100, s"$rowsInBatches rows in $batches")
406+
} finally {
407+
Utils.deleteRecursively(tempDir)
408+
tmm.cleanUpAllAllocatedMemory()
409+
}
410+
}
411+
345412
test("a fatal error during write() frees the task's buffered pages") {
346413
// Spark's ShuffleWriteProcessor only calls stop(false) when write() throws an Exception, so a
347414
// fatal error such as SparkOutOfMemoryError skips it. write() itself must therefore free the

0 commit comments

Comments
 (0)