@@ -342,6 +342,73 @@ class CometDiskBlockWriterSuite extends AnyFunSuite {
342342 }
343343 }
344344
345+ test(" growing pages under real memory pressure spill but still write full batches" ) {
346+ // The partitions' batches together do not fit, so writers must spill, and a growing writer
347+ // whose next page is refused spills itself and restarts at the small page. Spilling is
348+ // legitimate here, but it must not degrade to writing about one row per batch (#7058).
349+ val conf = new SparkConf ()
350+ .set(" spark.memory.offHeap.enabled" , " true" )
351+ .set(" spark.memory.offHeap.size" , " 1g" )
352+ val memoryManager = new TestMemoryManager (conf)
353+ val kib = 1024L
354+ val initialPageSize = 64 * kib
355+ val maxPageSize = 1024 * kib
356+ // 32 partitions fill about 8 MiB below, so this holds well under half of the batches.
357+ memoryManager.limit(3 * 1024 * kib)
358+ val tmm = new TaskMemoryManager (memoryManager, 0L )
359+ val taskContext = newTaskContext(tmm, 0L )
360+ val allocator = CometShuffleMemoryAllocator .getInstance(tmm, initialPageSize)
361+ val tempDir = Utils .createTempDir()
362+ try {
363+ val partitions = 32
364+ val rowsPerPartition = 2000
365+ val currentWriters = new JLinkedList [CometDiskBlockWriter ]()
366+ val serializer = new UnsafeRowSerializer (1 ).newInstance()
367+ val writers = (0 until partitions).map { partition =>
368+ new CometDiskBlockWriter (
369+ new File (tempDir, s " partition $partition" ),
370+ allocator,
371+ initialPageSize,
372+ maxPageSize,
373+ taskContext,
374+ serializer,
375+ schema,
376+ new ShuffleWriteMetrics ,
377+ conf,
378+ false ,
379+ currentWriters)
380+ }
381+
382+ val toUnsafe = UnsafeProjection .create(schema)
383+ // Round-robin, so that each partition gets a row before any gets a second one. A write
384+ // under memory pressure can be of any writer, so each insert checks all of them.
385+ val written = Array .fill(partitions)(0L )
386+ var batches = 0L
387+ var rowsInBatches = 0L
388+ for (_ <- 0 until rowsPerPartition; partition <- 0 until partitions) {
389+ writers(partition).insertRow(toUnsafe(InternalRow (new Array [Byte ](100 ))), 0 )
390+ for (i <- 0 until partitions if writers(i).getOutputRecords > written(i)) {
391+ batches += 1
392+ rowsInBatches += writers(i).getOutputRecords - written(i)
393+ written(i) = writers(i).getOutputRecords
394+ }
395+ }
396+ writers.foreach(_.close())
397+
398+ assert(writers.map(_.getOutputRecords).sum == partitions * rowsPerPartition)
399+ assert(taskContext.taskMetrics.memoryBytesSpilled > 0 )
400+ assert(taskContext.taskMetrics.diskBytesSpilled > 0 )
401+ assert(tmm.getMemoryConsumptionForThisTask == 0 )
402+ assert(allocator.getUsed == 0 )
403+ // Writing a page's worth of rows is far more than one row. Pages of Spark's size, when
404+ // every writer held one from its first row, gave about one row per batch here.
405+ assert(batches > 0 && rowsInBatches / batches >= 100 , s " $rowsInBatches rows in $batches" )
406+ } finally {
407+ Utils .deleteRecursively(tempDir)
408+ tmm.cleanUpAllAllocatedMemory()
409+ }
410+ }
411+
345412 test(" a fatal error during write() frees the task's buffered pages" ) {
346413 // Spark's ShuffleWriteProcessor only calls stop(false) when write() throws an Exception, so a
347414 // fatal error such as SparkOutOfMemoryError skips it. write() itself must therefore free the
0 commit comments