Skip to content

Commit acb71a1

Browse files
committed
Address shuffle spill review feedback
1 parent d9bf677 commit acb71a1

5 files changed

Lines changed: 172 additions & 73 deletions

File tree

‎spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,6 @@ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemory
6868
/** How often a thread blocked in {@link #allocateBlocking(long)} logs that it is waiting. */
6969
private static final long WAIT_LOG_INTERVAL_MS = 30_000L;
7070

71-
/** How long {@link #allocateBlocking(long)} waits in total before giving up. */
72-
private final long memoryWaitTimeoutMs;
73-
7471
/** The number of bits used to address the page table. */
7572
private static final int PAGE_NUMBER_BITS = 13;
7673

@@ -98,8 +95,6 @@ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemory
9895
this.pageSize = pageSize;
9996
this.totalMemory =
10097
CometSparkSessionExtensions$.MODULE$.getCometShuffleMemorySize(conf, SQLConf.get());
101-
this.memoryWaitTimeoutMs =
102-
(long) CometConf$.MODULE$.COMET_SHUFFLE_JVM_MEMORY_WAIT_TIMEOUT().get();
10398
}
10499

105100
private synchronized long _acquireMemory(long size) {
@@ -151,6 +146,8 @@ public synchronized MemoryBlock allocate(long required) {
151146
*/
152147
@Override
153148
public synchronized MemoryBlock allocateBlocking(long required) {
149+
long memoryWaitTimeoutMs =
150+
(long) CometConf$.MODULE$.COMET_SHUFFLE_JVM_MEMORY_WAIT_TIMEOUT().get();
154151
long size = Math.max(pageSize, required);
155152
Thread self = Thread.currentThread();
156153
long waitStart = 0;

‎spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -263,18 +263,32 @@ public void write(Iterator<Product2<K, V>> records) throws IOException {
263263
// Spark only calls stop(false) when write() throws an Exception; a fatal error such as
264264
// SparkOutOfMemoryError skips it, and the buffered pages are invisible to Spark's
265265
// task-memory cleanup (in on-heap mode they live in an executor-shared pool). Free them
266-
// here so they cannot starve other tasks' allocations.
266+
// and delete their temp files here so they cannot starve other tasks' allocations or leak
267+
// disk space.
267268
if (partitionWriters != null) {
268-
for (CometDiskBlockWriter writer : partitionWriters) {
269-
if (writer == null) {
270-
continue;
271-
}
272-
try {
273-
writer.freeMemory();
274-
} catch (Exception e2) {
275-
logger.error("Failed to free memory of partition writer", e2);
276-
e.addSuppressed(e2);
269+
try {
270+
for (CometDiskBlockWriter writer : partitionWriters) {
271+
if (writer == null) {
272+
continue;
273+
}
274+
try {
275+
writer.freeMemory();
276+
} catch (Exception e2) {
277+
logger.error("Failed to free memory of partition writer", e2);
278+
e.addSuppressed(e2);
279+
}
280+
try {
281+
File file = writer.getFile();
282+
if (file.exists() && !file.delete()) {
283+
logger.error("Error while deleting file {}", file.getAbsolutePath());
284+
}
285+
} catch (Exception e2) {
286+
logger.error("Failed to delete file of partition writer", e2);
287+
e.addSuppressed(e2);
288+
}
277289
}
290+
} finally {
291+
partitionWriters = null;
278292
}
279293
}
280294
try {

‎spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,11 @@ long doSpilling(boolean isLast) throws IOException {
376376
protected void spill(int required) throws IOException {
377377
// Cannot allocate enough memory, spill and try again
378378
synchronized (currentWriters) {
379+
// initialCurrentPage() requires this writer to have released its current page, even when
380+
// spilling a larger sibling would free enough memory for the allocation on its own.
381+
long totalFreed = getActiveMemoryUsage();
382+
CometDiskBlockWriter.this.doSpill();
383+
379384
// Spill from the largest writer first to maximize the amount of memory we can
380385
// acquire
381386
Collections.sort(
@@ -389,17 +394,18 @@ public int compare(CometDiskBlockWriter lhs, CometDiskBlockWriter rhs) {
389394
}
390395
});
391396

392-
long totalFreed = 0;
393397
for (CometDiskBlockWriter writer : currentWriters) {
398+
if (totalFreed >= required) {
399+
break;
400+
}
401+
if (writer == CometDiskBlockWriter.this) {
402+
continue;
403+
}
394404
long used = writer.getActiveMemoryUsage();
395405

396406
writer.doSpill();
397407

398408
totalFreed += used;
399-
400-
if (totalFreed >= required) {
401-
break;
402-
}
403409
}
404410
}
405411
}

‎spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometUnsafeShuffleWriter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ public void write(scala.collection.Iterator<Product2<K, V>> records) throws IOEx
272272
}
273273
}
274274
}
275-
if (tracingEnabled) {
275+
if (tracingEnabled && this.allocator != null) {
276276
nativeLib.logMemoryUsage(offheapMemKey, this.allocator.getUsed());
277277
}
278278
}

0 commit comments

Comments
 (0)