Skip to content

Commit 59b7cf2

Browse files
committed
Address follow-up shuffle review feedback
1 parent d8999ff commit 59b7cf2

5 files changed

Lines changed: 135 additions & 93 deletions

File tree

‎spark/src/main/java/org/apache/spark/shuffle/comet/CometBoundedShuffleMemoryAllocator.java‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.slf4j.LoggerFactory;
2828

2929
import org.apache.spark.SparkConf;
30+
import org.apache.spark.TaskContext;
3031
import org.apache.spark.memory.MemoryConsumer;
3132
import org.apache.spark.memory.MemoryMode;
3233
import org.apache.spark.memory.SparkOutOfMemoryError;
@@ -68,6 +69,9 @@ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemory
6869
/** How often a thread blocked in {@link #allocateBlocking(long)} logs that it is waiting. */
6970
private static final long WAIT_LOG_INTERVAL_MS = 30_000L;
7071

72+
/** How often a blocked thread checks for cooperative task cancellation. */
73+
private static final long TASK_KILL_POLL_INTERVAL_MS = 1_000L;
74+
7175
/** The number of bits used to address the page table. */
7276
private static final int PAGE_NUMBER_BITS = 13;
7377

@@ -146,25 +150,30 @@ public synchronized MemoryBlock allocate(long required) {
146150
/**
147151
* Like {@link #allocate(long)}, but waits for other tasks of this shared pool to free memory,
148152
* mirroring how Spark's unified memory manager blocks a task until memory becomes available.
149-
* Callers must only use this after spilling their own buffered data. The wait fails fast when it
150-
* can never succeed: when the request does not fit next to the memory this thread itself still
151-
* retains (e.g. the sorter's pointer array), or when all allocated memory is retained by threads
152-
* that are themselves blocked here and none of their requests fits in the free pool. Because the
153-
* holders it depends on may in turn be blocked on resources outside this pool that only a task
154-
* waiting here can release, the wait is also bounded by
155-
* `spark.comet.shuffle.jvm.memoryWaitTimeout`, after which the managed allocation error is thrown
156-
* and Spark's task retry can recover. Interrupting the task (e.g. task kill) aborts the wait.
153+
* Callers must first spill buffered data they can cheaply release; memory this thread still
154+
* retains (e.g. the sorter's pointer array or sibling writers' pages) is included in the liveness
155+
* checks below. The wait fails fast when it can never succeed: when the request does not fit next
156+
* to the requester's retained memory, or when all allocated memory is retained by blocked threads
157+
* and none of their requests fits in the free pool. Because the holders it depends on may in turn
158+
* be blocked on resources outside this pool that only a task waiting here can release, the wait
159+
* is also bounded by `spark.comet.shuffle.jvm.memoryWaitTimeout`, after which the managed
160+
* allocation error is thrown and Spark's task retry can recover. Task cancellation or Java
161+
* interruption aborts the wait.
157162
*/
158163
@Override
159164
public synchronized MemoryBlock allocateBlocking(long required) {
160165
long memoryWaitTimeoutMs =
161166
(long) CometConf$.MODULE$.COMET_SHUFFLE_JVM_MEMORY_WAIT_TIMEOUT().get();
162167
long size = Math.max(pageSize, required);
163168
Thread self = Thread.currentThread();
169+
TaskContext taskContext = TaskContext.get();
164170
long waitStart = 0;
165171
long lastLog = 0;
166172
try {
167173
while (true) {
174+
if (taskContext != null) {
175+
taskContext.killTaskIfInterrupted();
176+
}
168177
try {
169178
return allocateMemoryBlock(size);
170179
} catch (SparkOutOfMemoryError e) {
@@ -217,7 +226,8 @@ public synchronized MemoryBlock allocateBlocking(long required) {
217226
try {
218227
wait(
219228
Math.max(
220-
1L, Math.min(WAIT_LOG_INTERVAL_MS, memoryWaitTimeoutMs - (now - waitStart))));
229+
1L,
230+
Math.min(TASK_KILL_POLL_INTERVAL_MS, memoryWaitTimeoutMs - (now - waitStart))));
221231
} catch (InterruptedException ie) {
222232
Thread.currentThread().interrupt();
223233
// Not an allocation failure: stay non-fatal so that an intentional task kill is
@@ -229,7 +239,9 @@ public synchronized MemoryBlock allocateBlocking(long required) {
229239
}
230240
}
231241
} finally {
232-
waitingThreads.remove(self);
242+
if (waitingThreads.remove(self) != null) {
243+
notifyAll();
244+
}
233245
}
234246
}
235247

‎spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,10 @@ protected CometShuffleMemoryAllocatorTrait(
3535

3636
/**
3737
* Like {@link #allocate(long)}, but may wait for memory freed by other tasks instead of failing
38-
* immediately. Callers must only use this after spilling their own buffered data, so that a
39-
* waiting task holds no memory that other tasks could be waiting for. The default implementation
40-
* does not wait: `CometUnifiedShuffleMemoryAllocator` delegates to Spark's memory manager, which
41-
* already arbitrates memory between tasks.
38+
* immediately. Callers must first spill buffered data they can cheaply release; an implementation
39+
* that waits must account for any memory the caller retains while blocked. The default
40+
* implementation does not wait: `CometUnifiedShuffleMemoryAllocator` delegates to Spark's memory
41+
* manager, which already arbitrates memory between tasks.
4242
*/
4343
public MemoryBlock allocateBlocking(long required) {
4444
return allocate(required);

‎spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometBypassMergeSortShuffleWriter.java‎

Lines changed: 36 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -265,32 +265,7 @@ public void write(Iterator<Product2<K, V>> records) throws IOException {
265265
// task-memory cleanup (in on-heap mode they live in an executor-shared pool). Free them
266266
// and delete their temp files here so they cannot starve other tasks' allocations or leak
267267
// disk space.
268-
if (partitionWriters != null) {
269-
try {
270-
for (CometDiskBlockWriter writer : partitionWriters) {
271-
if (writer == null) {
272-
continue;
273-
}
274-
try {
275-
writer.freeMemory();
276-
} catch (Exception e2) {
277-
logger.error("Failed to free memory of partition writer", e2);
278-
e.addSuppressed(e2);
279-
}
280-
try {
281-
File file = writer.getFile();
282-
if (file.exists() && !file.delete()) {
283-
logger.error("Error while deleting file {}", file.getAbsolutePath());
284-
}
285-
} catch (Exception e2) {
286-
logger.error("Failed to delete file of partition writer", e2);
287-
e.addSuppressed(e2);
288-
}
289-
}
290-
} finally {
291-
partitionWriters = null;
292-
}
293-
}
268+
cleanupPartitionWriters(e);
294269
try {
295270
mapOutputWriter.abort(e);
296271
} catch (Exception e2) {
@@ -301,6 +276,40 @@ public void write(Iterator<Product2<K, V>> records) throws IOException {
301276
}
302277
}
303278

279+
private void cleanupPartitionWriters(@Nullable Throwable failure) {
280+
if (partitionWriters == null) {
281+
return;
282+
}
283+
try {
284+
for (CometDiskBlockWriter writer : partitionWriters) {
285+
if (writer == null) {
286+
continue;
287+
}
288+
try {
289+
writer.freeMemory();
290+
} catch (Exception e) {
291+
logger.error("Failed to free memory of partition writer", e);
292+
if (failure != null) {
293+
failure.addSuppressed(e);
294+
}
295+
}
296+
try {
297+
File file = writer.getFile();
298+
if (file.exists() && !file.delete()) {
299+
logger.error("Error while deleting file {}", file.getAbsolutePath());
300+
}
301+
} catch (Exception e) {
302+
logger.error("Failed to delete file of partition writer", e);
303+
if (failure != null) {
304+
failure.addSuppressed(e);
305+
}
306+
}
307+
}
308+
} finally {
309+
partitionWriters = null;
310+
}
311+
}
312+
304313
@Override
305314
public long[] getPartitionLengths() {
306315
return partitionLengths;
@@ -403,20 +412,7 @@ public Option<MapStatus> stop(boolean success) {
403412
return Option.apply(mapStatus);
404413
} else {
405414
// The map task failed, so delete our output data.
406-
if (partitionWriters != null) {
407-
try {
408-
for (CometDiskBlockWriter writer : partitionWriters) {
409-
writer.freeMemory();
410-
411-
File file = writer.getFile();
412-
if (!file.delete()) {
413-
logger.error("Error while deleting file {}", file.getAbsolutePath());
414-
}
415-
}
416-
} finally {
417-
partitionWriters = null;
418-
}
419-
}
415+
cleanupPartitionWriters(null);
420416
return None$.empty();
421417
}
422418
}

‎spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,8 @@ public final class CometDiskBlockWriter {
6464

6565
/**
6666
* All writers of the same shuffle task. Spilling under memory pressure only ever touches this
67-
* task's writers, never those of other tasks.
67+
* task's writers, never those of other tasks. Access is normally confined to the task thread;
68+
* synchronization below only guards against overlapping lifecycle calls.
6869
*/
6970
private final LinkedList<CometDiskBlockWriter> currentWriters;
7071

@@ -181,8 +182,6 @@ private void doSpill() throws IOException {
181182
// Set this into spilling state first, so it cannot recursively trigger another spill on itself.
182183
spilling = true;
183184

184-
// This spill could be triggered by other thread (i.e., other `CometDiskBlockWriter`),
185-
// so we need to synchronize it.
186185
synchronized (CometDiskBlockWriter.this) {
187186
totalWritten += activeWriter.doSpilling(false);
188187
activeWriter.freeMemory();
@@ -214,8 +213,6 @@ public void insertRow(UnsafeRow row, int partitionId) throws IOException {
214213
final int serializedRecordSize = serBuffer.size();
215214
assert (serializedRecordSize > 0);
216215

217-
// While proceeding with possible spilling and inserting the record, we need to synchronize
218-
// it, because other threads may be spilling this writer at the same time.
219216
synchronized (CometDiskBlockWriter.this) {
220217
if (activeWriter.numRecords() >= numElementsForSpillThreshold
221218
|| activeWriter.numRecords() >= columnarBatchSize) {
@@ -303,7 +300,6 @@ class ArrowIPCWriter extends SpillWriter {
303300

304301
/** Inserts a record into current allocated page. */
305302
void insertRecord(Object recordBase, long recordOffset, int length) {
306-
// This `ArrowIPCWriter` could be spilled by other threads, so we need to synchronize it.
307303
final Object base = currentPage.getBaseObject();
308304

309305
// Add row addresses
@@ -338,7 +334,6 @@ long doSpilling(boolean isLast) throws IOException {
338334

339335
final long written;
340336

341-
// All threads are writing to the same file, so we need to synchronize it.
342337
synchronized (file) {
343338
outputRecords += rowPartition.getNumRows();
344339
written =
@@ -354,7 +349,6 @@ long doSpilling(boolean isLast) throws IOException {
354349
}
355350

356351
// Update metrics
357-
// Other threads may be updating the metrics at the same time, so we need to synchronize it.
358352
synchronized (writeMetrics) {
359353
if (!isLast) {
360354
writeMetrics.incRecordsWritten(
@@ -380,6 +374,9 @@ protected void spill(int required) throws IOException {
380374
// spilling a larger sibling would free enough memory for the allocation on its own.
381375
long totalFreed = getActiveMemoryUsage();
382376
CometDiskBlockWriter.this.doSpill();
377+
if (totalFreed >= required) {
378+
return;
379+
}
383380

384381
// Spill from the largest writer first to maximize the amount of memory we can
385382
// acquire

0 commit comments

Comments
 (0)