2727import org .slf4j .LoggerFactory ;
2828
2929import org .apache .spark .SparkConf ;
30+ import org .apache .spark .TaskContext ;
3031import org .apache .spark .memory .MemoryConsumer ;
3132import org .apache .spark .memory .MemoryMode ;
3233import org .apache .spark .memory .SparkOutOfMemoryError ;
@@ -68,6 +69,9 @@ public final class CometBoundedShuffleMemoryAllocator extends CometShuffleMemory
6869 /** How often a thread blocked in {@link #allocateBlocking(long)} logs that it is waiting. */
6970 private static final long WAIT_LOG_INTERVAL_MS = 30_000L ;
7071
72+ /** How often a blocked thread checks for cooperative task cancellation. */
73+ private static final long TASK_KILL_POLL_INTERVAL_MS = 1_000L ;
74+
7175 /** The number of bits used to address the page table. */
7276 private static final int PAGE_NUMBER_BITS = 13 ;
7377
@@ -146,25 +150,30 @@ public synchronized MemoryBlock allocate(long required) {
146150 /**
147151 * Like {@link #allocate(long)}, but waits for other tasks of this shared pool to free memory,
148152 * mirroring how Spark's unified memory manager blocks a task until memory becomes available.
149- * Callers must only use this after spilling their own buffered data. The wait fails fast when it
150- * can never succeed: when the request does not fit next to the memory this thread itself still
151- * retains (e.g. the sorter's pointer array), or when all allocated memory is retained by threads
152- * that are themselves blocked here and none of their requests fits in the free pool. Because the
153- * holders it depends on may in turn be blocked on resources outside this pool that only a task
154- * waiting here can release, the wait is also bounded by
155- * `spark.comet.shuffle.jvm.memoryWaitTimeout`, after which the managed allocation error is thrown
156- * and Spark's task retry can recover. Interrupting the task (e.g. task kill) aborts the wait.
153+ * Callers must first spill buffered data they can cheaply release; memory this thread still
154+ * retains (e.g. the sorter's pointer array or sibling writers' pages) is included in the liveness
155+ * checks below. The wait fails fast when it can never succeed: when the request does not fit next
156+ * to the requester's retained memory, or when all allocated memory is retained by blocked threads
157+ * and none of their requests fits in the free pool. Because the holders it depends on may in turn
158+ * be blocked on resources outside this pool that only a task waiting here can release, the wait
159+ * is also bounded by `spark.comet.shuffle.jvm.memoryWaitTimeout`, after which the managed
160+ * allocation error is thrown and Spark's task retry can recover. Task cancellation or Java
161+ * interruption aborts the wait.
157162 */
158163 @ Override
159164 public synchronized MemoryBlock allocateBlocking (long required ) {
160165 long memoryWaitTimeoutMs =
161166 (long ) CometConf$ .MODULE$ .COMET_SHUFFLE_JVM_MEMORY_WAIT_TIMEOUT ().get ();
162167 long size = Math .max (pageSize , required );
163168 Thread self = Thread .currentThread ();
169+ TaskContext taskContext = TaskContext .get ();
164170 long waitStart = 0 ;
165171 long lastLog = 0 ;
166172 try {
167173 while (true ) {
174+ if (taskContext != null ) {
175+ taskContext .killTaskIfInterrupted ();
176+ }
168177 try {
169178 return allocateMemoryBlock (size );
170179 } catch (SparkOutOfMemoryError e ) {
@@ -217,7 +226,8 @@ public synchronized MemoryBlock allocateBlocking(long required) {
217226 try {
218227 wait (
219228 Math .max (
220- 1L , Math .min (WAIT_LOG_INTERVAL_MS , memoryWaitTimeoutMs - (now - waitStart ))));
229+ 1L ,
230+ Math .min (TASK_KILL_POLL_INTERVAL_MS , memoryWaitTimeoutMs - (now - waitStart ))));
221231 } catch (InterruptedException ie ) {
222232 Thread .currentThread ().interrupt ();
223233 // Not an allocation failure: stay non-fatal so that an intentional task kill is
@@ -229,7 +239,9 @@ public synchronized MemoryBlock allocateBlocking(long required) {
229239 }
230240 }
231241 } finally {
232- waitingThreads .remove (self );
242+ if (waitingThreads .remove (self ) != null ) {
243+ notifyAll ();
244+ }
233245 }
234246 }
235247
0 commit comments