Skip to content

Commit c1e00e7

Browse files
authored
fix: retry a native acquire that Spark failed after dropping the task's entry (#6310)
Spark removes a task's execution pool entry when its balance reaches zero, and an acquire that was parked in the pool then throws NoSuchElementException when it wakes up. Releases take no task monitor, so another native thread or a JVM consumer of the same task can empty the balance while a native acquire waits. The error is not a memory refusal, so the operator could not spill and the task failed. CometTaskMemoryManager now recognises that one exception and calls Spark again. A missing entry means the task held nothing from Spark, so the failed call has no grant to reconcile, and the retry registers the task again and waits for its share as the first call would have. After three attempts it returns a zero grant, which the native side treats as a refusal and spills. Closes #6224.
1 parent e7e7198 commit c1e00e7

3 files changed

Lines changed: 231 additions & 3 deletions

File tree

‎docs/source/contributor-guide/memory_management.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,15 @@ the call never returns. Nothing releases them until Spark's final task cleanup,
214214
nobody can use for the rest of the task. Any caller that swallows the exception has to reconcile
215215
that grant, and the only figure available for doing so is the task-wide one above.
216216

217+
**A parked acquire can wake up to a missing task entry.** Spark removes the task's `memoryForTask`
218+
entry when its balance reaches zero, and an acquire that was waiting in `lock.wait()` and wakes
219+
afterwards throws a `NoSuchElementException` ("key not found" and the task id) instead of a grant.
220+
`CometTaskMemoryManager` retries that one case. A missing entry means the task held nothing from
221+
Spark at that moment, so the failed call has no partial grant to reconcile, and the retry registers
222+
the task again and waits for its share as the first call would have. After a few attempts it returns
223+
a zero grant, which the native side treats as a refusal and spills. JVM consumers that call
224+
`allocatePage` directly are not covered; that gap is tracked in #6304.
225+
217226
**A consumer whose `spill` returns zero takes budget it can never give back.**
218227
`NativeMemoryConsumer.spill` returns `0`, so Spark can select it as a spill victim and reclaim
219228
nothing from it; it is only ever a spill trigger. The bytes it holds are real, so other consumers in

‎spark/src/main/java/org/apache/spark/CometTaskMemoryManager.java‎

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.apache.spark;
2121

2222
import java.io.IOException;
23+
import java.util.NoSuchElementException;
2324
import java.util.concurrent.atomic.AtomicLong;
2425

2526
import org.slf4j.Logger;
@@ -38,6 +39,9 @@ public class CometTaskMemoryManager {
3839

3940
private static final Logger logger = LoggerFactory.getLogger(CometTaskMemoryManager.class);
4041

42+
/** Attempts at an acquire whose task entry Spark keeps losing, see acquireFromSpark. */
43+
private static final int MAX_ACQUIRE_ATTEMPTS = 3;
44+
4145
/** The id of the native plan this memory manager was created for, the first in its task. */
4246
private final long id;
4347

@@ -70,7 +74,7 @@ public long acquireMemory(long size) {
7074
if (logger.isTraceEnabled()) {
7175
logger.trace("Task {} requested {} bytes", taskAttemptId, size);
7276
}
73-
long acquired = internal.acquireExecutionMemory(size, nativeMemoryConsumer);
77+
long acquired = acquireFromSpark(size);
7478
long newUsed = used.addAndGet(acquired);
7579
// A partial grant is routine, not an error: the native pool either refuses the reservation,
7680
// which tells the operator to spill, or carries the shortfall as overcommit. A refusal that
@@ -94,6 +98,57 @@ public long acquireMemory(long size) {
9498
return acquired;
9599
}
96100

101+
/**
102+
* Acquires from Spark, retrying when an acquire waiting in Spark lost the task's entry. Spark
103+
* removes a task's entry from its execution pool when the task's balance reaches zero, and an
104+
* acquire that was waiting for memory then fails when it wakes up with a NoSuchElementException
105+
* ("key not found: " and the task id). Releases take no task monitor, so another native thread or
106+
* a JVM consumer of the same task can empty the balance while an acquire waits. The failed call
107+
* was granted nothing, because Spark only drops the entry at a zero balance, which cannot hold
108+
* while this call has a partial grant. So trying again is safe: it registers the task again and
109+
* waits for its share as Spark would have. After a few failed attempts the grant is refused. The
110+
* helper holds no lock of its own across attempts, so a retry parks in Spark exactly as the first
111+
* attempt did.
112+
*/
113+
private long acquireFromSpark(long size) {
114+
for (int attempt = 1; ; attempt++) {
115+
try {
116+
return internal.acquireExecutionMemory(size, nativeMemoryConsumer);
117+
} catch (NoSuchElementException e) {
118+
if (!isMissingTaskEntry(e)) {
119+
throw e;
120+
}
121+
if (attempt >= MAX_ACQUIRE_ATTEMPTS) {
122+
logger.warn(
123+
"Task {} lost its execution memory entry in Spark on {} attempts to acquire {} "
124+
+ "bytes, refusing the request",
125+
taskAttemptId,
126+
attempt,
127+
size);
128+
return 0;
129+
}
130+
logger.info(
131+
"Task {} lost its execution memory entry in Spark while waiting to acquire {} bytes, "
132+
+ "trying again",
133+
taskAttemptId,
134+
size);
135+
}
136+
}
137+
}
138+
139+
/** Whether {@code e} is Spark's execution pool failing to find this task's entry. */
140+
private boolean isMissingTaskEntry(NoSuchElementException e) {
141+
if (!("key not found: " + taskAttemptId).equals(e.getMessage())) {
142+
return false;
143+
}
144+
for (StackTraceElement frame : e.getStackTrace()) {
145+
if ("org.apache.spark.memory.ExecutionMemoryPool".equals(frame.getClassName())) {
146+
return true;
147+
}
148+
}
149+
return false;
150+
}
151+
97152
// Called by Comet native through JNI
98153
public void releaseMemory(long size) {
99154
if (logger.isTraceEnabled()) {

‎spark/src/test/scala/org/apache/spark/CometTaskMemoryManagerSuite.scala‎

Lines changed: 166 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,13 @@
2020
package org.apache.spark
2121

2222
import java.util.Properties
23+
import java.util.concurrent.TimeUnit
24+
import java.util.concurrent.atomic.{AtomicInteger, AtomicLong, AtomicReference}
2325

2426
import org.apache.logging.log4j.{Level, LogManager}
2527
import org.apache.logging.log4j.core.{LogEvent, LoggerContext}
2628
import org.apache.spark.executor.TaskMetrics
27-
import org.apache.spark.memory.{MemoryConsumer, TaskMemoryManager, TestMemoryManager}
29+
import org.apache.spark.memory.{MemoryConsumer, MemoryMode, TaskMemoryManager, TestMemoryManager, UnifiedMemoryManager}
2830

2931
class CometTaskMemoryManagerSuite extends SparkFunSuite {
3032

@@ -152,10 +154,166 @@ class CometTaskMemoryManagerSuite extends SparkFunSuite {
152154
appender.loggingEvents.toSeq
153155
}
154156

157+
test("an acquire waiting in Spark survives a release that empties the task's balance") {
158+
val memoryManager = offHeapMemoryManager()
159+
val otherTask = new OffHeapConsumer(new TaskMemoryManager(memoryManager, 1L))
160+
161+
withTaskContext(new TaskMemoryManager(memoryManager, 0L)) { taskMemoryManager =>
162+
val manager = new CometTaskMemoryManager(1L, 0L)
163+
assert(otherTask.acquireMemory(90L) == 90L)
164+
assert(manager.acquireMemory(10L) == 10L)
165+
166+
// Another native thread releases the task's last 10 bytes while the acquire waits.
167+
acquireWhileBalanceEmpties(manager, taskMemoryManager, otherTask) {
168+
manager.releaseMemory(10L)
169+
}
170+
}
171+
}
172+
173+
test("an acquire waiting in Spark survives a sibling consumer freeing the task's last bytes") {
174+
val memoryManager = offHeapMemoryManager()
175+
val otherTask = new OffHeapConsumer(new TaskMemoryManager(memoryManager, 1L))
176+
177+
withTaskContext(new TaskMemoryManager(memoryManager, 0L)) { taskMemoryManager =>
178+
val manager = new CometTaskMemoryManager(1L, 0L)
179+
val sibling = new OffHeapConsumer(taskMemoryManager)
180+
assert(otherTask.acquireMemory(90L) == 90L)
181+
assert(sibling.acquireMemory(10L) == 10L)
182+
183+
// A JVM consumer of the same task frees with freeMemory, which takes no task monitor.
184+
acquireWhileBalanceEmpties(manager, taskMemoryManager, otherTask) {
185+
sibling.freeMemory(10L)
186+
}
187+
}
188+
}
189+
190+
test("a NoSuchElementException other than Spark's missing task entry is rethrown") {
191+
val wrongTask = missingEntryError("key not found: 7", fromExecutionMemoryPool = true)
192+
val notFromPool = missingEntryError("key not found: 0", fromExecutionMemoryPool = false)
193+
for (error <- Seq(wrongTask, notFromPool)) {
194+
val taskMemoryManager = new FailingTaskMemoryManager(error)
195+
withTaskContext(taskMemoryManager) { _ =>
196+
val manager = new CometTaskMemoryManager(1L, 0L)
197+
val thrown = intercept[NoSuchElementException](manager.acquireMemory(10L))
198+
assert(thrown eq error)
199+
assert(taskMemoryManager.calls.get == 1)
200+
assert(manager.getUsed == 0L)
201+
}
202+
}
203+
}
204+
205+
test("an acquire that keeps losing its task entry gives up with a zero grant") {
206+
val error = missingEntryError("key not found: 0", fromExecutionMemoryPool = true)
207+
val taskMemoryManager = new FailingTaskMemoryManager(error)
208+
withTaskContext(taskMemoryManager) { _ =>
209+
val manager = new CometTaskMemoryManager(1L, 0L)
210+
assert(manager.acquireMemory(10L) == 0L)
211+
assert(taskMemoryManager.calls.get == maxAcquireAttempts)
212+
assert(manager.getUsed == 0L)
213+
assert(nativeMemoryConsumer(manager).getUsed == 0L)
214+
assert(taskMemoryManager.getMemoryConsumptionForThisTask == 0L)
215+
}
216+
}
217+
218+
private val TimeoutSeconds = 10L
219+
220+
/**
221+
* The task holds 10 bytes of a 100 byte pool and another task holds 90. An acquire of 20 bytes
222+
* waits in Spark below its minimum share of 25 while `release` takes the task's balance to
223+
* zero, which removes the task's entry from Spark's pool. Once the other task frees its memory
224+
* the acquire must be granted all 20 bytes.
225+
*/
226+
private def acquireWhileBalanceEmpties(
227+
manager: CometTaskMemoryManager,
228+
taskMemoryManager: TaskMemoryManager,
229+
otherTask: OffHeapConsumer)(release: => Unit): Unit = {
230+
val granted = new AtomicLong(-1L)
231+
val failure = new AtomicReference[Throwable]()
232+
val acquire = new Thread(() =>
233+
try granted.set(manager.acquireMemory(20L))
234+
catch { case t: Throwable => failure.set(t) })
235+
acquire.setDaemon(true)
236+
237+
try {
238+
acquire.start()
239+
val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TimeoutSeconds)
240+
while (!waitingInSpark(acquire) && System.nanoTime() < deadline) Thread.sleep(10)
241+
assert(waitingInSpark(acquire), s"the acquire is ${acquire.getState}")
242+
release
243+
} finally {
244+
// Free the other task's memory so that the acquire does not outlive a failed test.
245+
otherTask.freeMemory(otherTask.getUsed)
246+
acquire.join(TimeUnit.SECONDS.toMillis(TimeoutSeconds))
247+
}
248+
249+
assert(!acquire.isAlive, s"the acquire is ${acquire.getState}")
250+
assert(failure.get == null, s"the acquire failed: ${failure.get}")
251+
assert(granted.get == 20L)
252+
assert(manager.getUsed == 20L)
253+
assert(taskMemoryManager.getMemoryConsumptionForThisTask == 20L)
254+
manager.releaseMemory(20L)
255+
assert(taskMemoryManager.getMemoryConsumptionForThisTask == 0L)
256+
}
257+
258+
private def waitingInSpark(thread: Thread): Boolean =
259+
thread.getState == Thread.State.WAITING &&
260+
thread.getStackTrace.exists(_.getClassName == "org.apache.spark.memory.ExecutionMemoryPool")
261+
262+
/** A 100 byte off-heap execution pool. */
263+
private def offHeapMemoryManager(): UnifiedMemoryManager = {
264+
val conf = new SparkConf()
265+
.set("spark.memory.offHeap.enabled", "true")
266+
.set("spark.memory.offHeap.size", "100")
267+
.set("spark.memory.storageFraction", "0")
268+
new UnifiedMemoryManager(conf, 1000L, 500L, 1)
269+
}
270+
271+
/** An off-heap consumer that never spills. */
272+
private class OffHeapConsumer(taskMemoryManager: TaskMemoryManager)
273+
extends MemoryConsumer(taskMemoryManager, 0L, MemoryMode.OFF_HEAP) {
274+
override def spill(size: Long, trigger: MemoryConsumer): Long = 0L
275+
}
276+
277+
private val MaxFailingCalls = 10
278+
279+
/** A task memory manager whose every acquire throws `error`. */
280+
private class FailingTaskMemoryManager(error: Throwable)
281+
extends TaskMemoryManager(new TestMemoryManager(new SparkConf()), 0L) {
282+
val calls = new AtomicInteger()
283+
284+
override def acquireExecutionMemory(required: Long, consumer: MemoryConsumer): Long = {
285+
// Past this bound a caller is retrying without a cap; fail the test instead of hanging it.
286+
if (calls.incrementAndGet() > MaxFailingCalls) {
287+
throw new IllegalStateException(
288+
s"acquireExecutionMemory called more than $MaxFailingCalls times")
289+
}
290+
throw error
291+
}
292+
}
293+
294+
private def missingEntryError(
295+
message: String,
296+
fromExecutionMemoryPool: Boolean): NoSuchElementException = {
297+
val error = new NoSuchElementException(message)
298+
if (fromExecutionMemoryPool) {
299+
val poolFrame = new StackTraceElement(
300+
"org.apache.spark.memory.ExecutionMemoryPool",
301+
"acquireMemory",
302+
"ExecutionMemoryPool.scala",
303+
115)
304+
error.setStackTrace(poolFrame +: error.getStackTrace)
305+
}
306+
error
307+
}
308+
155309
private def withTaskMemoryManager(f: TaskMemoryManager => Unit): Unit = {
156310
val memoryManager = new TestMemoryManager(new SparkConf())
157311
memoryManager.limit(1024)
158-
val taskMemoryManager = new TaskMemoryManager(memoryManager, 0L)
312+
withTaskContext(new TaskMemoryManager(memoryManager, 0L))(f)
313+
}
314+
315+
private def withTaskContext(taskMemoryManager: TaskMemoryManager)(
316+
f: TaskMemoryManager => Unit): Unit = {
159317
val taskContext = new TaskContextImpl(
160318
stageId = 0,
161319
stageAttemptNumber = 0,
@@ -182,6 +340,12 @@ class CometTaskMemoryManagerSuite extends SparkFunSuite {
182340
}
183341
}
184342

343+
private def maxAcquireAttempts: Int = {
344+
val field = classOf[CometTaskMemoryManager].getDeclaredField("MAX_ACQUIRE_ATTEMPTS")
345+
field.setAccessible(true)
346+
field.getInt(null)
347+
}
348+
185349
private def nativeMemoryConsumer(manager: CometTaskMemoryManager): MemoryConsumer = {
186350
val field = classOf[CometTaskMemoryManager].getDeclaredField("nativeMemoryConsumer")
187351
field.setAccessible(true)

0 commit comments

Comments
 (0)