2020package org .apache .spark
2121
2222import java .util .Properties
23+ import java .util .concurrent .TimeUnit
24+ import java .util .concurrent .atomic .{AtomicInteger , AtomicLong , AtomicReference }
2325
2426import org .apache .logging .log4j .{Level , LogManager }
2527import org .apache .logging .log4j .core .{LogEvent , LoggerContext }
2628import org .apache .spark .executor .TaskMetrics
27- import org .apache .spark .memory .{MemoryConsumer , TaskMemoryManager , TestMemoryManager }
29+ import org .apache .spark .memory .{MemoryConsumer , MemoryMode , TaskMemoryManager , TestMemoryManager , UnifiedMemoryManager }
2830
2931class CometTaskMemoryManagerSuite extends SparkFunSuite {
3032
@@ -152,10 +154,166 @@ class CometTaskMemoryManagerSuite extends SparkFunSuite {
152154 appender.loggingEvents.toSeq
153155 }
154156
157+ test(" an acquire waiting in Spark survives a release that empties the task's balance" ) {
158+ val memoryManager = offHeapMemoryManager()
159+ val otherTask = new OffHeapConsumer (new TaskMemoryManager (memoryManager, 1L ))
160+
161+ withTaskContext(new TaskMemoryManager (memoryManager, 0L )) { taskMemoryManager =>
162+ val manager = new CometTaskMemoryManager (1L , 0L )
163+ assert(otherTask.acquireMemory(90L ) == 90L )
164+ assert(manager.acquireMemory(10L ) == 10L )
165+
166+ // Another native thread releases the task's last 10 bytes while the acquire waits.
167+ acquireWhileBalanceEmpties(manager, taskMemoryManager, otherTask) {
168+ manager.releaseMemory(10L )
169+ }
170+ }
171+ }
172+
173+ test(" an acquire waiting in Spark survives a sibling consumer freeing the task's last bytes" ) {
174+ val memoryManager = offHeapMemoryManager()
175+ val otherTask = new OffHeapConsumer (new TaskMemoryManager (memoryManager, 1L ))
176+
177+ withTaskContext(new TaskMemoryManager (memoryManager, 0L )) { taskMemoryManager =>
178+ val manager = new CometTaskMemoryManager (1L , 0L )
179+ val sibling = new OffHeapConsumer (taskMemoryManager)
180+ assert(otherTask.acquireMemory(90L ) == 90L )
181+ assert(sibling.acquireMemory(10L ) == 10L )
182+
183+ // A JVM consumer of the same task frees with freeMemory, which takes no task monitor.
184+ acquireWhileBalanceEmpties(manager, taskMemoryManager, otherTask) {
185+ sibling.freeMemory(10L )
186+ }
187+ }
188+ }
189+
190+ test(" a NoSuchElementException other than Spark's missing task entry is rethrown" ) {
191+ val wrongTask = missingEntryError(" key not found: 7" , fromExecutionMemoryPool = true )
192+ val notFromPool = missingEntryError(" key not found: 0" , fromExecutionMemoryPool = false )
193+ for (error <- Seq (wrongTask, notFromPool)) {
194+ val taskMemoryManager = new FailingTaskMemoryManager (error)
195+ withTaskContext(taskMemoryManager) { _ =>
196+ val manager = new CometTaskMemoryManager (1L , 0L )
197+ val thrown = intercept[NoSuchElementException ](manager.acquireMemory(10L ))
198+ assert(thrown eq error)
199+ assert(taskMemoryManager.calls.get == 1 )
200+ assert(manager.getUsed == 0L )
201+ }
202+ }
203+ }
204+
205+ test(" an acquire that keeps losing its task entry gives up with a zero grant" ) {
206+ val error = missingEntryError(" key not found: 0" , fromExecutionMemoryPool = true )
207+ val taskMemoryManager = new FailingTaskMemoryManager (error)
208+ withTaskContext(taskMemoryManager) { _ =>
209+ val manager = new CometTaskMemoryManager (1L , 0L )
210+ assert(manager.acquireMemory(10L ) == 0L )
211+ assert(taskMemoryManager.calls.get == maxAcquireAttempts)
212+ assert(manager.getUsed == 0L )
213+ assert(nativeMemoryConsumer(manager).getUsed == 0L )
214+ assert(taskMemoryManager.getMemoryConsumptionForThisTask == 0L )
215+ }
216+ }
217+
218+ private val TimeoutSeconds = 10L
219+
220+ /**
221+ * The task holds 10 bytes of a 100 byte pool and another task holds 90. An acquire of 20 bytes
222+ * waits in Spark below its minimum share of 25 while `release` takes the task's balance to
223+ * zero, which removes the task's entry from Spark's pool. Once the other task frees its memory
224+ * the acquire must be granted all 20 bytes.
225+ */
226+ private def acquireWhileBalanceEmpties (
227+ manager : CometTaskMemoryManager ,
228+ taskMemoryManager : TaskMemoryManager ,
229+ otherTask : OffHeapConsumer )(release : => Unit ): Unit = {
230+ val granted = new AtomicLong (- 1L )
231+ val failure = new AtomicReference [Throwable ]()
232+ val acquire = new Thread (() =>
233+ try granted.set(manager.acquireMemory(20L ))
234+ catch { case t : Throwable => failure.set(t) })
235+ acquire.setDaemon(true )
236+
237+ try {
238+ acquire.start()
239+ val deadline = System .nanoTime() + TimeUnit .SECONDS .toNanos(TimeoutSeconds )
240+ while (! waitingInSpark(acquire) && System .nanoTime() < deadline) Thread .sleep(10 )
241+ assert(waitingInSpark(acquire), s " the acquire is ${acquire.getState}" )
242+ release
243+ } finally {
244+ // Free the other task's memory so that the acquire does not outlive a failed test.
245+ otherTask.freeMemory(otherTask.getUsed)
246+ acquire.join(TimeUnit .SECONDS .toMillis(TimeoutSeconds ))
247+ }
248+
249+ assert(! acquire.isAlive, s " the acquire is ${acquire.getState}" )
250+ assert(failure.get == null , s " the acquire failed: ${failure.get}" )
251+ assert(granted.get == 20L )
252+ assert(manager.getUsed == 20L )
253+ assert(taskMemoryManager.getMemoryConsumptionForThisTask == 20L )
254+ manager.releaseMemory(20L )
255+ assert(taskMemoryManager.getMemoryConsumptionForThisTask == 0L )
256+ }
257+
258+ private def waitingInSpark (thread : Thread ): Boolean =
259+ thread.getState == Thread .State .WAITING &&
260+ thread.getStackTrace.exists(_.getClassName == " org.apache.spark.memory.ExecutionMemoryPool" )
261+
262+ /** A 100 byte off-heap execution pool. */
263+ private def offHeapMemoryManager (): UnifiedMemoryManager = {
264+ val conf = new SparkConf ()
265+ .set(" spark.memory.offHeap.enabled" , " true" )
266+ .set(" spark.memory.offHeap.size" , " 100" )
267+ .set(" spark.memory.storageFraction" , " 0" )
268+ new UnifiedMemoryManager (conf, 1000L , 500L , 1 )
269+ }
270+
271+ /** An off-heap consumer that never spills. */
272+ private class OffHeapConsumer (taskMemoryManager : TaskMemoryManager )
273+ extends MemoryConsumer (taskMemoryManager, 0L , MemoryMode .OFF_HEAP ) {
274+ override def spill (size : Long , trigger : MemoryConsumer ): Long = 0L
275+ }
276+
277+ private val MaxFailingCalls = 10
278+
279+ /** A task memory manager whose every acquire throws `error`. */
280+ private class FailingTaskMemoryManager (error : Throwable )
281+ extends TaskMemoryManager (new TestMemoryManager (new SparkConf ()), 0L ) {
282+ val calls = new AtomicInteger ()
283+
284+ override def acquireExecutionMemory (required : Long , consumer : MemoryConsumer ): Long = {
285+ // Past this bound a caller is retrying without a cap; fail the test instead of hanging it.
286+ if (calls.incrementAndGet() > MaxFailingCalls ) {
287+ throw new IllegalStateException (
288+ s " acquireExecutionMemory called more than $MaxFailingCalls times " )
289+ }
290+ throw error
291+ }
292+ }
293+
294+ private def missingEntryError (
295+ message : String ,
296+ fromExecutionMemoryPool : Boolean ): NoSuchElementException = {
297+ val error = new NoSuchElementException (message)
298+ if (fromExecutionMemoryPool) {
299+ val poolFrame = new StackTraceElement (
300+ " org.apache.spark.memory.ExecutionMemoryPool" ,
301+ " acquireMemory" ,
302+ " ExecutionMemoryPool.scala" ,
303+ 115 )
304+ error.setStackTrace(poolFrame +: error.getStackTrace)
305+ }
306+ error
307+ }
308+
155309 private def withTaskMemoryManager (f : TaskMemoryManager => Unit ): Unit = {
156310 val memoryManager = new TestMemoryManager (new SparkConf ())
157311 memoryManager.limit(1024 )
158- val taskMemoryManager = new TaskMemoryManager (memoryManager, 0L )
312+ withTaskContext(new TaskMemoryManager (memoryManager, 0L ))(f)
313+ }
314+
315+ private def withTaskContext (taskMemoryManager : TaskMemoryManager )(
316+ f : TaskMemoryManager => Unit ): Unit = {
159317 val taskContext = new TaskContextImpl (
160318 stageId = 0 ,
161319 stageAttemptNumber = 0 ,
@@ -182,6 +340,12 @@ class CometTaskMemoryManagerSuite extends SparkFunSuite {
182340 }
183341 }
184342
343+ private def maxAcquireAttempts : Int = {
344+ val field = classOf [CometTaskMemoryManager ].getDeclaredField(" MAX_ACQUIRE_ATTEMPTS" )
345+ field.setAccessible(true )
346+ field.getInt(null )
347+ }
348+
185349 private def nativeMemoryConsumer (manager : CometTaskMemoryManager ): MemoryConsumer = {
186350 val field = classOf [CometTaskMemoryManager ].getDeclaredField(" nativeMemoryConsumer" )
187351 field.setAccessible(true )
0 commit comments