Skip to content

Commit 733a74b

Browse files
committed
fix: stop a broadcast read within a piece of an interrupt, and free a decode its task outlived
Arrow reads a whole message body with one read, which for a coalesced broadcast is all of it, so the read channel now checks for an interrupt before each piece, as Channels.newChannel did, instead of once per read. A task whose completion listener runs on another thread can give back its hold while its read decodes the broadcast. If it was the last holder, the entry was freed before the decode finished, and the decode was kept where nothing would free it. The decode is now freed instead, and the read decodes the broadcast for itself.
1 parent 2a68f4e commit 733a74b

4 files changed

Lines changed: 74 additions & 8 deletions

File tree

‎spark/src/main/scala/org/apache/spark/sql/comet/DecodedBroadcastCache.scala‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,9 @@ import org.apache.comet.vector.NativeUtil
5555
* An entry is freed when the last task holding it completes, and a task that reads the broadcast
5656
* after that decodes it again. A native join keeps a build side of one batch, which is what
5757
* coalescing makes of a broadcast, for as long as its task runs, so for such a broadcast the
58-
* cache holds no memory that the tasks would not hold anyway.
58+
* cache holds no memory that the tasks would not hold anyway. A native join copies a build side
59+
* of several batches, so the cache keeps one copy of such a broadcast that the tasks would have
60+
* freed once they built their tables.
5961
*
6062
* @param decode
6163
* decodes a broadcast's value; `None` makes each task decode the value for itself
@@ -132,7 +134,19 @@ private[comet] class DecodedBroadcastCache(
132134
/** Decodes the broadcast for every task that holds the entry. Runs in the first of them. */
133135
def load(source: String): Option[Decoded] = {
134136
try {
135-
decoded = decode(chunks, source)
137+
val result = decode(chunks, source)
138+
// A task whose completion listener runs on another thread can give the entry back
139+
// during the decode. If it was the last holder the entry has been freed already, and
140+
// nothing would free what it kept now.
141+
val kept = entries.synchronized {
142+
if (holders > 0) {
143+
decoded = result
144+
}
145+
holders > 0
146+
}
147+
if (!kept) {
148+
result.foreach(_.close())
149+
}
136150
decoded
137151
} catch {
138152
case t: Throwable =>

‎spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -359,8 +359,8 @@ object Utils extends CometTypeShim with Logging {
359359
* Reads `in` into each buffer it is asked to fill until the buffer is full or `in` ends, up to
360360
* 64 KiB at a time through one reused array. `Channels.newChannel` reads 8 KiB at a time, and
361361
* enters and leaves the JDK's interruptible I/O machinery for each piece. Like that channel,
362-
* this one closes and throws `ClosedByInterruptException` when a read starts on an interrupted
363-
* thread, so a cancelled decode still stops at its next read.
362+
* this one closes and throws `ClosedByInterruptException` before it reads a piece on an
363+
* interrupted thread, so a cancelled decode stops within a piece of where it was.
364364
*/
365365
private final class InputStreamChannel(in: InputStream) extends ReadableByteChannel {
366366
private var piece = Array.emptyByteArray
@@ -370,13 +370,13 @@ object Utils extends CometTypeShim with Logging {
370370
if (!open) {
371371
throw new ClosedChannelException
372372
}
373-
if (Thread.currentThread().isInterrupted) {
374-
close()
375-
throw new ClosedByInterruptException
376-
}
377373
var total = 0
378374
var end = false
379375
while (dst.hasRemaining && !end) {
376+
if (Thread.currentThread().isInterrupted) {
377+
close()
378+
throw new ClosedByInterruptException
379+
}
380380
val length = math.min(dst.remaining(), MaxReadPiece)
381381
if (piece.length < length) {
382382
piece = new Array[Byte](length)

‎spark/src/test/scala/org/apache/spark/sql/comet/DecodedBroadcastCacheSuite.scala‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,29 @@ class DecodedBroadcastCacheSuite extends CometTestBase {
211211
assert(CometArrowAllocator.getAllocatedMemory == before)
212212
}
213213

214+
test("a decode that outlives its task frees what it decoded") {
215+
val value = broadcastValue(rows(100))
216+
val decoding = new CountDownLatch(1)
217+
val finish = new CountDownLatch(1)
218+
val cache = new DecodedBroadcastCache((chunks, source) => {
219+
decoding.countDown()
220+
finish.await()
221+
DecodedBroadcastCache.decode(chunks, source)
222+
})
223+
val before = CometArrowAllocator.getAllocatedMemory
224+
val task = newTask(1)
225+
226+
val read = inThread(readAll(cache.batches(value, task, "test")))
227+
assert(decoding.await(Timeout.toSeconds, TimeUnit.SECONDS))
228+
// The task completes on another thread while its read decodes the broadcast, which gives
229+
// back the task's hold and frees the entry before the decode keeps anything in it.
230+
task.markTaskCompleted(None)
231+
assert(cache.size == 0)
232+
finish.countDown()
233+
assert(Await.result(read, Timeout) == rows(100))
234+
assert(CometArrowAllocator.getAllocatedMemory == before)
235+
}
236+
214237
test("each task decodes a broadcast with dictionaries for itself") {
215238
val values = Seq("a", "b", null, "a", "c", "b")
216239
val value = dictionaryBroadcastValue(values)

‎spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919

2020
package org.apache.spark.sql.comet.util
2121

22+
import java.io.{FilterInputStream, InputStream, OutputStream}
23+
import java.nio.ByteBuffer
2224
import java.nio.channels.ClosedByInterruptException
2325
import java.nio.charset.StandardCharsets
2426

@@ -28,6 +30,7 @@ import org.apache.arrow.c.CDataDictionaryProvider
2830
import org.apache.arrow.memory.BufferAllocator
2931
import org.apache.arrow.vector.VarCharVector
3032
import org.apache.arrow.vector.ipc.message.ArrowFieldNode
33+
import org.apache.spark.io.CompressionCodec
3134
import org.apache.spark.sql.CometTestBase
3235
import org.apache.spark.sql.execution.vectorized.ConstantColumnVector
3336
import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType, TimestampType}
@@ -142,6 +145,32 @@ class UtilsSuite extends CometTestBase {
142145
assert(CometArrowAllocator.getAllocatedMemory == before)
143146
}
144147

148+
test("a read of a broadcast stream stops at the next piece once its thread is interrupted") {
149+
// Arrow reads a message body with one call, which for a coalesced broadcast is all of it.
150+
var pieces = 0
151+
val codec = new CompressionCodec {
152+
override def compressedOutputStream(s: OutputStream): OutputStream = s
153+
// Interrupts the reading thread during the first piece, as killing its task does.
154+
override def compressedInputStream(s: InputStream): InputStream =
155+
new FilterInputStream(s) {
156+
override def read(b: Array[Byte], off: Int, len: Int): Int = {
157+
pieces += 1
158+
Thread.currentThread().interrupt()
159+
super.read(b, off, len)
160+
}
161+
}
162+
}
163+
val channel =
164+
Utils.ipcChannel(new ChunkedByteBuffer(ByteBuffer.wrap(new Array[Byte](1 << 20))), codec)
165+
try {
166+
intercept[ClosedByInterruptException](channel.read(ByteBuffer.allocate(1 << 20)))
167+
} finally {
168+
Thread.interrupted()
169+
}
170+
assert(pieces == 1)
171+
assert(!channel.isOpen)
172+
}
173+
145174
test("serializeBatches closes the slice it writes when the write fails") {
146175
// A cancelled broadcast interrupts its task, so the next write to the interruptible channel
147176
// serializeBatches writes through fails. The slice holds its own references on the batch's

0 commit comments

Comments
 (0)