Skip to content

Commit 7b6bc88

Browse files
committed
fix: RepartitionExec deadlock when its spill pool holds two open files
In non-preserve-order mode, all input tasks of a `RepartitionExec` share one spill pool for each output partition. When two input tasks spilled at the same time, the pool had two open spill files. The `SpillPoolReader` read only the oldest file. When that file had no unread batch and was not finished, the reader waited for it. But the batch that the reader needed was in the newer file. When each distributor channel had data, the gate closed. Then both input tasks stopped in `send`, no sink was dropped, and nothing could wake the reader. The reader now keeps all the files that it received, oldest first. It gives the first batch that is available in one of them. It skips a file that has no unread batch. It waits for the oldest file that has an unread batch when the read of that file is not complete. Thus one writer gets its batches in FIFO order. This commit makes the tests of the two commits before it green. The fuzzer passes 500 seeds. Ref: apache#24883 Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01RauD5PqPCp4y92RUZUD6C9
1 parent cd1f82b commit 7b6bc88

1 file changed

Lines changed: 128 additions & 86 deletions

File tree

‎datafusion/physical-plan/src/spill/spill_pool.rs‎

Lines changed: 128 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,11 @@ use super::spill_manager::SpillManager;
4848
/// **Lock ordering discipline**: Never hold both locks simultaneously to prevent deadlock.
4949
/// Always: acquire outer lock → release outer lock → acquire inner lock (if needed).
5050
struct SpillPoolShared {
51-
/// Queue of ALL files (including the current write files if any exist).
52-
/// Readers always read from the front of this queue (FIFO).
53-
/// Each file has its own lock to enable concurrent reader/writer access.
54-
files: VecDeque<Arc<Mutex<ActiveSpillFileShared>>>,
51+
/// Files created by writers that the reader has not picked up yet, in creation
52+
/// order. The reader moves them into its own list on every poll (see
53+
/// [`SpillPoolReader`]), so this queue only ever holds files the reader has not
54+
/// seen. Each file has its own lock to enable concurrent reader/writer access.
55+
new_files: VecDeque<Arc<Mutex<ActiveSpillFileShared>>>,
5556
/// SpillManager for creating files and tracking metrics
5657
spill_manager: Arc<SpillManager>,
5758
/// Pool-level waker to notify when new files are available (single reader)
@@ -70,7 +71,7 @@ impl SpillPoolShared {
7071
/// Creates a new shared pool state
7172
fn new(spill_manager: Arc<SpillManager>) -> Self {
7273
Self {
73-
files: VecDeque::new(),
74+
new_files: VecDeque::new(),
7475
spill_manager,
7576
waker: None,
7677
open_write_files: VecDeque::new(),
@@ -237,7 +238,7 @@ impl SpillPoolSink {
237238

238239
// Re-acquire lock and push to shared queue
239240
shared = self.shared.lock();
240-
shared.files.push_back(Arc::clone(&file_shared));
241+
shared.new_files.push_back(Arc::clone(&file_shared));
241242
shared.wake(); // Wake readers waiting for new files
242243
file_shared
243244
};
@@ -570,42 +571,58 @@ struct SpillPoolFile {
570571
spill_manager: Arc<SpillManager>,
571572
}
572573

573-
impl Stream for SpillPoolFile {
574-
type Item = Result<RecordBatch>;
574+
/// Outcome of polling a single file of the pool, see [`SpillPoolFile::poll_file`].
575+
enum FilePoll {
576+
/// A batch was read from the file, or reading it failed.
577+
Item(Result<RecordBatch>),
578+
/// The file's stream ended: every batch written to it has been read and the
579+
/// writer has finished it (the caller verifies the latter).
580+
Done,
581+
/// Every batch written so far has been read, but the writer has not finished
582+
/// the file. The file's waker is registered, so the task is woken when the
583+
/// writer appends another batch or finishes the file.
584+
CaughtUp,
585+
/// The file has unread batches, but the underlying I/O is not ready yet. The
586+
/// I/O has registered the task's waker.
587+
Pending,
588+
}
575589

576-
fn poll_next(
577-
mut self: std::pin::Pin<&mut Self>,
578-
cx: &mut std::task::Context<'_>,
579-
) -> std::task::Poll<Option<Self::Item>> {
590+
impl SpillPoolFile {
591+
/// Polls the file for its next batch.
592+
///
593+
/// Unlike a plain stream poll this tells the caller *why* no batch is
594+
/// available, so [`SpillPoolReader`] can move on to a newer file when this
595+
/// one is only waiting for its writer, but keeps waiting on it while its
596+
/// data is still being read from disk (which keeps batches in file order).
597+
fn poll_file(&mut self, cx: &mut std::task::Context<'_>) -> FilePoll {
580598
use std::task::Poll;
581599

582600
// Step 1: Lock shared state and check coordination
583-
let (should_read, file) = {
601+
let file = {
584602
let mut shared = self.shared.lock();
585603

586604
// Determine if we can read
587605
let batches_read = self.reader.as_ref().map_or(0, |r| r.batches_read);
588606

589607
if batches_read < shared.batches_written {
590608
// More data available to read - take the file if we don't have a reader yet
591-
let file = if self.reader.is_none() {
609+
if self.reader.is_none() {
592610
shared.file.take()
593611
} else {
594612
None
595-
};
596-
(true, file)
613+
}
597614
} else if shared.writer_finished {
598615
// No more data and writer is done - EOF
599-
return Poll::Ready(None);
616+
return FilePoll::Done;
600617
} else {
601618
// Caught up to writer, but writer still active - register waker and wait
602619
shared.register_waker(cx.waker().clone());
603-
return Poll::Pending;
620+
return FilePoll::CaughtUp;
604621
}
605622
}; // Lock released here
606623

607624
// Step 2: Lazy-create reader stream if needed
608-
if self.reader.is_none() && should_read {
625+
if self.reader.is_none() {
609626
if let Some(file) = file {
610627
// we want this unbuffered because files are actively being written to
611628
match self
@@ -618,36 +635,35 @@ impl Stream for SpillPoolFile {
618635
batches_read: 0,
619636
});
620637
}
621-
Err(e) => return Poll::Ready(Some(Err(e))),
638+
Err(e) => return FilePoll::Item(Err(e)),
622639
}
623640
} else {
624641
// File not available yet (writer hasn't finished or already taken)
625642
// Register waker and wait for file to be ready
626643
let mut shared = self.shared.lock();
627644
shared.register_waker(cx.waker().clone());
628-
return Poll::Pending;
645+
return FilePoll::CaughtUp;
629646
}
630647
}
631648

632649
// Step 3: Poll the reader stream (no lock held)
633-
if let Some(reader) = &mut self.reader {
634-
match reader.stream.poll_next_unpin(cx) {
635-
Poll::Ready(Some(Ok(batch))) => {
636-
// Successfully read a batch - increment counter
637-
reader.batches_read += 1;
638-
Poll::Ready(Some(Ok(batch)))
639-
}
640-
Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
641-
Poll::Ready(None) => {
642-
// Stream exhausted unexpectedly
643-
// This shouldn't happen if coordination is correct, but handle gracefully
644-
Poll::Ready(None)
645-
}
646-
Poll::Pending => Poll::Pending,
647-
}
648-
} else {
650+
let Some(reader) = &mut self.reader else {
649651
// Should not reach here, but handle gracefully
650-
Poll::Ready(None)
652+
return FilePoll::Done;
653+
};
654+
match reader.stream.poll_next_unpin(cx) {
655+
Poll::Ready(Some(Ok(batch))) => {
656+
// Successfully read a batch - increment counter
657+
reader.batches_read += 1;
658+
FilePoll::Item(Ok(batch))
659+
}
660+
Poll::Ready(Some(Err(e))) => FilePoll::Item(Err(e)),
661+
Poll::Ready(None) => {
662+
// Stream exhausted unexpectedly
663+
// This shouldn't happen if coordination is correct, but handle gracefully
664+
FilePoll::Done
665+
}
666+
Poll::Pending => FilePoll::Pending,
651667
}
652668
}
653669
}
@@ -669,11 +685,31 @@ impl Stream for SpillPoolFile {
669685
///
670686
/// This makes it suitable for continuous streaming scenarios where the writer may
671687
/// produce data intermittently.
688+
///
689+
/// # Reading from several open files
690+
///
691+
/// With a single writer ([`spsc_channel`]) at most one file is open for writing at
692+
/// a time and every earlier file is finished, so reading the files oldest first is
693+
/// exactly FIFO. With several writers ([`mpsc_channel`]) concurrent pushes can
694+
/// leave more than one file open, and a batch may land in a newer file while the
695+
/// oldest file is drained but not finished. The reader therefore polls every file
696+
/// it knows about, oldest first, and returns the first batch that is available.
697+
/// A file that is merely caught up with its writer is skipped; the reader only
698+
/// waits on a file while that file has unread batches whose bytes are still being
699+
/// read from disk, which keeps batches in file order (strict FIFO for one writer).
700+
///
701+
/// This matters for [`RepartitionExec`](crate::repartition::RepartitionExec), which
702+
/// sends a "spilled" marker through its channel after every `push_batch` and then
703+
/// blocks on this stream until it yields a batch. If the reader only ever drained the
704+
/// oldest file, it could wait for a batch that had been written to a newer file; with
705+
/// the channel gate closed the writers could not push anything to wake it, so the
706+
/// query would deadlock (<https://github.com/apache/datafusion/issues/24883>).
672707
pub struct SpillPoolReader {
673708
/// Shared reference to the spill pool
674709
shared: Arc<Mutex<SpillPoolShared>>,
675-
/// Current SpillPoolFile we're reading from
676-
current_file: Option<SpillPoolFile>,
710+
/// Files this reader has picked up from the pool and not fully consumed yet, in
711+
/// creation order. Each carries its own lazily-created stream and read position.
712+
files: VecDeque<SpillPoolFile>,
677713
/// Schema of the spilled data
678714
schema: SchemaRef,
679715
}
@@ -689,7 +725,7 @@ impl SpillPoolReader {
689725
fn new(shared: Arc<Mutex<SpillPoolShared>>, schema: SchemaRef) -> Self {
690726
Self {
691727
shared,
692-
current_file: None,
728+
files: VecDeque::new(),
693729
schema,
694730
}
695731
}
@@ -704,75 +740,81 @@ impl Stream for SpillPoolReader {
704740
) -> std::task::Poll<Option<Self::Item>> {
705741
use std::task::Poll;
706742

743+
// `Self: Unpin`; reborrow once so `files` and `shared` can be used together
744+
let this = &mut *self;
745+
707746
loop {
708-
// If we have a current file, try to read from it
709-
if let Some(ref mut file) = self.current_file {
710-
match file.poll_next_unpin(cx) {
711-
Poll::Ready(Some(Ok(batch))) => {
712-
// Got a batch, return it
713-
return Poll::Ready(Some(Ok(batch)));
714-
}
715-
Poll::Ready(Some(Err(e))) => {
716-
// Error reading batch
717-
return Poll::Ready(Some(Err(e)));
747+
// Pick up the files writers have created since the last poll, in
748+
// creation order.
749+
{
750+
let mut shared = this.shared.lock();
751+
while let Some(file_shared) = shared.new_files.pop_front() {
752+
let spill_manager = Arc::clone(&shared.spill_manager);
753+
this.files.push_back(SpillPoolFile {
754+
shared: file_shared,
755+
reader: None,
756+
spill_manager,
757+
});
758+
}
759+
} // Lock released here
760+
761+
// Poll the files oldest first and return the first available batch.
762+
// A file that is finished and fully read is dropped, which releases
763+
// its disk space. A file that is only waiting for its writer has
764+
// registered its own waker, so we move on to the next file instead
765+
// of waiting on it.
766+
let mut idx = 0;
767+
while idx < this.files.len() {
768+
match this.files[idx].poll_file(cx) {
769+
FilePoll::Item(item) => {
770+
// Got a batch (or an error), return it
771+
return Poll::Ready(Some(item));
718772
}
719-
Poll::Ready(None) => {
720-
// Current file stream exhausted
773+
FilePoll::Done => {
774+
// File stream exhausted
721775
// Check if this file is marked as writer_finished
722-
let writer_finished = { file.shared.lock().writer_finished };
776+
let writer_finished =
777+
{ this.files[idx].shared.lock().writer_finished };
723778

724779
if writer_finished {
725-
// File is complete, pop it from the queue and move to next
726-
let mut shared = self.shared.lock();
727-
shared.files.pop_front();
728-
drop(shared); // Release lock
729-
730-
// Clear current file and continue loop to get next file
731-
self.current_file = None;
732-
continue;
780+
// File is complete, drop it and move on to the next
781+
this.files.remove(idx);
733782
} else {
734783
// Stream exhausted but writer not finished - unexpected
735784
// This shouldn't happen with proper coordination
736785
return Poll::Ready(None);
737786
}
738787
}
739-
Poll::Pending => {
740-
// File not ready yet (waiting for writer)
741-
// Register waker so we get notified when writer adds more batches
742-
let mut shared = self.shared.lock();
743-
shared.register_waker(cx.waker().clone());
788+
FilePoll::Pending => {
789+
// The oldest file with unread batches is still being read
790+
// from disk. Wait for it rather than skipping ahead, so
791+
// batches come back in file order (FIFO for one writer).
744792
return Poll::Pending;
745793
}
794+
FilePoll::CaughtUp => {
795+
// Nothing to read here until its writer appends more; a
796+
// newer file may have unread batches, so try the next one
797+
idx += 1;
798+
}
746799
}
747800
}
748801

749-
// No current file, need to get the next one
750-
let mut shared = self.shared.lock();
751-
752-
// Peek at the front of the queue (don't pop yet)
753-
if let Some(file_shared) = shared.files.front() {
754-
// Create a SpillPoolFile from the shared state
755-
let spill_manager = Arc::clone(&shared.spill_manager);
756-
let file_shared = Arc::clone(file_shared);
757-
drop(shared); // Release lock before creating SpillPoolFile
758-
759-
self.current_file = Some(SpillPoolFile {
760-
shared: file_shared,
761-
reader: None,
762-
spill_manager,
763-
});
802+
// No known file has an unread batch
803+
let mut shared = this.shared.lock();
764804

765-
// Continue loop to poll the new file
805+
if !shared.new_files.is_empty() {
806+
// A writer created a file while we were polling; pick it up
766807
continue;
767808
}
768809

769-
// No files in queue - check if writer is done
770-
if shared.remaining_writer_count == 0 {
810+
if this.files.is_empty() && shared.remaining_writer_count == 0 {
771811
// Writer is done and no more files will be added - EOF
772812
return Poll::Ready(None);
773813
}
774814

775-
// Writer still active, register waker that will get notified when new files are added
815+
// Writer still active: register the pool-level waker so we are
816+
// notified when a new file is created or the last writer is dropped.
817+
// Each pending file has registered its own waker for new batches.
776818
shared.register_waker(cx.waker().clone());
777819
return Poll::Pending;
778820
}

0 commit comments

Comments
 (0)