1717
1818use futures:: { Stream , StreamExt } ;
1919use std:: collections:: VecDeque ;
20+ use std:: mem;
2021use std:: sync:: Arc ;
2122use std:: task:: Waker ;
2223
@@ -56,11 +57,9 @@ struct SpillPoolShared {
5657 spill_manager : Arc < SpillManager > ,
5758 /// Pool-level waker to notify when new files are available (single reader)
5859 waker : Option < Waker > ,
59- /// Whether the writer has been dropped (no more files will be added)
60- writer_dropped : bool ,
6160 /// Writer's reference to the current file (shared by all cloned writers).
6261 /// Has its own lock to allow I/O without blocking queue access.
63- current_write_file : Option < Arc < Mutex < ActiveSpillFileShared > > > ,
62+ open_write_files : VecDeque < Arc < Mutex < ActiveSpillFileShared > > > ,
6463 /// Number of active writer clones. Only when this reaches zero should
6564 /// `writer_dropped` be set to true. This prevents premature EOF signaling
6665 /// when one writer clone is dropped while others are still active.
@@ -74,8 +73,7 @@ impl SpillPoolShared {
7473 files : VecDeque :: new ( ) ,
7574 spill_manager,
7675 waker : None ,
77- writer_dropped : false ,
78- current_write_file : None ,
76+ open_write_files : VecDeque :: new ( ) ,
7977 active_writer_count : 1 ,
8078 }
8179 }
@@ -171,8 +169,10 @@ impl SpillPoolWriter {
171169 // Fine-grained locking: Lock shared state briefly for queue access
172170 let mut shared = self . shared . lock ( ) ;
173171
174- // Create new file if we don't have one yet
175- if shared. current_write_file . is_none ( ) {
172+ // Create new file if there is none available to append to
173+ let write_file = if !shared. open_write_files . is_empty ( ) {
174+ shared. open_write_files . pop_front ( ) . unwrap ( )
175+ } else {
176176 let spill_manager = Arc :: clone ( & shared. spill_manager ) ;
177177 // Release shared lock before disk I/O (fine-grained locking)
178178 drop ( shared) ;
@@ -193,52 +193,49 @@ impl SpillPoolWriter {
193193 // Re-acquire lock and push to shared queue
194194 shared = self . shared . lock ( ) ;
195195 shared. files . push_back ( Arc :: clone ( & file_shared) ) ;
196- shared. current_write_file = Some ( file_shared) ;
197196 shared. wake ( ) ; // Wake readers waiting for new files
198- }
197+ file_shared
198+ } ;
199199
200- let current_write_file = shared. current_write_file . take ( ) ;
201200 // Release shared lock before file I/O (fine-grained locking)
202201 // This allows readers to access the queue while we do disk I/O
203202 drop ( shared) ;
204203
205204 // Write batch to current file - lock only the specific file
206- if let Some ( current_file) = current_write_file {
207- // Now lock just this file for I/O (separate from shared lock)
208- let mut file_shared = current_file. lock ( ) ;
209-
210- // Append the batch
211- if let Some ( ref mut writer) = file_shared. writer {
212- writer. append_batch ( batch) ?;
213- // make sure we flush the writer for readers
214- writer. flush ( ) ?;
215- file_shared. batches_written += 1 ;
216- file_shared. estimated_size += batch_size;
217- }
205+ let mut file_shared = write_file. lock ( ) ;
206+
207+ // Append the batch
208+ if let Some ( ref mut writer) = file_shared. writer {
209+ writer. append_batch ( batch) ?;
210+ // make sure we flush the writer for readers
211+ writer. flush ( ) ?;
212+ file_shared. batches_written += 1 ;
213+ file_shared. estimated_size += batch_size;
214+ }
218215
219- // Wake reader waiting on this specific file
220- file_shared. wake ( ) ;
216+ // Wake reader waiting on this specific file
217+ file_shared. wake ( ) ;
221218
222- // Check if we need to rotate
223- let needs_rotation = file_shared. estimated_size > self . max_file_size_bytes ;
219+ let max_file_size_reached = file_shared. estimated_size > self . max_file_size_bytes ;
224220
225- if needs_rotation {
226- // Finish the IPC writer
227- if let Some ( mut writer) = file_shared. writer . take ( ) {
228- writer. finish ( ) ?;
229- }
230- // Mark as finished so readers know not to wait for more data
231- file_shared. writer_finished = true ;
232- // Wake reader waiting on this file (it's now finished)
233- file_shared. wake ( ) ;
234- // Don't put back current_write_file - let it rotate
235- } else {
236- // Release file lock
237- drop ( file_shared) ;
238- // Put back the current file for further writing
239- let mut shared = self . shared . lock ( ) ;
240- shared. current_write_file = Some ( current_file) ;
221+ if max_file_size_reached {
222+ // Finish the IPC writer
223+ if let Some ( mut writer) = file_shared. writer . take ( ) {
224+ writer. finish ( ) ?;
241225 }
226+ // Mark as finished so readers know not to wait for more data
227+ file_shared. writer_finished = true ;
228+ // Wake reader waiting on this file (it's now finished)
229+ file_shared. wake ( ) ;
230+
231+ // Don't place `write_file` back in the `open_write_files` queue so we don't
232+ // try writing to it again
233+ } else {
234+ // Release file lock
235+ drop ( file_shared) ;
236+ // Put back the current file for further writing
237+ let mut shared = self . shared . lock ( ) ;
238+ shared. open_write_files . push_back ( write_file) ;
242239 }
243240
244241 Ok ( ( ) )
@@ -258,31 +255,32 @@ impl Drop for SpillPoolWriter {
258255 return ;
259256 }
260257
261- // Finalize the current file when the last writer is dropped
262- if let Some ( current_file ) = shared. current_write_file . take ( ) {
263- // Release shared lock before locking file
258+ // Finalize any spill files that were not finished yet
259+ if ! shared. open_write_files . is_empty ( ) {
260+ let files = mem :: take ( & mut shared . open_write_files ) ;
264261 drop ( shared) ;
265262
266- let mut file_shared = current_file. lock ( ) ;
263+ for file in files {
264+ let mut file_shared = file. lock ( ) ;
267265
268- // Finish the current writer if it exists
269- if let Some ( mut writer) = file_shared. writer . take ( ) {
270- // Ignore errors on drop - we're in destructor
271- let _ = writer. finish ( ) ;
272- }
266+ // Finish the current writer if it exists
267+ if let Some ( mut writer) = file_shared. writer . take ( ) {
268+ // Ignore errors on drop - we're in destructor
269+ let _ = writer. finish ( ) ;
270+ }
273271
274- // Mark as finished so readers know not to wait for more data
275- file_shared. writer_finished = true ;
272+ // Mark as finished so readers know not to wait for more data
273+ file_shared. writer_finished = true ;
276274
277- // Wake reader waiting on this file (it's now finished)
278- file_shared. wake ( ) ;
275+ // Wake reader waiting on this file (it's now finished)
276+ file_shared. wake ( ) ;
277+ drop ( file_shared) ;
278+ }
279279
280- drop ( file_shared) ;
281280 shared = self . shared . lock ( ) ;
282281 }
283282
284- // Mark writer as dropped and wake pool-level readers
285- shared. writer_dropped = true ;
283+ // Wake pool-level readers
286284 shared. wake ( ) ;
287285 }
288286}
@@ -722,7 +720,7 @@ impl Stream for SpillPoolReader {
722720 }
723721
724722 // No files in queue - check if writer is done
725- if shared. writer_dropped {
723+ if shared. active_writer_count == 0 {
726724 // Writer is done and no more files will be added - EOF
727725 return Poll :: Ready ( None ) ;
728726 }
@@ -746,7 +744,7 @@ mod tests {
746744 use crate :: metrics:: { ExecutionPlanMetricsSet , SpillMetrics } ;
747745 use arrow:: array:: { ArrayRef , Int32Array } ;
748746 use arrow:: datatypes:: { DataType , Field , Schema } ;
749- use datafusion_common_runtime:: SpawnedTask ;
747+ use datafusion_common_runtime:: { JoinSet , SpawnedTask } ;
750748 use datafusion_execution:: runtime_env:: RuntimeEnv ;
751749
752750 fn create_test_schema ( ) -> SchemaRef {
@@ -1204,6 +1202,57 @@ mod tests {
12041202 Ok ( ( ) )
12051203 }
12061204
1205+ #[ tokio:: test( flavor = "multi_thread" , worker_threads = 10 ) ]
1206+ async fn test_concurrent_writers ( ) -> Result < ( ) > {
1207+ let ( writer, mut reader) = create_spill_channel ( 1024 * 1024 ) ;
1208+
1209+ // Spawn writer tasks
1210+ let mut writer_join_set = JoinSet :: new ( ) ;
1211+ for w in 0 ..10 {
1212+ let writer = writer. clone ( ) ;
1213+ writer_join_set. spawn ( async move {
1214+ for b in 0 ..10 {
1215+ let batch = create_test_batch ( ( w * 100 ) + ( b * 10 ) , 10 ) ;
1216+ writer. push_batch ( & batch) . unwrap ( ) ;
1217+ }
1218+ } ) ;
1219+ }
1220+ drop ( writer) ;
1221+
1222+ // Reader task (runs concurrently)
1223+ let reader_handle = SpawnedTask :: spawn ( async move {
1224+ let mut batch_order = vec ! [ ] ;
1225+ loop {
1226+ match reader. next ( ) . await {
1227+ None => break ,
1228+ Some ( batch) => {
1229+ let batch = batch. unwrap ( ) ;
1230+
1231+ assert_eq ! ( batch. num_rows( ) , 10 ) ;
1232+
1233+ let col = batch
1234+ . column ( 0 )
1235+ . as_any ( )
1236+ . downcast_ref :: < Int32Array > ( )
1237+ . unwrap ( ) ;
1238+ batch_order. push ( col. value ( 0 ) / 10 ) ;
1239+ }
1240+ }
1241+ }
1242+ batch_order
1243+ } ) ;
1244+
1245+ // Wait for both to complete
1246+ writer_join_set. join_all ( ) . await ;
1247+ let mut batch_order = reader_handle. await . unwrap ( ) ;
1248+
1249+ // When used with multiple writers, order is not guaranteed
1250+ batch_order. sort ( ) ;
1251+ assert_eq ! ( batch_order, ( 0i32 ..100i32 ) . collect:: <Vec <_>>( ) ) ;
1252+
1253+ Ok ( ( ) )
1254+ }
1255+
12071256 #[ tokio:: test]
12081257 async fn test_reader_catches_up_to_writer ( ) -> Result < ( ) > {
12091258 let ( writer, mut reader) = create_spill_channel ( 1024 * 1024 ) ;
0 commit comments