@@ -419,8 +419,16 @@ impl ExternalSorter {
419419 self . metrics . spill_metrics . spill_file_count . value ( )
420420 }
421421
422- /// Appends a globally sorted batch, retaining its reservation until written.
423- async fn consume_and_spill_append ( & mut self , batch : RecordBatch ) -> Result < ( ) > {
422+ /// Appending globally sorted batches to the in-progress spill file, and clears
423+ /// the `globally_sorted_batches` (also its memory reservation) afterwards.
424+ async fn consume_and_spill_append (
425+ & mut self ,
426+ globally_sorted_batches : & mut Vec < RecordBatch > ,
427+ ) -> Result < ( ) > {
428+ if globally_sorted_batches. is_empty ( ) {
429+ return Ok ( ( ) ) ;
430+ }
431+
424432 // Lazily initialize the in-progress spill file
425433 if self . in_progress_spill_file . is_none ( ) {
426434 self . in_progress_spill_file =
@@ -429,7 +437,8 @@ impl ExternalSorter {
429437
430438 debug ! ( "Spilling sort data of ExternalSorter to disk whilst inserting" ) ;
431439
432- // Keep the reservation alive while the batch remains in memory across
440+ let batches_to_spill = std:: mem:: take ( globally_sorted_batches) ;
441+ // Keep the reservation alive while the batches remain in memory across
433442 // asynchronous writes. It is released on success or error via RAII.
434443 let _spill_reservation = self . reservation . take ( ) ;
435444
@@ -438,8 +447,16 @@ impl ExternalSorter {
438447 internal_datafusion_err ! ( "In-progress spill file should be initialized" )
439448 } ) ?;
440449
441- let gc_sliced_size = in_progress_file. append_batch_async ( & batch) . await ?;
442- * max_record_batch_size = ( * max_record_batch_size) . max ( gc_sliced_size) ;
450+ for batch in batches_to_spill {
451+ let gc_sliced_size = in_progress_file. append_batch_async ( & batch) . await ?;
452+
453+ * max_record_batch_size = ( * max_record_batch_size) . max ( gc_sliced_size) ;
454+ }
455+
456+ assert_or_internal_err ! (
457+ globally_sorted_batches. is_empty( ) ,
458+ "This function consumes globally_sorted_batches, so it should be empty after taking."
459+ ) ;
443460
444461 Ok ( ( ) )
445462 }
@@ -503,35 +520,52 @@ impl ExternalSorter {
503520 self . in_mem_batches. is_empty( ) ,
504521 "in_mem_batches should be empty after constructing sorted stream"
505522 ) ;
523+ // 'global' here refers to all buffered batches when the memory limit is
524+ // reached. This variable will buffer the sorted batches after
525+ // sort-preserving merge and incrementally append to spill files.
526+ let mut globally_sorted_batches: Vec < RecordBatch > = vec ! [ ] ;
527+
506528 while let Some ( batch) = sorted_stream. next ( ) . await {
507529 let batch = batch?;
508- // Sorting is complete: retain the batch's footprint, not the input
509- // estimate that also budgets for creating a sorted copy.
530+ // The output is already sorted, so no sorted-copy budget is needed.
510531 let sorted_size = get_record_batch_memory_size ( & batch) ;
511532 let spill_workspace = match self . reservation . try_grow ( sorted_size) {
512533 Ok ( ( ) ) => None ,
513534 Err ( _) => {
514- // Reuse already-reserved workspace without bypassing the
515- // execution pool's limit. Any remainder still needs a grant
516- // through the original sort consumer.
517535 let workspace = self . merge_pool . borrow ( sorted_size) ;
518- self . reservation . try_grow ( sorted_size - workspace. size ( ) ) ?;
536+ // The batch is already materialized, so account for it while
537+ // spilling, reusing available workspace before growing the
538+ // reservation even if this exceeds the pool limit.
539+ self . reservation . grow ( sorted_size - workspace. size ( ) ) ;
519540 Some ( workspace)
520541 }
521542 } ;
522- // Write each batch before polling the merge again. Accumulating
523- // output would compete with the merge's workspace without combining
524- // any writes. Keep both forms of reservation alive across the await.
525- self . consume_and_spill_append ( batch) . await ?;
526- drop ( spill_workspace) ;
543+ // Even if the reservation is not enough, the batch is already in
544+ // memory, so it's okay to combine it with previously sorted
545+ // batches, and spill together.
546+ globally_sorted_batches. push ( batch) ;
547+ if let Some ( _spill_workspace) = spill_workspace {
548+ self . consume_and_spill_append ( & mut globally_sorted_batches)
549+ . await ?; // reservation is released when the spill completes
550+ }
527551 }
528552
529553 // Drop early to free up memory reserved by the sorted stream, otherwise the
530554 // upcoming `self.reserve_memory_for_merge()` may fail due to insufficient memory.
531555 drop ( sorted_stream) ;
532556
557+ self . consume_and_spill_append ( & mut globally_sorted_batches)
558+ . await ?;
533559 self . spill_finish ( ) . await ?;
534560
561+ // Sanity check after spilling
562+ let buffers_cleared_property =
563+ self . in_mem_batches . is_empty ( ) && globally_sorted_batches. is_empty ( ) ;
564+ assert_or_internal_err ! (
565+ buffers_cleared_property,
566+ "in_mem_batches and globally_sorted_batches should be cleared before"
567+ ) ;
568+
535569 // Reserve headroom for next sort/merge
536570 self . reserve_memory_for_merge ( ) ?;
537571
@@ -3355,32 +3389,6 @@ mod tests {
33553389 Ok ( ( ) )
33563390 }
33573391
3358- #[ tokio:: test]
3359- async fn test_spill_output_respects_memory_limit ( ) -> Result < ( ) > {
3360- let result = test_sort_output_batch_size_and_base_metrics ( 10 , 25 , |batches| {
3361- let batches_memory = batches. iter ( ) . map ( |b| b. get_array_memory_size ( ) ) . sum ( ) ;
3362- TaskContext :: default ( )
3363- . with_session_config (
3364- SessionConfig :: new ( )
3365- . with_batch_size ( 100 )
3366- . with_sort_in_place_threshold_bytes ( 1 )
3367- . with_sort_spill_reservation_bytes ( 1 ) ,
3368- )
3369- . with_runtime (
3370- RuntimeEnvBuilder :: default ( )
3371- . with_memory_limit ( batches_memory, 1.0 )
3372- . build_arc ( )
3373- . unwrap ( ) ,
3374- )
3375- } )
3376- . await ;
3377- assert ! ( matches!(
3378- result,
3379- Err ( DataFusionError :: ResourcesExhausted ( _) )
3380- ) ) ;
3381- Ok ( ( ) )
3382- }
3383-
33843392 #[ tokio:: test]
33853393 async fn should_return_stream_with_batches_in_the_requested_size_and_update_metrics_when_having_to_spill ( )
33863394 -> Result < ( ) > {
@@ -3391,23 +3399,18 @@ mod tests {
33913399 . iter ( )
33923400 . map ( |b| b. get_array_memory_size ( ) )
33933401 . sum :: < usize > ( ) ;
3394- // Leave space for one output batch while the merge still holds
3395- // partially consumed input batches. The insufficient-budget case
3396- // is covered by test_spill_output_respects_memory_limit.
3397- let spill_workspace =
3398- make_partition ( batch_size as i32 ) . get_array_memory_size ( ) ;
33993402
34003403 TaskContext :: default ( )
34013404 . with_session_config (
34023405 SessionConfig :: new ( )
34033406 . with_batch_size ( batch_size)
34043407 // To make sure there is no in place sorting
34053408 . with_sort_in_place_threshold_bytes ( 1 )
3406- . with_sort_spill_reservation_bytes ( spill_workspace ) ,
3409+ . with_sort_spill_reservation_bytes ( 1 ) ,
34073410 )
34083411 . with_runtime (
34093412 RuntimeEnvBuilder :: default ( )
3410- . with_memory_limit ( batches_memory + spill_workspace , 1.0 )
3413+ . with_memory_limit ( batches_memory, 1.0 )
34113414 . build_arc ( )
34123415 . unwrap ( ) ,
34133416 )
@@ -3955,29 +3958,15 @@ mod tests {
39553958 Ok ( ( ) )
39563959 }
39573960
3961+ #[ rstest:: rstest]
3962+ #[ case:: no_workspace( 0 ) ]
3963+ #[ case:: partial_workspace( 6 ) ]
3964+ #[ case:: full_workspace( 12 ) ]
39583965 #[ tokio:: test]
3959- async fn test_spill_reservation_held_during_async_write ( ) -> Result < ( ) > {
3960- let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new(
3961- "x" ,
3962- DataType :: Utf8View ,
3963- false ,
3964- ) ] ) ) ;
3965- let batch = RecordBatch :: try_new (
3966- Arc :: clone ( & schema) ,
3967- vec ! [ Arc :: new( StringViewArray :: from_iter_values(
3968- ( 0 ..4096 ) . rev( ) . map( |i| format!( "{i:08}{}" , "x" . repeat( 87 ) ) ) ,
3969- ) ) ] ,
3970- ) ?;
3971- let ordering =
3972- [ PhysicalSortExpr :: new_default ( Arc :: new ( Column :: new ( "x" , 0 ) ) ) ] . into ( ) ;
3973- let batch_size = 1024 ;
3974- let sorted_bytes = sort_batch_chunked ( & batch, & ordering, batch_size) ?
3975- . iter ( )
3976- . map ( get_record_batch_memory_size)
3977- . sum :: < usize > ( ) ;
3978- let workspace_bytes = sorted_bytes - get_reserved_bytes_for_record_batch ( & batch) ?;
3979- assert ! ( workspace_bytes > 0 ) ;
3980- let pool = spill_tests:: AdjustablePool :: new ( sorted_bytes) ;
3966+ async fn test_spill_reservation_held_during_async_write (
3967+ #[ case] workspace_bytes : usize ,
3968+ ) -> Result < ( ) > {
3969+ let pool: Arc < dyn MemoryPool > = Arc :: new ( GreedyMemoryPool :: new ( 0 ) ) ;
39813970 let write_started = Arc :: new ( Notify :: new ( ) ) ;
39823971 let aborted = Arc :: new ( Notify :: new ( ) ) ;
39833972 let abort_count = Arc :: new ( AtomicUsize :: new ( 0 ) ) ;
@@ -3989,26 +3978,34 @@ mod tests {
39893978 } ) ,
39903979 ) ;
39913980 let runtime = RuntimeEnvBuilder :: new ( )
3992- . with_memory_pool ( Arc :: clone ( & pool) as Arc < dyn MemoryPool > )
3981+ . with_memory_pool ( Arc :: clone ( & pool) )
39933982 . with_disk_manager_builder ( disk_manager_builder)
39943983 . build_arc ( ) ?;
3984+ let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "x" , DataType :: Int32 , false ) ] ) ) ;
39953985 let metrics = ExecutionPlanMetricsSet :: new ( ) ;
39963986 let mut sorter = ExternalSorter :: new (
39973987 0 ,
3998- schema,
3999- ordering,
4000- batch_size,
4001- workspace_bytes,
3988+ Arc :: clone ( & schema) ,
3989+ [ PhysicalSortExpr :: new_default ( Arc :: new ( Column :: new ( "x" , 0 ) ) ) ] . into ( ) ,
3990+ 128 ,
40023991 0 ,
3992+ usize:: MAX ,
40033993 SpillCompression :: Uncompressed ,
40043994 & metrics,
40053995 runtime,
40063996 ) ?;
4007- sorter. insert_batch ( batch) . await ?;
4008- // Existing reservations remain valid, but the emitted batch must reuse
4009- // workspace rather than request new parent capacity.
4010- pool. set_limit ( sorted_bytes - 1 ) ;
3997+ let batch = RecordBatch :: try_new (
3998+ schema,
3999+ vec ! [ Arc :: new( Int32Array :: from( vec![ 3 , 2 , 1 ] ) ) ] ,
4000+ ) ?;
4001+ let reserved_bytes = get_record_batch_memory_size ( & batch) ;
4002+ sorter
4003+ . reservation
4004+ . grow ( get_reserved_bytes_for_record_batch ( & batch) ?) ;
4005+ sorter. merge_reservation . grow ( workspace_bytes) ;
4006+ sorter. merge_pool . retain ( workspace_bytes) ;
40114007 let merge_pool = Arc :: clone ( & sorter. merge_pool ) ;
4008+ sorter. in_mem_batches . push ( batch) ;
40124009
40134010 #[ expect( clippy:: disallowed_methods) ] // spawn allowed only in tests
40144011 let task =
@@ -4025,12 +4022,12 @@ mod tests {
40254022 let _ = task. await ;
40264023 panic ! ( "spill write did not start before the timeout" ) ;
40274024 }
4028- // Idle workspace must be releasable without releasing the loan held by
4029- // the pending write. The stream still owns the remaining sorted batches.
4025+ // An idle retained workspace must not mask an early release of the loan.
40304026 merge_pool. release_unused ( ) ;
4027+ assert_eq ! ( merge_pool. reserved( ) , workspace_bytes) ;
40314028 assert_eq ! (
40324029 pool. reserved( ) ,
4033- sorted_bytes ,
4030+ reserved_bytes ,
40344031 "resident batches must remain accounted for while an async spill is pending"
40354032 ) ;
40364033
0 commit comments