@@ -82,8 +82,9 @@ use crate::{RecordBatchStream, SendableRecordBatchStream};
8282/// only the current group remains active after completed groups are emitted, and
8383/// memory usage does not grow with the total number of groups.
8484///
85- /// If a memory reservation nevertheless fails, the stream returns the error
86- /// directly, indicating an unexpected behavior.
85+ /// If the reservation after an input batch fails, `ClusteredSingleAggregateStream`
86+ /// still emits the completed groups, and returns the error only when no group is
87+ /// complete.
8788///
8889/// ## Partial group clustering
8990///
@@ -335,7 +336,27 @@ impl ClusteredSingleAggregateStream {
335336 let Some ( spill_context) = spill_context else {
336337 // `None` means spilling is not supported, see comments
337338 // at `ClusteredSingleAggregateState` for details.
338- return Self :: break_with_err ( e) ;
339+ // Emitting completed groups is then the only way to
340+ // release memory. Fail only when none is complete.
341+ let timer = elapsed_compute. timer ( ) ;
342+ let result = table. take_completed_result_batch ( ) ;
343+ timer. done ( ) ;
344+ return match result {
345+ Ok ( Some ( batch) ) => {
346+ let table_memory =
347+ Self :: reservation_size_for_table ( & table, None ) ;
348+ self . start_outputting (
349+ batch,
350+ table_memory,
351+ ClusteredSingleAggregateState :: ReadingInput {
352+ table,
353+ spill_context : None ,
354+ } ,
355+ )
356+ }
357+ Ok ( None ) => Self :: break_with_err ( e) ,
358+ Err ( e) => Self :: break_with_err ( e) ,
359+ } ;
339360 } ;
340361 if table. is_empty ( ) {
341362 return Self :: break_with_internal_err (
@@ -766,10 +787,10 @@ mod tests {
766787 use crate :: test:: TestMemoryExec ;
767788 use arrow:: array:: Int64Array ;
768789 use arrow:: datatypes:: { DataType , Field , Schema } ;
769- use datafusion_common:: test_util:: batches_to_sort_string;
790+ use datafusion_common:: test_util:: { batches_to_sort_string, batches_to_string } ;
770791 use datafusion_execution:: config:: SessionConfig ;
771792 use datafusion_execution:: memory_pool:: {
772- GreedyMemoryPool , MemoryPool , PeakRecordingPool ,
793+ GreedyMemoryPool , MemoryPool , PeakRecordingPool , UnboundedMemoryPool ,
773794 } ;
774795 use datafusion_execution:: runtime_env:: RuntimeEnvBuilder ;
775796 use datafusion_functions_aggregate:: sum:: sum_udaf;
@@ -851,4 +872,70 @@ mod tests {
851872 ) ;
852873 Ok ( ( ) )
853874 }
875+
876+ /// Runs a single `SUM(k) GROUP BY k` over two batches sorted on `k` that
877+ /// share a boundary key. Returns the output and the most memory the stream
878+ /// held while handing off output.
879+ async fn run_fully_sorted (
880+ pool : Arc < dyn MemoryPool > ,
881+ ) -> Result < ( Vec < RecordBatch > , usize ) > {
882+ let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "k" , DataType :: Int64 , false ) ] ) ) ;
883+ let batches = [ 0 ..=256 , 256 ..=512 ]
884+ . into_iter ( )
885+ . map ( |keys| {
886+ let k = Int64Array :: from_iter_values ( keys) ;
887+ RecordBatch :: try_new ( Arc :: clone ( & schema) , vec ! [ Arc :: new( k) ] )
888+ } )
889+ . collect :: < Result < Vec < _ > , _ > > ( ) ?;
890+ let input = TestMemoryExec :: try_new ( & [ batches] , Arc :: clone ( & schema) , None ) ?
891+ . try_with_sort_information ( vec ! [
892+ LexOrdering :: new( [ PhysicalSortExpr :: new_default( col( "k" , & schema) ?) ] )
893+ . unwrap( ) ,
894+ ] ) ?;
895+ let aggregate = AggregateExec :: try_new (
896+ AggregateMode :: Single ,
897+ PhysicalGroupBy :: new_single ( vec ! [ ( col( "k" , & schema) ?, "k" . into( ) ) ] ) ,
898+ vec ! [ Arc :: new(
899+ AggregateExprBuilder :: new( sum_udaf( ) , vec![ col( "k" , & schema) ?] )
900+ . schema( Arc :: clone( & schema) )
901+ . alias( "sum" )
902+ . build( ) ?,
903+ ) ] ,
904+ vec ! [ None ] ,
905+ Arc :: new ( TestMemoryExec :: update_cache ( & Arc :: new ( input) ) ) ,
906+ Arc :: clone ( & schema) ,
907+ ) ?;
908+ assert ! ( matches!(
909+ aggregate. group_clustering_mode,
910+ GroupClusteringMode :: Full
911+ ) ) ;
912+ let context = Arc :: new (
913+ TaskContext :: default ( ) . with_runtime (
914+ RuntimeEnvBuilder :: new ( )
915+ . with_memory_pool ( Arc :: clone ( & pool) )
916+ . build_arc ( ) ?,
917+ ) ,
918+ ) ;
919+ let mut stream = ClusteredSingleAggregateStream :: new ( & aggregate, & context, 0 ) ?;
920+ // Each output batch is a single slice, so the stream releases it before
921+ // handing it off and holds only the table.
922+ let mut held = pool. reserved ( ) ;
923+ let mut output = vec ! [ ] ;
924+ while let Some ( batch) = stream. next ( ) . await . transpose ( ) ? {
925+ held = held. max ( pool. reserved ( ) ) ;
926+ output. push ( batch) ;
927+ }
928+ Ok ( ( output, held) )
929+ }
930+
931+ /// The limit fits the table with only the active group, but not the table
932+ /// before the completed groups are emitted. The result must not change.
933+ #[ tokio:: test]
934+ async fn fully_sorted_completed_groups_under_memory_pressure ( ) -> Result < ( ) > {
935+ let ( expected, held) =
936+ run_fully_sorted ( Arc :: new ( UnboundedMemoryPool :: default ( ) ) ) . await ?;
937+ let ( actual, _) = run_fully_sorted ( Arc :: new ( GreedyMemoryPool :: new ( held) ) ) . await ?;
938+ assert_eq ! ( batches_to_string( & actual) , batches_to_string( & expected) ) ;
939+ Ok ( ( ) )
940+ }
854941}
0 commit comments