@@ -791,7 +791,9 @@ impl ExternalSorter {
791791 concurrency : & SortConcurrency ,
792792 ) -> Result < Vec < SendableRecordBatchStream > > {
793793 let mut streams: Vec < _ > = ( 0 ..runs. len ( ) ) . map ( |_| None ) . collect ( ) ;
794- let ( sizes, shared_input) = self . sort_run_reservations ( & runs) ?;
794+ let ( sizes, shared_input_size) = self . sort_run_sizes ( & runs) ?;
795+ let shared_input =
796+ shared_input_size. map ( |size| Arc :: new ( self . reservation . split ( size) ) ) ;
795797 let jobs =
796798 runs. into_iter ( )
797799 . zip ( sizes)
@@ -927,17 +929,19 @@ impl ExternalSorter {
927929 . map ( |_| RecordBatchMemoryCounter :: new ( ) )
928930 }
929931
930- /// Calculates each run's reservation size and reserves any shared input.
932+ /// Calculates each run's reservation size and any shared input size .
931933 ///
932934 /// Runs can share buffers, for example zero-copy slices of one larger batch,
933935 /// and `self.reservation` covers each shared buffer once. A shared buffer
934936 /// stays alive until the last run holding it is sorted, so if any buffers are
935937 /// shared, the reservation for every run's input is held until all the runs
936938 /// are sorted. Otherwise each run releases its input once it is sorted.
937- fn sort_run_reservations (
939+ /// With shared input, each run's size covers only its sorted copy. Otherwise
940+ /// it also includes that run's input.
941+ fn sort_run_sizes (
938942 & self ,
939943 runs : & [ RecordBatch ] ,
940- ) -> Result < ( Vec < usize > , Option < Arc < MemoryReservation > > ) > {
944+ ) -> Result < ( Vec < usize > , Option < usize > ) > {
941945 let mut counter = self . new_buffered_batches_counter ( ) ;
942946 let mut shares_buffers = false ;
943947 let mut sizes = Vec :: with_capacity ( runs. len ( ) ) ;
@@ -950,19 +954,17 @@ impl ExternalSorter {
950954 sizes. push ( ( retained, run. get_sliced_size ( ) ?) ) ;
951955 }
952956
953- let shared_input = shares_buffers. then ( || {
954- let retained = sizes. iter ( ) . map ( |( retained, _) | retained) . sum ( ) ;
955- Arc :: new ( self . reservation . split ( retained) )
956- } ) ;
957+ let shared_input_size =
958+ shares_buffers. then ( || sizes. iter ( ) . map ( |( retained, _) | retained) . sum ( ) ) ;
957959
958960 let sizes = sizes
959961 . into_iter ( )
960- . map ( |( retained, sliced) | match shared_input {
962+ . map ( |( retained, sliced) | match shared_input_size {
961963 Some ( _) => sliced,
962964 None => get_reserved_bytes_for_record_batch_size ( retained, sliced) ,
963965 } )
964966 . collect ( ) ;
965- Ok ( ( sizes, shared_input ) )
967+ Ok ( ( sizes, shared_input_size ) )
966968 }
967969
968970 /// Sorts each run into its own stream, splitting `self.reservation` between
@@ -971,7 +973,9 @@ impl ExternalSorter {
971973 & self ,
972974 runs : Vec < RecordBatch > ,
973975 ) -> Result < Vec < SendableRecordBatchStream > > {
974- let ( sizes, shared_input) = self . sort_run_reservations ( & runs) ?;
976+ let ( sizes, shared_input_size) = self . sort_run_sizes ( & runs) ?;
977+ let shared_input =
978+ shared_input_size. map ( |size| Arc :: new ( self . reservation . split ( size) ) ) ;
975979 runs. into_iter ( )
976980 . zip ( sizes)
977981 . map ( |( run, size) | {
0 commit comments