@@ -1057,7 +1057,8 @@ fn get_reserved_bytes_for_next_record_batch(
10571057/// Sorting takes the selected rows into new buffers, except that `take` keeps
10581058/// the data buffers of view arrays and the values of dictionaries and list
10591059/// views. Every sorted run is charged for those buffers in full, so buffered
1060- /// batches that share them must be charged for them each, too.
1060+ /// batches that share them must be charged for them each, too. Otherwise the
1061+ /// sorter buffers more batches than the merge of their sorted runs can hold.
10611062fn sorted_copy_keeps_buffers ( data_type : & DataType ) -> bool {
10621063 match data_type {
10631064 DataType :: Utf8View
@@ -3899,60 +3900,6 @@ mod tests {
38993900 Ok ( ( ) )
39003901 }
39013902
3902- /// Zero-copy slices of one batch share its buffers, so buffering them
3903- /// reserves those buffers once. Sorting the slices as separate runs must
3904- /// keep the shared buffers reserved while any run still holds them.
3905- #[ tokio:: test]
3906- async fn test_sliced_runs_reserve_shared_buffers_once ( ) -> Result < ( ) > {
3907- let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "x" , DataType :: Int64 , false ) ] ) ) ;
3908- let parent = RecordBatch :: try_new (
3909- Arc :: clone ( & schema) ,
3910- vec ! [ Arc :: new( Int64Array :: from_iter_values( ( 0 ..4096 ) . rev( ) ) ) ] ,
3911- ) ?;
3912- let slices: Vec < _ > = ( 0 ..4 ) . map ( |i| parent. slice ( i * 1024 , 1024 ) ) . collect ( ) ;
3913-
3914- let pool: Arc < dyn MemoryPool > = Arc :: new ( GreedyMemoryPool :: new ( 1024 * 1024 ) ) ;
3915- let runtime = RuntimeEnvBuilder :: new ( )
3916- . with_memory_pool ( Arc :: clone ( & pool) )
3917- . build_arc ( ) ?;
3918- let ordering: LexOrdering =
3919- [ PhysicalSortExpr :: new_default ( Arc :: new ( Column :: new ( "x" , 0 ) ) ) ] . into ( ) ;
3920- let mut sorter = ExternalSorter :: new (
3921- 0 ,
3922- Arc :: clone ( & schema) ,
3923- ordering. clone ( ) ,
3924- 1024 ,
3925- 0 ,
3926- 0 , // Sort each slice as its own run, rather than concatenating them.
3927- SpillCompression :: Uncompressed ,
3928- & ExecutionPlanMetricsSet :: new ( ) ,
3929- runtime,
3930- ) ?;
3931- for slice in & slices {
3932- sorter. insert_batch ( slice. clone ( ) ) . await ?;
3933- }
3934- let parent_bytes = get_record_batch_memory_size ( & parent) ;
3935- let sliced_bytes = slices
3936- . iter ( )
3937- . map ( |slice| slice. get_sliced_size ( ) )
3938- . sum :: < Result < usize > > ( ) ?;
3939- assert_eq ! ( pool. reserved( ) , parent_bytes + sliced_bytes) ;
3940-
3941- let runs = std:: mem:: take ( & mut sorter. in_mem_batches ) ;
3942- let mut streams = sorter. sort_run_streams ( runs) ?;
3943- let last = streams. pop ( ) . unwrap ( ) ;
3944- for stream in streams {
3945- stream. try_collect :: < Vec < _ > > ( ) . await ?;
3946- }
3947- // The last run still holds the parent's buffers until it is sorted
3948- assert_eq ! ( pool. reserved( ) , parent_bytes + slices[ 3 ] . get_sliced_size( ) ?) ;
3949-
3950- let sorted = concat_batches ( & schema, & last. try_collect :: < Vec < _ > > ( ) . await ?) ?;
3951- assert_eq ! ( sorted, sort_batch( & slices[ 3 ] , & ordering, None ) ?) ;
3952- assert_eq ! ( pool. reserved( ) , 0 ) ;
3953- Ok ( ( ) )
3954- }
3955-
39563903 #[ test]
39573904 fn test_with_dynamic_filter ( ) -> Result < ( ) > {
39583905 let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ) ;
0 commit comments