@@ -29,11 +29,9 @@ use super::{ExecutionPlan, RecordBatchStream, SendableRecordBatchStream};
2929use crate :: displayable;
3030
3131use arrow:: { datatypes:: SchemaRef , record_batch:: RecordBatch } ;
32- use datafusion_common:: utils:: memory:: RecordBatchMemoryCounter ;
3332use datafusion_common:: { Result , exec_err} ;
3433use datafusion_common_runtime:: JoinSet ;
3534use datafusion_execution:: TaskContext ;
36- use datafusion_execution:: memory_pool:: MemoryReservation ;
3735
3836use futures:: ready;
3937use futures:: stream:: BoxStream ;
@@ -742,79 +740,9 @@ impl RecordBatchStream for BatchSplitStream {
742740 }
743741}
744742
745- /// A stream that holds a memory reservation for its lifetime,
746- /// shrinking the reservation as batches are consumed.
747- /// The reservation must cover the batches as a [`RecordBatchMemoryCounter`] counts them in order.
748- /// On error, the reservation is *NOT* freed, until the stream is dropped.
749- pub ( crate ) struct ReservationStream {
750- schema : SchemaRef ,
751- inner : SendableRecordBatchStream ,
752- reservation : MemoryReservation ,
753- counter : RecordBatchMemoryCounter ,
754- }
755-
756- impl ReservationStream {
757- pub ( crate ) fn new (
758- schema : SchemaRef ,
759- inner : SendableRecordBatchStream ,
760- reservation : MemoryReservation ,
761- ) -> Self {
762- Self {
763- schema,
764- inner,
765- reservation,
766- counter : RecordBatchMemoryCounter :: new ( ) ,
767- }
768- }
769- }
770-
771- impl Stream for ReservationStream {
772- type Item = Result < RecordBatch > ;
773-
774- fn poll_next (
775- mut self : Pin < & mut Self > ,
776- cx : & mut Context < ' _ > ,
777- ) -> Poll < Option < Self :: Item > > {
778- let res = self . inner . poll_next_unpin ( cx) ;
779-
780- match res {
781- Poll :: Ready ( res) => {
782- match res {
783- Some ( Ok ( batch) ) => {
784- let size = self . counter . count_batch ( & batch) ;
785- self . reservation . shrink ( size) ;
786- Poll :: Ready ( Some ( Ok ( batch) ) )
787- }
788- Some ( Err ( err) ) => Poll :: Ready ( Some ( Err ( err) ) ) ,
789- None => {
790- // Stream is done so free the reservation completely
791- self . reservation . free ( ) ;
792- // Release the input pipeline's resources.
793- let inner_schema = self . inner . schema ( ) ;
794- self . inner = Box :: pin ( EmptyRecordBatchStream :: new ( inner_schema) ) ;
795- Poll :: Ready ( None )
796- }
797- }
798- }
799- Poll :: Pending => Poll :: Pending ,
800- }
801- }
802-
803- fn size_hint ( & self ) -> ( usize , Option < usize > ) {
804- self . inner . size_hint ( )
805- }
806- }
807-
808- impl RecordBatchStream for ReservationStream {
809- fn schema ( & self ) -> SchemaRef {
810- Arc :: clone ( & self . schema )
811- }
812- }
813-
814743#[ cfg( test) ]
815744mod test {
816745 use super :: * ;
817- use crate :: spill:: get_record_batch_memory_size;
818746 use crate :: test:: exec:: {
819747 BlockingExec , MockExec , PanicExec , assert_strong_count_converges_to_zero,
820748 } ;
@@ -1039,123 +967,4 @@ mod test {
1039967 "Should have received exactly two empty batches"
1040968 ) ;
1041969 }
1042-
1043- #[ tokio:: test]
1044- async fn test_reservation_stream_shrinks_on_poll ( ) {
1045- use arrow:: array:: Int32Array ;
1046- use datafusion_execution:: memory_pool:: MemoryConsumer ;
1047- use datafusion_execution:: runtime_env:: RuntimeEnvBuilder ;
1048-
1049- let runtime = RuntimeEnvBuilder :: new ( )
1050- . with_memory_limit ( 10 * 1024 * 1024 , 1.0 )
1051- . build_arc ( )
1052- . unwrap ( ) ;
1053-
1054- let reservation = MemoryConsumer :: new ( "test" ) . register ( & runtime. memory_pool ) ;
1055-
1056- let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ) ;
1057-
1058- // Create batches
1059- let batch1 = RecordBatch :: try_new (
1060- Arc :: clone ( & schema) ,
1061- vec ! [ Arc :: new( Int32Array :: from( vec![ 1 , 2 , 3 , 4 , 5 ] ) ) ] ,
1062- )
1063- . unwrap ( ) ;
1064- let batch2 = RecordBatch :: try_new (
1065- Arc :: clone ( & schema) ,
1066- vec ! [ Arc :: new( Int32Array :: from( vec![ 6 , 7 , 8 , 9 , 10 ] ) ) ] ,
1067- )
1068- . unwrap ( ) ;
1069-
1070- let batch1_size = get_record_batch_memory_size ( & batch1) ;
1071- let batch2_size = get_record_batch_memory_size ( & batch2) ;
1072-
1073- // Reserve memory upfront
1074- reservation. try_grow ( batch1_size + batch2_size) . unwrap ( ) ;
1075- let initial_reserved = runtime. memory_pool . reserved ( ) ;
1076- assert_eq ! ( initial_reserved, batch1_size + batch2_size) ;
1077-
1078- // Create stream with batches
1079- let stream = futures:: stream:: iter ( vec ! [ Ok ( batch1) , Ok ( batch2) ] ) ;
1080- let inner = Box :: pin ( RecordBatchStreamAdapter :: new ( Arc :: clone ( & schema) , stream) )
1081- as SendableRecordBatchStream ;
1082-
1083- let mut res_stream =
1084- ReservationStream :: new ( Arc :: clone ( & schema) , inner, reservation) ;
1085-
1086- // Poll first batch
1087- let result1 = res_stream. next ( ) . await ;
1088- assert ! ( result1. is_some( ) ) ;
1089-
1090- // Memory should be reduced by batch1_size
1091- let after_first = runtime. memory_pool . reserved ( ) ;
1092- assert_eq ! ( after_first, batch2_size) ;
1093-
1094- // Poll second batch
1095- let result2 = res_stream. next ( ) . await ;
1096- assert ! ( result2. is_some( ) ) ;
1097-
1098- // Memory should be reduced by batch2_size
1099- let after_second = runtime. memory_pool . reserved ( ) ;
1100- assert_eq ! ( after_second, 0 ) ;
1101-
1102- // Poll None (end of stream)
1103- let result3 = res_stream. next ( ) . await ;
1104- assert ! ( result3. is_none( ) ) ;
1105-
1106- // Memory should still be 0
1107- assert_eq ! ( runtime. memory_pool. reserved( ) , 0 ) ;
1108- }
1109-
1110- #[ tokio:: test]
1111- async fn test_reservation_stream_error_handling ( ) {
1112- use datafusion_execution:: memory_pool:: MemoryConsumer ;
1113- use datafusion_execution:: runtime_env:: RuntimeEnvBuilder ;
1114-
1115- let runtime = RuntimeEnvBuilder :: new ( )
1116- . with_memory_limit ( 10 * 1024 * 1024 , 1.0 )
1117- . build_arc ( )
1118- . unwrap ( ) ;
1119-
1120- let reservation = MemoryConsumer :: new ( "test" ) . register ( & runtime. memory_pool ) ;
1121-
1122- let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ) ;
1123-
1124- reservation. try_grow ( 1000 ) . unwrap ( ) ;
1125- let initial = runtime. memory_pool . reserved ( ) ;
1126- assert_eq ! ( initial, 1000 ) ;
1127-
1128- // Create a stream that errors
1129- let stream = futures:: stream:: iter ( vec ! [ exec_err!( "Test error" ) ] ) ;
1130- let inner = Box :: pin ( RecordBatchStreamAdapter :: new ( Arc :: clone ( & schema) , stream) )
1131- as SendableRecordBatchStream ;
1132-
1133- let mut res_stream =
1134- ReservationStream :: new ( Arc :: clone ( & schema) , inner, reservation) ;
1135-
1136- // Get the error
1137- let result = res_stream. next ( ) . await ;
1138- assert ! ( result. is_some( ) ) ;
1139- assert ! ( result. unwrap( ) . is_err( ) ) ;
1140-
1141- // Verify reservation is NOT automatically freed on error
1142- // The reservation is only freed when poll_next returns Poll::Ready(None)
1143- // After an error, the stream may continue to hold the reservation
1144- // until it's explicitly dropped or polled to None
1145- let after_error = runtime. memory_pool . reserved ( ) ;
1146- assert_eq ! (
1147- after_error, 1000 ,
1148- "Reservation should still be held after error"
1149- ) ;
1150-
1151- // Drop the stream to free the reservation
1152- drop ( res_stream) ;
1153-
1154- // Now memory should be freed
1155- assert_eq ! (
1156- runtime. memory_pool. reserved( ) ,
1157- 0 ,
1158- "Memory should be freed when stream is dropped"
1159- ) ;
1160- }
1161970}
0 commit comments