@@ -181,7 +181,8 @@ pub fn project_plan_to_schema(
181181}
182182
183183/// If running in a tokio context spawns the execution of `stream` to a separate task
184- /// allowing it to execute in parallel with an intermediate buffer of size `buffer`
184+ /// allowing it to execute in parallel with an intermediate buffer of size `buffer`.
185+ /// At most `buffer` record batches will be produced ahead of the consumer.
185186pub fn spawn_buffered (
186187 mut input : SendableRecordBatchStream ,
187188 buffer : usize ,
@@ -196,11 +197,22 @@ pub fn spawn_buffered(
196197 let sender = builder. tx ( ) ;
197198
198199 builder. spawn ( async move {
199- while let Some ( item) = input. next ( ) . await {
200- if sender. send ( item) . await . is_err ( ) {
201- // Receiver dropped when query is shutdown early (e.g., limit) or error,
202- // no need to return propagate the send error.
203- return Ok ( ( ) ) ;
200+ // We call `reserve` (which waits until there's room for at least 1 message in the
201+ // channel buffer) **before** polling from input to ensure we hold a maximum of
202+ // `buffer` record batches in memory.
203+ // Polling from input and then calling send() would block when the channel is full
204+ // so it would essentially hold `buffer` + 1 record batches:
205+ // * `buffer`: this many elements would live inside the channel, since this is the
206+ // channel's capacity
207+ // * 1 extra RecordBatch which was produced, but there was no room for it in the
208+ // channel, so it's being owned by the send() future, which keeps the batch in
209+ // memory while it waits for a slot to free up
210+ while let Ok ( permit) = sender. reserve ( ) . await {
211+ // Receiver dropped when query is shutdown early (e.g., limit) or error,
212+ // no need to return propagate the send error.
213+ match input. next ( ) . await {
214+ Some ( item) => permit. send ( item) ,
215+ None => break ,
204216 }
205217 }
206218
@@ -298,10 +310,13 @@ mod tests {
298310 use crate :: empty:: EmptyExec ;
299311 use crate :: projection:: ProjectionExec ;
300312
313+ use crate :: stream:: RecordBatchStreamAdapter ;
314+ use futures:: stream;
301315 use std:: collections:: HashMap ;
316+ use std:: sync:: atomic:: { AtomicUsize , Ordering } ;
302317
303318 use arrow:: {
304- array:: { Float32Array , Float64Array , UInt64Array } ,
319+ array:: { Float32Array , Float64Array , Int32Array , UInt64Array } ,
305320 datatypes:: { DataType , Field , Schema } ,
306321 } ;
307322
@@ -554,4 +569,55 @@ mod tests {
554569 let err = project_plan_to_schema ( input, & expected_schema) . unwrap_err ( ) ;
555570 assert ! ( err. to_string( ) . contains( "schema metadata differ" ) ) ;
556571 }
572+
573+ /// Verifies that `spawn_buffered` holds exactly `buffer` record batches in memory
574+ /// when no receiver is polling
575+ async fn spawn_buffered_max_in_flight_batches ( buffer_size : usize ) {
576+ let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new( "a" , DataType :: Int32 , false ) ] ) ) ;
577+ let num_batches = 10 ;
578+
579+ let produced_count = Arc :: new ( AtomicUsize :: new ( 0 ) ) ;
580+ let produced_clone = Arc :: clone ( & produced_count) ;
581+ let schema_clone = Arc :: clone ( & schema) ;
582+
583+ // Stream increments the counter each time a batch is pulled by the producer.
584+ let input_stream = stream:: unfold ( 0usize , move |i| {
585+ let schema = Arc :: clone ( & schema_clone) ;
586+ let counter = Arc :: clone ( & produced_clone) ;
587+ async move {
588+ if i >= num_batches {
589+ return None ;
590+ }
591+ let batch = RecordBatch :: try_new (
592+ Arc :: clone ( & schema) ,
593+ vec ! [ Arc :: new( Int32Array :: from( vec![ i as i32 ] ) ) ] ,
594+ )
595+ . unwrap ( ) ;
596+ counter. fetch_add ( 1 , Ordering :: SeqCst ) ;
597+ Some ( ( Ok ( batch) , i + 1 ) )
598+ }
599+ } ) ;
600+
601+ let input = Box :: pin ( RecordBatchStreamAdapter :: new (
602+ Arc :: clone ( & schema) ,
603+ input_stream,
604+ ) ) ;
605+ // Drop the returned stream immediately so no receiver is ever polled.
606+ let _buffered = spawn_buffered ( input, buffer_size) ;
607+
608+ // Give the producer task time to fill the channel and stall on send().
609+ tokio:: time:: sleep ( std:: time:: Duration :: from_millis ( 100 ) ) . await ;
610+
611+ assert_eq ! (
612+ produced_count. load( Ordering :: SeqCst ) ,
613+ buffer_size,
614+ "expected exactly {buffer_size} batch(es) in memory with no receiver polling"
615+ ) ;
616+ }
617+
618+ #[ tokio:: test( flavor = "multi_thread" ) ]
619+ async fn test_spawn_buffered_max_in_flight_batches ( ) {
620+ spawn_buffered_max_in_flight_batches ( 1 ) . await ;
621+ spawn_buffered_max_in_flight_batches ( 2 ) . await ;
622+ }
557623}
0 commit comments