Skip to content

Commit 9202698

Browse files
fix: ensure a maximum of buffer_len RecordBatches are cached in spawn_buffered (apache#23560)
## Which issue does this PR close? Fixes: apache#23559 ## Rationale for this change See the issue description ## What changes are included in this PR? Change spawn_buffered's behavior to ensure a maximum of `buffer` RecordBatches are produced before any consumer polls the stream. ## Are these changes tested? Yes ## Are there any user-facing changes? No <!-- If there are any breaking changes to public APIs, please add the `api change` label. -->
1 parent f5a20d7 commit 9202698

1 file changed

Lines changed: 73 additions & 7 deletions

File tree

‎datafusion/physical-plan/src/common.rs‎

Lines changed: 73 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,8 @@ pub fn project_plan_to_schema(
181181
}
182182

183183
/// If running in a tokio context spawns the execution of `stream` to a separate task
184-
/// allowing it to execute in parallel with an intermediate buffer of size `buffer`
184+
/// allowing it to execute in parallel with an intermediate buffer of size `buffer`.
185+
/// At most `buffer` record batches will be produced ahead of the consumer.
185186
pub fn spawn_buffered(
186187
mut input: SendableRecordBatchStream,
187188
buffer: usize,
@@ -196,11 +197,22 @@ pub fn spawn_buffered(
196197
let sender = builder.tx();
197198

198199
builder.spawn(async move {
199-
while let Some(item) = input.next().await {
200-
if sender.send(item).await.is_err() {
201-
// Receiver dropped when query is shutdown early (e.g., limit) or error,
202-
// no need to return propagate the send error.
203-
return Ok(());
200+
// We call `reserve` (which waits until there's room for at least 1 message in the
201+
// channel buffer) **before** polling from input to ensure we hold a maximum of
202+
// `buffer` record batches in memory.
203+
// Polling from input and then calling send() would block when the channel is full
204+
// so it would essentially hold `buffer` + 1 record batches:
205+
// * `buffer`: this many elements would live inside the channel, since this is the
206+
// channel's capacity
207+
// * 1 extra RecordBatch which was produced, but there was no room for it in the
208+
// channel, so it's being owned by the send() future, which keeps the batch in
209+
// memory while it waits for a slot to free up
210+
while let Ok(permit) = sender.reserve().await {
211+
// Receiver dropped when query is shutdown early (e.g., limit) or error,
212+
// no need to return propagate the send error.
213+
match input.next().await {
214+
Some(item) => permit.send(item),
215+
None => break,
204216
}
205217
}
206218

@@ -298,10 +310,13 @@ mod tests {
298310
use crate::empty::EmptyExec;
299311
use crate::projection::ProjectionExec;
300312

313+
use crate::stream::RecordBatchStreamAdapter;
314+
use futures::stream;
301315
use std::collections::HashMap;
316+
use std::sync::atomic::{AtomicUsize, Ordering};
302317

303318
use arrow::{
304-
array::{Float32Array, Float64Array, UInt64Array},
319+
array::{Float32Array, Float64Array, Int32Array, UInt64Array},
305320
datatypes::{DataType, Field, Schema},
306321
};
307322

@@ -554,4 +569,55 @@ mod tests {
554569
let err = project_plan_to_schema(input, &expected_schema).unwrap_err();
555570
assert!(err.to_string().contains("schema metadata differ"));
556571
}
572+
573+
/// Verifies that `spawn_buffered` holds exactly `buffer` record batches in memory
574+
/// when no receiver is polling
575+
async fn spawn_buffered_max_in_flight_batches(buffer_size: usize) {
576+
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
577+
let num_batches = 10;
578+
579+
let produced_count = Arc::new(AtomicUsize::new(0));
580+
let produced_clone = Arc::clone(&produced_count);
581+
let schema_clone = Arc::clone(&schema);
582+
583+
// Stream increments the counter each time a batch is pulled by the producer.
584+
let input_stream = stream::unfold(0usize, move |i| {
585+
let schema = Arc::clone(&schema_clone);
586+
let counter = Arc::clone(&produced_clone);
587+
async move {
588+
if i >= num_batches {
589+
return None;
590+
}
591+
let batch = RecordBatch::try_new(
592+
Arc::clone(&schema),
593+
vec![Arc::new(Int32Array::from(vec![i as i32]))],
594+
)
595+
.unwrap();
596+
counter.fetch_add(1, Ordering::SeqCst);
597+
Some((Ok(batch), i + 1))
598+
}
599+
});
600+
601+
let input = Box::pin(RecordBatchStreamAdapter::new(
602+
Arc::clone(&schema),
603+
input_stream,
604+
));
605+
// Drop the returned stream immediately so no receiver is ever polled.
606+
let _buffered = spawn_buffered(input, buffer_size);
607+
608+
// Give the producer task time to fill the channel and stall on send().
609+
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
610+
611+
assert_eq!(
612+
produced_count.load(Ordering::SeqCst),
613+
buffer_size,
614+
"expected exactly {buffer_size} batch(es) in memory with no receiver polling"
615+
);
616+
}
617+
618+
#[tokio::test(flavor = "multi_thread")]
619+
async fn test_spawn_buffered_max_in_flight_batches() {
620+
spawn_buffered_max_in_flight_batches(1).await;
621+
spawn_buffered_max_in_flight_batches(2).await;
622+
}
557623
}

0 commit comments

Comments
 (0)