Skip to content

Commit 329b267

Browse files
goutamadwantkosiew
authored andcommitted
fix: avoid buffering unbounded repartition output indefinitely (apache#24193)
## Which issue does this PR close? - Closes apache#24044. ## Rationale for this change `RepartitionExec` coalesces small batches on the producer side until the configured batch size is reached or all input senders finish. An unbounded input may never finish, so an available partial batch can be withheld indefinitely. This prevents an incremental query from emitting rows that are ready for downstream processing. ## What changes are included in this PR? - Skip producer-side batch coalescing when the input is unbounded. - Preserve the existing coalescing behavior for bounded inputs and preserve-order execution. - Add a regression test with an unbounded source that emits one partial batch and then remains open. ## Are these changes tested? Yes. - `cargo fmt --all -- --check` - `cargo test -p datafusion-physical-plan` - `cargo clippy --all-targets --all-features -- -D warnings` - The extended workspace test command required by `AGENTS.md`, including all 503 sqllogictest files The regression test fails on the previous implementation because the partial batch is never emitted, and passes with this change. The existing bounded-input coalescing test also continues to pass. ## Are there any user-facing changes? Yes. Repartitioning an unbounded input can now emit partial batches promptly instead of waiting indefinitely to fill the configured batch size. This may expose smaller batches to downstream operators. There are no public API changes. Signed-off-by: goutamadwant <[email protected]>
1 parent e8407a2 commit 329b267

1 file changed

Lines changed: 60 additions & 4 deletions

File tree

  • datafusion/physical-plan/src/repartition

‎datafusion/physical-plan/src/repartition/mod.rs‎

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -456,6 +456,7 @@ impl RepartitionExecState {
456456

457457
let num_input_partitions = streams_and_metrics.len();
458458
let num_output_partitions = partitioning.partition_count();
459+
let coalesce_batches = !preserve_order && !input.boundedness().is_unbounded();
459460

460461
let spill_manager = Arc::new(spill_manager);
461462

@@ -524,9 +525,10 @@ impl RepartitionExecState {
524525

525526
// Coalesce on the producer side, before the channel's gate, so
526527
// the consumer never sees the per-input-task small batches.
527-
// Skip in preserve-order mode: each input has its own dedicated
528-
// channel and `StreamingMergeBuilder` handles batching.
529-
let shared_coalescer = (!preserve_order).then(|| {
528+
// Skip in preserve-order mode, where `StreamingMergeBuilder`
529+
// handles batching, and for unbounded inputs, where a residual
530+
// batch could otherwise be withheld indefinitely.
531+
let shared_coalescer = coalesce_batches.then(|| {
530532
SharedCoalescer::new(
531533
input.schema(),
532534
context.session_config().batch_size(),
@@ -1127,7 +1129,8 @@ impl BatchPartitioner {
11271129
/// Repartitioning one [`RecordBatch`] implies creating multiple smaller batches, potentially
11281130
/// as many as the number of output partitions. [`RepartitionExec`] makes sure that the returned
11291131
/// batches adhere to the configured `datafusion.execution.batch_size` for efficient operations,
1130-
/// and for that, it will automatically coalesce batches right after repartitioning.
1132+
/// and for that, it will automatically coalesce batches right after repartitioning for bounded
1133+
/// inputs. Coalescing is skipped for unbounded inputs so partial batches are emitted promptly.
11311134
///
11321135
/// For this, one shared [`LimitedBatchCoalescer`] per output partition is used:
11331136
///
@@ -2239,6 +2242,7 @@ mod tests {
22392242
use super::*;
22402243
use crate::empty::EmptyExec;
22412244
use crate::projection::ProjectionExpr;
2245+
use crate::streaming::{PartitionStream, StreamingTableExec};
22422246
use crate::test::TestMemoryExec;
22432247
use crate::{
22442248
test::{
@@ -2263,6 +2267,27 @@ mod tests {
22632267
use datafusion_physical_expr::{PhysicalSortExpr, RangePartitioning, SplitPoint};
22642268
use insta::assert_snapshot;
22652269

2270+
#[derive(Debug)]
2271+
struct UnboundedTestPartition {
2272+
schema: SchemaRef,
2273+
batch: RecordBatch,
2274+
}
2275+
2276+
impl PartitionStream for UnboundedTestPartition {
2277+
fn schema(&self) -> &SchemaRef {
2278+
&self.schema
2279+
}
2280+
2281+
fn execute(&self, _ctx: Arc<TaskContext>) -> SendableRecordBatchStream {
2282+
let stream = futures::stream::iter([Ok(self.batch.clone())])
2283+
.chain(futures::stream::pending());
2284+
Box::pin(RecordBatchStreamAdapter::new(
2285+
Arc::clone(&self.schema),
2286+
stream,
2287+
))
2288+
}
2289+
}
2290+
22662291
#[test]
22672292
fn strength_reduced_u64_remainder_matches_modulo() {
22682293
let divisors = [
@@ -3007,6 +3032,37 @@ mod tests {
30073032
Ok(())
30083033
}
30093034

3035+
#[tokio::test]
3036+
async fn unbounded_input_emits_before_batch_size() -> Result<()> {
3037+
let schema = test_schema(false);
3038+
let batch = create_batch();
3039+
let source = Arc::new(StreamingTableExec::try_new(
3040+
Arc::clone(&schema),
3041+
vec![Arc::new(UnboundedTestPartition {
3042+
schema: Arc::clone(&schema),
3043+
batch: batch.clone(),
3044+
})],
3045+
None,
3046+
vec![],
3047+
true,
3048+
None,
3049+
)?);
3050+
let exec = RepartitionExec::try_new(source, Partitioning::RoundRobinBatch(1))?;
3051+
let session_config = SessionConfig::new().with_batch_size(batch.num_rows() * 2);
3052+
let task_ctx =
3053+
Arc::new(TaskContext::default().with_session_config(session_config));
3054+
3055+
let mut stream = exec.execute(0, task_ctx)?;
3056+
let output =
3057+
tokio::time::timeout(std::time::Duration::from_secs(5), stream.next())
3058+
.await
3059+
.expect("unbounded repartition withheld a partial batch")
3060+
.expect("unbounded input ended unexpectedly")?;
3061+
3062+
assert_eq!(batch, output);
3063+
Ok(())
3064+
}
3065+
30103066
fn test_schema(nullable: bool) -> Arc<Schema> {
30113067
Arc::new(Schema::new(vec![Field::new(
30123068
"c0",

0 commit comments

Comments
 (0)