Skip to content

Commit c1a35a7

Browse files
refactor: simplify sort run lookahead with buffered
1 parent 42309e4 commit c1a35a7

1 file changed

Lines changed: 3 additions & 11 deletions

File tree

  • datafusion/physical-plan/src/sorts

‎datafusion/physical-plan/src/sorts/sort.rs‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -850,17 +850,9 @@ impl ExternalSorter {
850850
/// Each run can hold one prefetched batch whose reservation is already released.
851851
fn with_lookahead(stream: SendableRecordBatchStream) -> SendableRecordBatchStream {
852852
let schema = stream.schema();
853-
let batches = futures::stream::unfold(
854-
(None, stream),
855-
|(lookahead, mut stream)| async move {
856-
let current = match lookahead {
857-
Some(batch) => batch,
858-
None => stream.next().await?,
859-
};
860-
let next = stream.next().await;
861-
Some((current, (next, stream)))
862-
},
863-
);
853+
// Buffer the current batch and one ahead to return borrowed spill workspace
854+
// before the merge processes the current batch.
855+
let batches = stream.map(std::future::ready).buffered(2);
864856
Box::pin(RecordBatchStreamAdapter::new(schema, batches))
865857
}
866858

0 commit comments

Comments
 (0)