Skip to content

Commit ac17f05

Browse files
Keep merge initialization diff focused
1 parent af64408 commit ac17f05

2 files changed

Lines changed: 18 additions & 7 deletions

File tree

‎datafusion/physical-plan/src/sorts/merge.rs‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,8 @@ impl<C: CursorValues> SortPreservingMergeStream<C> {
155155

156156
/// If the stream at the given index is not exhausted, and the last cursor for the
157157
/// stream is finished, poll the stream for the next RecordBatch and create a new
158-
/// cursor for the stream from the returned result
158+
/// cursor for the stream from the returned result.
159+
///
159160
/// Returns the number of rows read, or zero when no new batch was read.
160161
fn maybe_poll_stream(
161162
&mut self,
@@ -215,7 +216,16 @@ impl<C: CursorValues> SortPreservingMergeStream<C> {
215216
fn create_stream(mut self) -> impl Stream<Item = Result<RecordBatch>> {
216217
async_try_stream(|mut emitter| async move {
217218
// 1. Make sure we have data from each stream so we can initialize the loser tree
218-
self.initialize_partitions().await?;
219+
{
220+
// This vector contains the indices of the partitions that have not started emitting yet.
221+
let mut uninitiated_partitions =
222+
(0..self.streams.partitions()).collect::<Vec<_>>();
223+
224+
self.initialize_all_partitions(&mut uninitiated_partitions)
225+
.await?;
226+
227+
assert_eq!(uninitiated_partitions.len(), 0);
228+
}
219229

220230
let elapsed_compute = self.metrics.elapsed_compute().clone();
221231
let mut timer = elapsed_compute.timer();
@@ -290,22 +300,23 @@ impl<C: CursorValues> SortPreservingMergeStream<C> {
290300
self.cursors[winner].is_none()
291301
}
292302

293-
async fn initialize_partitions(&mut self) -> Result<()> {
303+
async fn initialize_all_partitions(
304+
&mut self,
305+
uninitiated_partitions: &mut Vec<usize>,
306+
) -> Result<()> {
294307
assert_eq!(
295308
self.loser_tree.len(),
296309
0,
297310
"loser tree must be empty when initializing"
298311
);
299-
let mut uninitiated_partitions =
300-
(0..self.streams.partitions()).collect::<Vec<_>>();
301312
let mut next_partition = 0;
302313
// Initializing many ready inputs can monopolize a worker. Use the batch
303314
// target as a simple work budget for both row and field cursors.
304315
let mut initialized_rows = 0usize;
305316
while !uninitiated_partitions.is_empty() {
306317
let rows = poll_fn(|cx| {
307318
self.initialize_next_partition(
308-
&mut uninitiated_partitions,
319+
uninitiated_partitions,
309320
&mut next_partition,
310321
cx,
311322
)

‎datafusion/physical-plan/src/sorts/merge_trace.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@ async fn test_distinct_keys_are_unaffected_by_the_tie_breaker() {
394394
///
395395
/// This is a known limitation, recorded here so that widening the tie breaker
396396
/// to inner nodes shows up as a deliberate snapshot change. Note also that the
397-
/// initial polls are not in partition order — initialization uses
397+
/// initial polls are not in partition order — `initialize_all_partitions` uses
398398
/// `swap_remove`, which reorders the pending list.
399399
#[tokio::test]
400400
async fn test_tie_breaker_only_balances_the_root_comparison() {

0 commit comments

Comments
 (0)