Skip to content

Commit e580f21

Browse files
committed
drop batch coalescer and address review comments
1 parent ed9a2b5 commit e580f21

1 file changed

Lines changed: 10 additions & 14 deletions

File tree

  • datafusion/physical-plan/src/topk

‎datafusion/physical-plan/src/topk/mod.rs‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1959,8 +1959,10 @@ struct DenseRankPartitionState {
19591959
///
19601960
/// INVARIANT: equals `recompute_contents_bytes` (test-only, so not
19611961
/// linkable from rustdoc). Every mutation of `groups` or `keys` must
1962-
/// adjust it; the `dense_rank_contents_bytes_tracks_recompute` test
1963-
/// checks this against a full recompute after a randomized workload.
1962+
/// adjust it; the
1963+
/// `test_partitioned_topk_dense_rank_contents_bytes_tracks_recompute`
1964+
/// test checks this against a full recompute after a randomized
1965+
/// workload.
19641966
contents_bytes: usize,
19651967
}
19661968

@@ -2359,7 +2361,7 @@ impl PartitionedTopKDenseRank {
23592361
let mut sorted_pks: Vec<Vec<u8>> = states.keys().cloned().collect();
23602362
sorted_pks.sort();
23612363

2362-
let mut coalescer = BatchCoalescer::new(Arc::clone(&schema), batch_size);
2364+
let mut out: Vec<Result<RecordBatch>> = Vec::new();
23632365

23642366
// Gather every retained row with a single `interleave_record_batch`
23652367
// per output batch rather than one `take_record_batch` per
@@ -2383,7 +2385,7 @@ impl PartitionedTopKDenseRank {
23832385
}
23842386

23852387
// Chunk at `batch_size` so the operator emits the same batch sizes
2386-
// as before and never materializes all retained rows at once.
2388+
// as before and no single `interleave` output exceeds `batch_size`.
23872389
let mut indices: Vec<(usize, usize)> = Vec::with_capacity(batch_size);
23882390
for pk in sorted_pks {
23892391
let DenseRankPartitionState {
@@ -2402,23 +2404,17 @@ impl PartitionedTopKDenseRank {
24022404
for row in entry.row_indices {
24032405
indices.push((array_pos, row as usize));
24042406
if indices.len() == batch_size {
2405-
coalescer.push_batch(interleave_record_batch(
2406-
&batch_refs,
2407-
&indices,
2408-
)?)?;
2407+
let b = interleave_record_batch(&batch_refs, &indices)?;
2408+
(&b).record_output(&metrics.baseline);
2409+
out.push(Ok(b));
24092410
indices.clear();
24102411
}
24112412
}
24122413
}
24132414
}
24142415
}
24152416
if !indices.is_empty() {
2416-
coalescer.push_batch(interleave_record_batch(&batch_refs, &indices)?)?;
2417-
}
2418-
coalescer.finish_buffered_batch()?;
2419-
2420-
let mut out: Vec<Result<RecordBatch>> = Vec::new();
2421-
while let Some(b) = coalescer.next_completed_batch() {
2417+
let b = interleave_record_batch(&batch_refs, &indices)?;
24222418
(&b).record_output(&metrics.baseline);
24232419
out.push(Ok(b));
24242420
}

0 commit comments

Comments
 (0)