Skip to content

Commit 313a3a9

Browse files
Rachel IntRachel Int
authored andcommitted
perf: benchmark 32 aggregate buckets without compaction
1 parent b964422 commit 313a3a9

1 file changed

Lines changed: 11 additions & 2 deletions

File tree

‎datafusion/physical-plan/src/aggregates/final_buckets.rs‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,11 +39,14 @@ use crate::spill::spill_manager::SpillManager;
3939
use crate::stream::RecordBatchStreamAdapter;
4040

4141
/// Number of bits of the routing hash consumed by one bucketing level.
42-
const BUCKET_BITS: u32 = 6;
42+
const BUCKET_BITS: u32 = 5;
4343

4444
/// Number of buckets rows are split into at each level.
4545
const NUM_BUCKETS: usize = 1 << BUCKET_BITS;
4646

47+
/// Disable intermediate bucket aggregation for this benchmark POC.
48+
const ENABLE_BUCKET_COMPACTION: bool = false;
49+
4750
/// A bucket that is still too large is split again with the next hash bits,
4851
/// up to this many levels.
4952
pub(super) const MAX_BUCKET_LEVELS: u32 = 4;
@@ -55,7 +58,7 @@ pub(super) const MAX_BUCKET_LEVELS: u32 = 4;
5558
const BUCKET_HASH_SEED: RandomState = RandomState::with_seed(5364907223173859721);
5659

5760
/// A bucket is compacted for the first time once it holds this many batches
58-
/// worth of rows. With 64 buckets per partition this floor is what the
61+
/// worth of rows. With 32 buckets per partition this floor is what the
5962
/// buckets hold when every bucket has few groups, so it is kept small.
6063
const MIN_COMPACTION_BATCHES: usize = 1;
6164

@@ -242,6 +245,9 @@ impl ColumnBatchBuilder {
242245
///
243246
/// # Compaction
244247
///
248+
/// Disabled in this POC by [`ENABLE_BUCKET_COMPACTION`]. The policy below is
249+
/// retained for comparison with the original implementation.
250+
///
245251
/// The input may hold the same group many times (for example when the partial
246252
/// aggregation was skipped), so buffering it as is could take far more memory
247253
/// than the single table it replaces. A bucket that has grown enough is
@@ -411,6 +417,9 @@ impl FinalBuckets {
411417

412418
/// Returns a bucket that is due for compaction, if any.
413419
pub(super) fn bucket_to_compact(&self) -> Option<usize> {
420+
if !ENABLE_BUCKET_COMPACTION {
421+
return None;
422+
}
414423
self.buckets.iter().position(|bucket| {
415424
bucket.spill_file.is_none() && bucket.rows >= bucket.compact_at
416425
})

0 commit comments

Comments
 (0)