Skip to content

Commit df55545

Browse files
committed
perf: increase final aggregation fanout to 256 buckets
1 parent 33aad7a commit df55545

2 files changed

Lines changed: 8 additions & 5 deletions

File tree

‎datafusion/physical-plan/src/aggregates/final_buckets.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ use crate::spill::spill_manager::SpillManager;
4141
use crate::stream::RecordBatchStreamAdapter;
4242

4343
/// Number of bits of the routing hash consumed by one bucketing level.
44-
const BUCKET_BITS: u32 = 7;
44+
const BUCKET_BITS: u32 = 8;
4545

4646
/// Number of buckets rows are split into at each level.
4747
const NUM_BUCKETS: usize = 1 << BUCKET_BITS;
@@ -60,7 +60,7 @@ pub(super) const MAX_BUCKET_LEVELS: u32 = 4;
6060
const BUCKET_HASH_SEED: RandomState = RandomState::with_seed(5364907223173859721);
6161

6262
/// A bucket is compacted for the first time once it holds this many batches
63-
/// worth of rows. With 128 buckets per partition this floor is what the
63+
/// worth of rows. With 256 buckets per partition this floor is what the
6464
/// buckets hold when every bucket has few groups, so it is kept small.
6565
const MIN_COMPACTION_BATCHES: usize = 1;
6666

@@ -816,7 +816,9 @@ mod tests {
816816
]));
817817
// Rows of one level-0 bucket must spread again at level 1
818818
let mut level0 = FinalBuckets::new(&schema, 1, 1024, 0, None);
819-
level0.route(&test_batch(&schema, 0..50_000))?;
819+
// Keep enough rows per second-level bucket as fanout grows.
820+
let rows = (NUM_BUCKETS * NUM_BUCKETS * 8) as i64;
821+
level0.route(&test_batch(&schema, 0..rows))?;
820822
let source = level0.into_sources()?.swap_remove(0);
821823
let BucketSource::Memory { batches, .. } = source else {
822824
unreachable!()

‎datafusion/physical-plan/src/aggregates/hash_stream.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1737,7 +1737,8 @@ mod tests {
17371737
}
17381738

17391739
#[tokio::test]
1740-
async fn final_hash_aggregate_compacts_repeated_groups() -> Result<()> {
1740+
async fn final_hash_aggregate_buffers_repeated_groups_without_compaction()
1741+
-> Result<()> {
17411742
// Every group arrives 40 times, so buffering the input as is would
17421743
// hold 40 rows per group.
17431744
let (rows, splits, spills) =
@@ -1747,7 +1748,7 @@ mod tests {
17471748
assert!(splits >= 1);
17481749
assert_eq!(spills, 0);
17491750
let compactions = BUCKET_COMPACTIONS.with(|compactions| compactions.get());
1750-
assert!(compactions > 0, "buckets were compacted");
1751+
assert_eq!(compactions, 0, "intermediate compaction is disabled");
17511752
Ok(())
17521753
}
17531754

0 commit comments

Comments
 (0)