Skip to content

Commit 90335ae

Browse files
Rachel IntRachel Int
authored andcommitted
perf: flush partial batches with fixed group capacity
1 parent 634a59f commit 90335ae

17 files changed

Lines changed: 259 additions & 8 deletions

File tree

‎datafusion/common/src/config.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1152,6 +1152,14 @@ config_namespace! {
11521152
/// threshold. If both thresholds are enabled, reaching either triggers a flush.
11531153
pub partial_aggregation_flush_rows: usize, default = 0
11541154

1155+
/// (experimental) Flush unordered partial aggregation after a complete input
1156+
/// batch when the group count reaches batch_size. Overrides both other flush
1157+
/// thresholds and reserves supported group-key and accumulator storage for
1158+
/// twice batch_size groups, initially and after each threshold flush.
1159+
/// Requires a single grouping set and the same eligibility as the other
1160+
/// flush thresholds. String payload buffers use their normal growth policy.
1161+
pub partial_aggregation_flush_batch: bool, default = false
1162+
11551163
/// Should DataFusion use row number estimates at the input to decide
11561164
/// whether increasing parallelism is beneficial or not. By default,
11571165
/// only exact row numbers (not estimates) are used for this decision.

‎datafusion/expr-common/src/groups_accumulator.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,10 @@ pub trait GroupsAccumulator: Send + std::any::Any {
381381
///
382382
/// May be expensive; check the implementation before calling on hot paths.
383383
fn size(&self) -> usize;
384+
385+
/// Reserves row-addressed state for `capacity` groups where supported.
386+
/// This hint must not change the logical number of groups.
387+
fn reserve_groups(&mut self, _capacity: usize) {}
384388
}
385389

386390
#[cfg(test)]

‎datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/prim_op.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,11 @@ where
216216

217217
Ok(vec![Arc::new(state_values)])
218218
}
219+
fn reserve_groups(&mut self, capacity: usize) {
220+
self.values
221+
.reserve_exact(capacity.saturating_sub(self.values.len()));
222+
}
223+
219224
fn size(&self) -> usize {
220225
self.values.capacity() * size_of::<T::Native>() + self.null_state.size()
221226
}

‎datafusion/functions-aggregate/src/average.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1247,6 +1247,13 @@ where
12471247

12481248
Ok(vec![Arc::new(counts) as ArrayRef, Arc::new(sums)])
12491249
}
1250+
fn reserve_groups(&mut self, capacity: usize) {
1251+
self.counts
1252+
.reserve_exact(capacity.saturating_sub(self.counts.len()));
1253+
self.sums
1254+
.reserve_exact(capacity.saturating_sub(self.sums.len()));
1255+
}
1256+
12501257
fn size(&self) -> usize {
12511258
// Heap buffers
12521259
self.counts.capacity() * size_of::<u64>()

‎datafusion/functions-aggregate/src/count.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -823,6 +823,11 @@ impl GroupsAccumulator for CountGroupsAccumulator {
823823

824824
Ok(vec![state_array])
825825
}
826+
fn reserve_groups(&mut self, capacity: usize) {
827+
self.counts
828+
.reserve_exact(capacity.saturating_sub(self.counts.len()));
829+
}
830+
826831
fn size(&self) -> usize {
827832
self.counts.heap_size(&mut DFHeapSizeCtx::default())
828833
}

‎datafusion/physical-expr-common/src/binary_view_map.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,31 @@ where
197197
}
198198
}
199199

200+
/// Reserves group slots without preallocating variable-length string payloads.
201+
pub fn reserve_groups(&mut self, capacity: usize) {
202+
self.map
203+
.reserve(capacity.saturating_sub(self.map.len()), |entry| entry.hash);
204+
self.views
205+
.reserve_exact(capacity.saturating_sub(self.views.len()));
206+
self.hashes_buffer
207+
.reserve_exact(capacity.saturating_sub(self.hashes_buffer.len()));
208+
}
209+
210+
/// Emits keys, retains the hash table and scratch buffer, and reserves fixed
211+
/// view capacity. Payload buffers keep their normal block size and growth.
212+
pub fn take_with_capacity(&mut self, capacity: usize) -> ArrayRef {
213+
let mut outgoing = Self::new(self.output_type);
214+
std::mem::swap(self, &mut outgoing);
215+
std::mem::swap(&mut self.map, &mut outgoing.map);
216+
self.map.clear();
217+
std::mem::swap(&mut self.hashes_buffer, &mut outgoing.hashes_buffer);
218+
self.hashes_buffer.clear();
219+
self.initial_map_capacity = outgoing.initial_map_capacity;
220+
std::mem::swap(&mut self.random_state, &mut outgoing.random_state);
221+
self.reserve_groups(capacity);
222+
outgoing.into_state()
223+
}
224+
200225
/// Return the contents of this map and replace it with a new empty map with
201226
/// the same output type
202227
pub fn take(&mut self) -> Self {

‎datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,27 @@ impl<AggrMode> AggregateHashTable<AggrMode> {
372372
/// spilling without finalizing the same group more than once.
373373
pub(in crate::aggregates) fn take_state_batch(
374374
&mut self,
375+
) -> Result<Option<RecordBatch>> {
376+
self.take_state_batch_with_capacity(0)
377+
}
378+
379+
/// Reserves fixed row capacity without changing the number of groups.
380+
pub(in crate::aggregates) fn reserve_groups(&mut self, capacity: usize) {
381+
let state = self.state.building_mut();
382+
state.group_values.reserve_groups(capacity);
383+
state
384+
.batch_group_indices
385+
.reserve_exact(capacity.saturating_sub(state.batch_group_indices.len()));
386+
for acc in &mut state.accumulators {
387+
acc.accumulator.reserve_groups(capacity);
388+
}
389+
}
390+
391+
/// Emits all states and optionally reserves a fixed capacity for the next table.
392+
/// A zero capacity uses the normal memory-releasing path.
393+
pub(in crate::aggregates) fn take_state_batch_with_capacity(
394+
&mut self,
395+
capacity: usize,
375396
) -> Result<Option<RecordBatch>> {
376397
let state_schema = Arc::clone(&self.state_schema);
377398
let accumulator_metrics = Arc::clone(&self.aggregate_accumulator_metrics);
@@ -382,7 +403,11 @@ impl<AggrMode> AggregateHashTable<AggrMode> {
382403
}
383404

384405
let output = group_by_metrics.time_emitting(|| {
385-
let mut output = state.group_values.emit(EmitTo::All)?;
406+
let mut output = if capacity > 0 {
407+
state.group_values.emit_with_capacity(capacity)?
408+
} else {
409+
state.group_values.emit(EmitTo::All)?
410+
};
386411
for (idx, acc) in state.accumulators.iter_mut().enumerate() {
387412
output.extend(accumulator_metrics.time(
388413
idx,
@@ -399,9 +424,15 @@ impl<AggrMode> AggregateHashTable<AggrMode> {
399424
// `emit(EmitTo::All)` resets accumulator state. Explicitly shrink the
400425
// key/index buffers too so the memory reservation can be released
401426
// before the batch is sorted for spilling.
402-
state.group_values.clear_shrink(0);
403427
state.batch_group_indices.clear();
404-
state.batch_group_indices.shrink_to_fit();
428+
if capacity == 0 {
429+
state.group_values.clear_shrink(0);
430+
state.batch_group_indices.shrink_to_fit();
431+
} else {
432+
for acc in &mut state.accumulators {
433+
acc.accumulator.reserve_groups(capacity);
434+
}
435+
}
405436

406437
Ok(Some(batch))
407438
}

‎datafusion/physical-plan/src/aggregates/group_values/mod.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,18 @@ pub trait GroupValues: Send {
111111
/// Emits the group values
112112
fn emit(&mut self, emit_to: EmitTo) -> Result<Vec<ArrayRef>>;
113113

114+
/// Reserves room for at least `capacity` groups in supported storage layouts.
115+
/// Does not change the number of groups or the string payload growth policy.
116+
fn reserve_groups(&mut self, _capacity: usize) {}
117+
118+
/// Emits all keys and reserves a fixed capacity for the next table.
119+
fn emit_with_capacity(&mut self, capacity: usize) -> Result<Vec<ArrayRef>> {
120+
let output = self.emit(EmitTo::All)?;
121+
self.clear_shrink(capacity);
122+
self.reserve_groups(capacity);
123+
Ok(output)
124+
}
125+
114126
/// Materializes selected group values without changing the stored values or
115127
/// their group indices.
116128
///

‎datafusion/physical-plan/src/aggregates/group_values/multi_group_by/bytes_view.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -626,6 +626,14 @@ impl<B: ByteViewType> GroupColumn for ByteViewGroupValueBuilder<B> {
626626
self.vectorized_append_inner(array, rows)
627627
}
628628

629+
fn reserve_groups(&mut self, capacity: usize) {
630+
self.views
631+
.reserve_exact(capacity.saturating_sub(self.views.len()));
632+
if self.nulls.is_empty() {
633+
self.nulls = NullBufferBuilder::new(capacity);
634+
}
635+
}
636+
629637
fn len(&self) -> usize {
630638
self.views.len()
631639
}

‎datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,9 @@ pub trait GroupColumn: Send + Sync {
9595
/// The vectorized version `append_val`
9696
fn vectorized_append(&mut self, array: &ArrayRef, rows: &[usize]) -> Result<()>;
9797

98+
/// Reserves row-addressed storage without changing the logical length.
99+
fn reserve_groups(&mut self, _capacity: usize) {}
100+
98101
/// Returns the number of rows stored in this builder
99102
fn len(&self) -> usize;
100103

@@ -1083,6 +1086,44 @@ impl<const STREAMING: bool> GroupValues for GroupValuesColumn<STREAMING> {
10831086
}
10841087
}
10851088

1089+
fn emit_with_capacity(&mut self, capacity: usize) -> Result<Vec<ArrayRef>> {
1090+
let output = self.emit(EmitTo::All)?;
1091+
self.map.clear();
1092+
self.hashes_buffer.clear();
1093+
self.group_index_lists.clear();
1094+
self.emit_group_index_list_buffer.clear();
1095+
self.vectorized_operation_buffers.clear();
1096+
self.reserve_groups(capacity);
1097+
Ok(output)
1098+
}
1099+
1100+
fn reserve_groups(&mut self, capacity: usize) {
1101+
self.map
1102+
.reserve(capacity.saturating_sub(self.map.len()), |entry| entry.0);
1103+
self.map_size = self.map.capacity() * size_of::<(u64, usize)>();
1104+
self.hashes_buffer
1105+
.reserve_exact(capacity.saturating_sub(self.hashes_buffer.len()));
1106+
for column in &mut self.group_values {
1107+
column.reserve_groups(capacity);
1108+
}
1109+
let buffers = &mut self.vectorized_operation_buffers;
1110+
buffers
1111+
.append_row_indices
1112+
.reserve_exact(capacity.saturating_sub(buffers.append_row_indices.len()));
1113+
buffers
1114+
.equal_to_row_indices
1115+
.reserve_exact(capacity.saturating_sub(buffers.equal_to_row_indices.len()));
1116+
buffers
1117+
.equal_to_group_indices
1118+
.reserve_exact(capacity.saturating_sub(buffers.equal_to_group_indices.len()));
1119+
buffers
1120+
.remaining_row_indices
1121+
.reserve_exact(capacity.saturating_sub(buffers.remaining_row_indices.len()));
1122+
buffers
1123+
.equal_to_results
1124+
.reserve(capacity.saturating_sub(buffers.equal_to_results.len()));
1125+
}
1126+
10861127
fn size(&self) -> usize {
10871128
let group_values_size = self.group_values.allocated_size()
10881129
+ self

0 commit comments

Comments
 (0)