Skip to content

Commit 7b716d4

Browse files
adriangbclaude
andcommitted
perf: reuse GroupsAccumulatorAdapter scratch buffers across batches
`invoke_per_accumulator` allocated `groups_with_rows` and `offsets` again for every input batch. Keep them in a `Scratch` struct on the adapter and clear them at the start of each batch. The adapter takes the buffers out of `self` for the batch and puts them back on every return path, errors included, then charges the change in their capacity to `allocation_bytes`. It releases them when an emit leaves no group, so an adapter that has emitted everything holds no memory. `batch_indices` stays a new `Vec` for each batch: `into()` moves it into the `UInt32Array` that `take_arrays` reads, so reusing it would add a copy. Closes apache#25604 Co-Authored-By: Claude Opus 5 <[email protected]>
1 parent 73768ee commit 7b716d4

1 file changed

Lines changed: 78 additions & 3 deletions

File tree

‎datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs‎

Lines changed: 78 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,29 @@ pub struct GroupsAccumulatorAdapter {
111111

112112
/// Optional aggregate-owned metric timed once for a grouped update batch.
113113
grouped_update_metric: OnceLock<Option<Arc<dyn AggregateMetric>>>,
114+
115+
/// Buffers reused by every call to `invoke_per_accumulator`
116+
scratch: Scratch,
117+
}
118+
119+
/// Per-batch buffers of [`GroupsAccumulatorAdapter`], kept between batches so
120+
/// that each batch does not allocate them again. Their capacity is part of
121+
/// [`GroupsAccumulatorAdapter::allocation_bytes`].
122+
#[derive(Default)]
123+
struct Scratch {
124+
/// Group indexes that have rows in the batch, in order of first
125+
/// appearance in the batch
126+
groups_with_rows: Vec<usize>,
127+
128+
/// `offsets[i]` is the index into the reordered rows where the rows for
129+
/// `groups_with_rows[i]` start
130+
offsets: Vec<usize>,
131+
}
132+
133+
impl Scratch {
134+
fn allocated_size(&self) -> usize {
135+
self.groups_with_rows.allocated_size() + self.offsets.allocated_size()
136+
}
114137
}
115138

116139
/// Maximum number of prepared group inputs retained while timing an
@@ -154,6 +177,7 @@ impl GroupsAccumulatorAdapter {
154177
allocation_bytes: 0,
155178
metrics: None,
156179
grouped_update_metric: OnceLock::new(),
180+
scratch: Scratch::default(),
157181
}
158182
}
159183

@@ -235,10 +259,43 @@ impl GroupsAccumulatorAdapter {
235259

236260
assert_eq!(values[0].len(), group_indices.len());
237261

262+
// Take the scratch buffers out of `self` for the batch, and put them
263+
// back on every return path, including errors
264+
let mut scratch = std::mem::take(&mut self.scratch);
265+
let scratch_size_pre = scratch.allocated_size();
266+
let result = self.invoke_per_accumulator_with_scratch(
267+
&mut scratch,
268+
values,
269+
group_indices,
270+
opt_filter,
271+
f,
272+
);
273+
self.adjust_allocation(scratch_size_pre, scratch.allocated_size());
274+
self.scratch = scratch;
275+
result
276+
}
277+
278+
/// Body of [`Self::invoke_per_accumulator`], with its scratch buffers
279+
fn invoke_per_accumulator_with_scratch<F>(
280+
&mut self,
281+
scratch: &mut Scratch,
282+
values: &[ArrayRef],
283+
group_indices: &[usize],
284+
opt_filter: Option<&BooleanArray>,
285+
f: F,
286+
) -> Result<()>
287+
where
288+
F: Fn(&mut dyn Accumulator, &[ArrayRef]) -> Result<()>,
289+
{
290+
let Scratch {
291+
groups_with_rows,
292+
offsets,
293+
} = scratch;
294+
238295
// groups_with_rows holds a list of group indexes that have any rows
239296
// that need to be accumulated, stored in order of first appearance in
240297
// this batch
241-
let mut groups_with_rows = vec![];
298+
groups_with_rows.clear();
242299

243300
// figure out which input rows correspond to which groups.
244301
// Note that self.state.indices starts empty for all groups (it is
@@ -267,7 +324,7 @@ impl GroupsAccumulatorAdapter {
267324

268325
// offsets[i] is index into batch_indices where the rows for
269326
// groups_with_rows[i] start
270-
let mut offsets = Vec::with_capacity(groups_with_rows.len() + 1);
327+
offsets.clear();
271328
offsets.push(0);
272329

273330
let mut offset_so_far = 0;
@@ -398,6 +455,15 @@ impl GroupsAccumulatorAdapter {
398455
self.allocation_bytes = self.allocation_bytes.saturating_sub(size)
399456
}
400457

458+
/// Release the scratch buffers when no group is left, so that an adapter
459+
/// that has emitted every group holds no memory
460+
fn free_scratch_if_empty(&mut self) {
461+
if self.states.is_empty() {
462+
let scratch = std::mem::take(&mut self.scratch);
463+
self.free_allocation(scratch.allocated_size());
464+
}
465+
}
466+
401467
/// Release the allocation held by a state that is being emitted.
402468
fn free_state_allocation(&mut self, state: &AccumulatorState) {
403469
self.free_allocation(state.size());
@@ -456,6 +522,7 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
456522
let result = ScalarValue::iter_to_array(results);
457523

458524
self.adjust_allocation(vec_size_pre, self.states.allocated_size());
525+
self.free_scratch_if_empty();
459526

460527
result
461528
}
@@ -520,6 +587,7 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
520587
}
521588
}
522589
self.adjust_allocation(vec_size_pre, self.states.allocated_size());
590+
self.free_scratch_if_empty();
523591

524592
Ok(arrays)
525593
}
@@ -820,7 +888,12 @@ mod tests {
820888
let retained_indices = adapter.states[0].indices.allocated_size();
821889
assert!(retained_indices > 0);
822890
assert!(adapter.states[0].indices.is_empty());
823-
assert_eq!(adapter.size(), allocation_before_update + retained_indices);
891+
let retained_scratch = adapter.scratch.allocated_size();
892+
assert!(retained_scratch > 0);
893+
assert_eq!(
894+
adapter.size(),
895+
allocation_before_update + retained_indices + retained_scratch
896+
);
824897

825898
let allocation_after_first_update = adapter.size();
826899
adapter.update_batch(&[values], &[0, 0, 0, 0], None, 1)?;
@@ -1085,6 +1158,7 @@ mod tests {
10851158
.iter()
10861159
.map(AccumulatorState::size)
10871160
.sum::<usize>()
1161+
+ accumulator.scratch.allocated_size()
10881162
);
10891163
accumulator.update_batch(&[values], &[0, 1], None, 2)?;
10901164

@@ -1129,6 +1203,7 @@ mod tests {
11291203
.iter()
11301204
.map(AccumulatorState::size)
11311205
.sum::<usize>()
1206+
+ accumulator.scratch.allocated_size()
11321207
);
11331208
Ok(())
11341209
}

0 commit comments

Comments
 (0)