Skip to content

Commit e3fe1da

Browse files
Dandandanclaude
andcommitted
Switch to blocks at exactly one block, so nothing is copied or split
The threshold was eight blocks, so the state was chopped into eight pieces when it switched, copying all of it. Setting the threshold to one block means the flat allocation *is* block zero and is handed over as it stands. A block is now 2MB for an eight byte state, a whole number of output batches, and the flat state starts at one batch rather than growing into one from nothing. `take_first` can still leave more than a block behind, so the splitting path stays for that case, covered by the existing test. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_0117yCevpWYN82FH6PLrM8qu
1 parent ab42ba5 commit e3fe1da

1 file changed

Lines changed: 35 additions & 11 deletions

File tree

  • datafusion/functions-aggregate-common/src/aggregate/groups_accumulator

‎datafusion/functions-aggregate-common/src/aggregate/groups_accumulator/blocked_vec.rs‎

Lines changed: 35 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,25 @@ use std::mem::size_of;
2222

2323
use datafusion_expr_common::groups_accumulator::EmitTo;
2424

25-
/// Groups per block once blocked. Sixteen batches' worth, and a megabyte for an
26-
/// eight byte state, so a block can be handed to Arrow as a buffer of its own.
27-
pub const BLOCK_LEN: usize = 1 << 17;
25+
/// Groups per block once blocked: 2 MB for an eight byte state, and a whole number of
26+
/// output batches, so a block can be handed to Arrow as a buffer of its own.
27+
///
28+
/// Counted in groups rather than bytes so that two states of different element types
29+
/// (`avg`'s sums and counts, say) always agree on where a block starts.
30+
pub const BLOCK_LEN: usize = 1 << 18;
2831

2932
/// Groups a state may reach before it is worth holding in blocks.
3033
///
3134
/// Below this the state is one plain `Vec`: growing it copies at most this many elements
3235
/// in total, which is not worth paying a second load on every group update to avoid.
33-
pub const THRESHOLD_LEN: usize = 1 << 20;
36+
///
37+
/// Equal to [`BLOCK_LEN`] so that the flat state is exactly one block when it switches,
38+
/// and becomes block zero without being copied or split.
39+
pub const THRESHOLD_LEN: usize = BLOCK_LEN;
40+
41+
/// The flat state starts at a whole output batch rather than growing into one from
42+
/// nothing, which is a handful of reallocations every grouping would otherwise pay.
43+
const INITIAL_LEN: usize = 8192;
3444

3545
const BLOCK_SHIFT: u32 = BLOCK_LEN.trailing_zeros();
3646
const BLOCK_MASK: usize = BLOCK_LEN - 1;
@@ -135,7 +145,12 @@ impl<T: Clone> BlockedVec<T> {
135145
self.switch_to_blocked();
136146
}
137147
match &mut self.storage {
138-
Storage::Flat(values) => values.resize(total_num_groups, value),
148+
Storage::Flat(values) => {
149+
if values.capacity() == 0 {
150+
values.reserve(std::cmp::min(INITIAL_LEN, THRESHOLD_LEN));
151+
}
152+
values.resize(total_num_groups, value)
153+
}
139154
Storage::Blocked(blocks) => {
140155
let mut len = self.len;
141156
while len < total_num_groups {
@@ -152,22 +167,31 @@ impl<T: Clone> BlockedVec<T> {
152167
self.len = total_num_groups;
153168
}
154169

155-
/// Chops the single allocation into blocks. Paid once, at the threshold.
170+
/// Hands the single allocation over as block zero. Paid once, at the threshold.
171+
///
172+
/// The threshold is one block, so what is already stored fits a block exactly and
173+
/// nothing has to be copied or split; it is only given room to fill that block out.
156174
fn switch_to_blocked(&mut self) {
157175
let Storage::Flat(values) = &mut self.storage else {
158176
return;
159177
};
160178
let mut rest = std::mem::take(values);
161-
let mut blocks: Vec<Vec<T>> = Vec::with_capacity(rest.len() / BLOCK_LEN + 1);
179+
if rest.len() <= BLOCK_LEN {
180+
// The usual case: growth switches at exactly one block, so the allocation
181+
// is handed over as it stands and only given room to fill the block out.
182+
rest.reserve_exact(BLOCK_LEN - rest.len());
183+
self.storage = Storage::Blocked(vec![rest]);
184+
return;
185+
}
186+
// `take_first` can leave more than a block behind, which has to be split.
187+
let mut blocks: Vec<Vec<T>> = Vec::with_capacity(rest.len().div_ceil(BLOCK_LEN));
162188
while rest.len() > BLOCK_LEN {
163189
let tail = rest.split_off(BLOCK_LEN);
164190
blocks.push(rest);
165191
rest = tail;
166192
}
167-
// The last piece keeps room to fill out its block.
168-
let mut last = Vec::with_capacity(BLOCK_LEN);
169-
last.extend(rest);
170-
blocks.push(last);
193+
rest.reserve_exact(BLOCK_LEN - rest.len());
194+
blocks.push(rest);
171195
self.storage = Storage::Blocked(blocks);
172196
}
173197

0 commit comments

Comments
 (0)