@@ -111,6 +111,29 @@ pub struct GroupsAccumulatorAdapter {
111111
112112 /// Optional aggregate-owned metric timed once for a grouped update batch.
113113 grouped_update_metric : OnceLock < Option < Arc < dyn AggregateMetric > > > ,
114+
115+ /// Buffers reused by every call to `invoke_per_accumulator`
116+ scratch : Scratch ,
117+ }
118+
119+ /// Per-batch buffers of [`GroupsAccumulatorAdapter`], kept between batches so
120+ /// that each batch does not allocate them again. Their capacity is part of
121+ /// [`GroupsAccumulatorAdapter::allocation_bytes`].
122+ #[ derive( Default ) ]
123+ struct Scratch {
124+ /// Group indexes that have rows in the batch, in order of first
125+ /// appearance in the batch
126+ groups_with_rows : Vec < usize > ,
127+
128+ /// `offsets[i]` is the index into the reordered rows where the rows for
129+ /// `groups_with_rows[i]` start
130+ offsets : Vec < usize > ,
131+ }
132+
133+ impl Scratch {
134+ fn allocated_size ( & self ) -> usize {
135+ self . groups_with_rows . allocated_size ( ) + self . offsets . allocated_size ( )
136+ }
114137}
115138
116139/// Maximum number of prepared group inputs retained while timing an
@@ -154,6 +177,7 @@ impl GroupsAccumulatorAdapter {
154177 allocation_bytes : 0 ,
155178 metrics : None ,
156179 grouped_update_metric : OnceLock :: new ( ) ,
180+ scratch : Scratch :: default ( ) ,
157181 }
158182 }
159183
@@ -235,10 +259,43 @@ impl GroupsAccumulatorAdapter {
235259
236260 assert_eq ! ( values[ 0 ] . len( ) , group_indices. len( ) ) ;
237261
262+ // Take the scratch buffers out of `self` for the batch, and put them
263+ // back on every return path, including errors
264+ let mut scratch = std:: mem:: take ( & mut self . scratch ) ;
265+ let scratch_size_pre = scratch. allocated_size ( ) ;
266+ let result = self . invoke_per_accumulator_with_scratch (
267+ & mut scratch,
268+ values,
269+ group_indices,
270+ opt_filter,
271+ f,
272+ ) ;
273+ self . adjust_allocation ( scratch_size_pre, scratch. allocated_size ( ) ) ;
274+ self . scratch = scratch;
275+ result
276+ }
277+
278+ /// Body of [`Self::invoke_per_accumulator`], with its scratch buffers
279+ fn invoke_per_accumulator_with_scratch < F > (
280+ & mut self ,
281+ scratch : & mut Scratch ,
282+ values : & [ ArrayRef ] ,
283+ group_indices : & [ usize ] ,
284+ opt_filter : Option < & BooleanArray > ,
285+ f : F ,
286+ ) -> Result < ( ) >
287+ where
288+ F : Fn ( & mut dyn Accumulator , & [ ArrayRef ] ) -> Result < ( ) > ,
289+ {
290+ let Scratch {
291+ groups_with_rows,
292+ offsets,
293+ } = scratch;
294+
238295 // groups_with_rows holds a list of group indexes that have any rows
239296 // that need to be accumulated, stored in order of first appearance in
240297 // this batch
241- let mut groups_with_rows = vec ! [ ] ;
298+ groups_with_rows. clear ( ) ;
242299
243300 // figure out which input rows correspond to which groups.
244301 // Note that self.state.indices starts empty for all groups (it is
@@ -267,7 +324,7 @@ impl GroupsAccumulatorAdapter {
267324
268325 // offsets[i] is index into batch_indices where the rows for
269326 // groups_with_rows[i] start
270- let mut offsets = Vec :: with_capacity ( groups_with_rows . len ( ) + 1 ) ;
327+ offsets. clear ( ) ;
271328 offsets. push ( 0 ) ;
272329
273330 let mut offset_so_far = 0 ;
@@ -398,6 +455,15 @@ impl GroupsAccumulatorAdapter {
398455 self . allocation_bytes = self . allocation_bytes . saturating_sub ( size)
399456 }
400457
458+ /// Release the scratch buffers when no group is left, so that an adapter
459+ /// that has emitted every group holds no memory
460+ fn free_scratch_if_empty ( & mut self ) {
461+ if self . states . is_empty ( ) {
462+ let scratch = std:: mem:: take ( & mut self . scratch ) ;
463+ self . free_allocation ( scratch. allocated_size ( ) ) ;
464+ }
465+ }
466+
401467 /// Release the allocation held by a state that is being emitted.
402468 fn free_state_allocation ( & mut self , state : & AccumulatorState ) {
403469 self . free_allocation ( state. size ( ) ) ;
@@ -456,6 +522,7 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
456522 let result = ScalarValue :: iter_to_array ( results) ;
457523
458524 self . adjust_allocation ( vec_size_pre, self . states . allocated_size ( ) ) ;
525+ self . free_scratch_if_empty ( ) ;
459526
460527 result
461528 }
@@ -520,6 +587,7 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
520587 }
521588 }
522589 self . adjust_allocation ( vec_size_pre, self . states . allocated_size ( ) ) ;
590+ self . free_scratch_if_empty ( ) ;
523591
524592 Ok ( arrays)
525593 }
@@ -820,7 +888,12 @@ mod tests {
820888 let retained_indices = adapter. states [ 0 ] . indices . allocated_size ( ) ;
821889 assert ! ( retained_indices > 0 ) ;
822890 assert ! ( adapter. states[ 0 ] . indices. is_empty( ) ) ;
823- assert_eq ! ( adapter. size( ) , allocation_before_update + retained_indices) ;
891+ let retained_scratch = adapter. scratch . allocated_size ( ) ;
892+ assert ! ( retained_scratch > 0 ) ;
893+ assert_eq ! (
894+ adapter. size( ) ,
895+ allocation_before_update + retained_indices + retained_scratch
896+ ) ;
824897
825898 let allocation_after_first_update = adapter. size ( ) ;
826899 adapter. update_batch ( & [ values] , & [ 0 , 0 , 0 , 0 ] , None , 1 ) ?;
@@ -1085,6 +1158,7 @@ mod tests {
10851158 . iter( )
10861159 . map( AccumulatorState :: size)
10871160 . sum:: <usize >( )
1161+ + accumulator. scratch. allocated_size( )
10881162 ) ;
10891163 accumulator. update_batch ( & [ values] , & [ 0 , 1 ] , None , 2 ) ?;
10901164
@@ -1129,6 +1203,7 @@ mod tests {
11291203 . iter( )
11301204 . map( AccumulatorState :: size)
11311205 . sum:: <usize >( )
1206+ + accumulator. scratch. allocated_size( )
11321207 ) ;
11331208 Ok ( ( ) )
11341209 }
0 commit comments