Skip to content

Commit 86a9abb

Browse files
viiryaIsaac
andcommitted
fix: release per-batch accumulator scratch after a clustered partial emit
Add GroupsAccumulator::shrink_to_fit, a no-op by default, and call it for every accumulator from ClusteredAggregateTable::shrink_to_fit. HllGroupsAccumulator drops its per-batch hashes, and GroupsAccumulatorAdapter drops its per-batch scratch, which it otherwise keeps while the active group remains. Closes apache#26781. Co-authored-by: Isaac <[email protected]>
1 parent 1bfc59e commit 86a9abb

6 files changed

Lines changed: 117 additions & 12 deletions

File tree

‎datafusion/expr-common/src/groups_accumulator.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,14 @@ pub trait GroupsAccumulator: Send + std::any::Any {
368368
///
369369
/// May be expensive; check the implementation before calling on hot paths.
370370
fn size(&self) -> usize;
371+
372+
/// Keeps the stored groups but releases capacity they do not need, such as
373+
/// per-batch scratch buffers, so that [`Self::size`] reflects the stored
374+
/// groups. For example, after [`EmitTo::First`] removes most groups, an
375+
/// accumulator may keep a buffer it sized for the last input batch.
376+
///
377+
/// The default does nothing.
378+
fn shrink_to_fit(&mut self) {}
371379
}
372380

373381
#[cfg(test)]

‎datafusion/functions-aggregate-common/src/aggregate/groups_accumulator.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -653,6 +653,14 @@ impl GroupsAccumulator for GroupsAccumulatorAdapter {
653653
self.allocation_bytes
654654
}
655655

656+
fn shrink_to_fit(&mut self) {
657+
let vec_size_pre = self.states.allocated_size();
658+
self.states.shrink_to_fit();
659+
self.adjust_allocation(vec_size_pre, self.states.allocated_size());
660+
let scratch = std::mem::take(&mut self.scratch);
661+
self.free_allocation(scratch.allocated_size());
662+
}
663+
656664
fn convert_to_state(
657665
&self,
658666
values: &[ArrayRef],

‎datafusion/functions-aggregate/src/approx_distinct.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,6 +629,11 @@ impl GroupsAccumulator for HllGroupsAccumulator {
629629
+ self.allocated_bytes
630630
+ self.hashes.capacity() * size_of::<u64>()
631631
}
632+
633+
fn shrink_to_fit(&mut self) {
634+
self.groups.shrink_to_fit();
635+
self.hashes = Vec::new();
636+
}
632637
}
633638

634639
impl Debug for ApproxDistinct {

‎datafusion/physical-plan/src/aggregates/aggregate_hash_table/common.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -816,6 +816,11 @@ impl HashAggregateAccumulator {
816816
self.accumulator.size()
817817
}
818818

819+
/// See [`GroupsAccumulator::shrink_to_fit`].
820+
pub(super) fn shrink_to_fit(&mut self) {
821+
self.accumulator.shrink_to_fit();
822+
}
823+
819824
pub(super) fn update_batch(
820825
&mut self,
821826
values: &CompactedAccumulatorArgs,

‎datafusion/physical-plan/src/aggregates/aggregate_hash_table/common_clustered.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -287,14 +287,16 @@ impl<AggrMode> ClusteredAggregateTable<AggrMode> {
287287
}
288288

289289
/// Releases the capacity that the buffer holds beyond its stored groups,
290-
/// such as hash table slots and the per-batch `group_indices` scratch, so
291-
/// that [`Self::memory_size`] reflects the remaining groups.
292-
///
293-
/// Accumulator capacity is not released.
290+
/// such as hash table slots, the per-batch `group_indices` scratch and
291+
/// per-batch accumulator scratch, so that [`Self::memory_size`] reflects
292+
/// the remaining groups.
294293
pub(in crate::aggregates) fn shrink_to_fit(&mut self) {
295294
self.buffer.group_values.shrink_to_fit();
296295
self.buffer.group_indices.clear();
297296
self.buffer.group_indices.shrink_to_fit();
297+
for acc in &mut self.buffer.accumulators {
298+
acc.shrink_to_fit();
299+
}
298300
}
299301

300302
pub(in crate::aggregates) fn metrics(&self) -> ClusteredAggregateTableMetrics {

‎datafusion/physical-plan/src/aggregates/clustered_partial_stream.rs‎

Lines changed: 85 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -434,9 +434,14 @@ mod tests {
434434
GreedyMemoryPool, MemoryPool, UnboundedMemoryPool,
435435
};
436436
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
437+
use datafusion_expr::AggregateUDF;
438+
use datafusion_functions_aggregate::any_value::any_value_udaf;
439+
use datafusion_functions_aggregate::approx_distinct::approx_distinct_udaf;
437440
use datafusion_functions_aggregate::sum::sum_udaf;
438441
use datafusion_physical_expr::PhysicalSortExpr;
439-
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
442+
use datafusion_physical_expr::aggregate::{
443+
AggregateExprBuilder, AggregateFunctionExpr,
444+
};
440445
use datafusion_physical_expr::expressions::col;
441446
use datafusion_physical_expr_common::sort_expr::LexOrdering;
442447

@@ -484,6 +489,21 @@ mod tests {
484489
async fn run(
485490
batches: Vec<RecordBatch>,
486491
pool: Arc<dyn MemoryPool>,
492+
) -> Result<Vec<RecordBatch>> {
493+
let schema = batches[0].schema();
494+
let sum = AggregateExprBuilder::new(sum_udaf(), vec![col("v", &schema)?])
495+
.schema(Arc::clone(&schema))
496+
.alias("sum")
497+
.build()?;
498+
run_aggregates(batches, vec![Arc::new(sum)], pool).await
499+
}
500+
501+
/// Runs a partial aggregate of `aggregates` grouped by `k0, ..` over
502+
/// `batches`, which are sorted on every key.
503+
async fn run_aggregates(
504+
batches: Vec<RecordBatch>,
505+
aggregates: Vec<Arc<AggregateFunctionExpr>>,
506+
pool: Arc<dyn MemoryPool>,
487507
) -> Result<Vec<RecordBatch>> {
488508
let schema = batches[0].schema();
489509
let keys = (0..schema.fields().len() - 1)
@@ -499,13 +519,8 @@ mod tests {
499519
let aggregate = AggregateExec::try_new(
500520
AggregateMode::Partial,
501521
PhysicalGroupBy::new_single(keys),
502-
vec![Arc::new(
503-
AggregateExprBuilder::new(sum_udaf(), vec![col("v", &schema)?])
504-
.schema(Arc::clone(&schema))
505-
.alias("sum")
506-
.build()?,
507-
)],
508-
vec![None],
522+
aggregates.clone(),
523+
vec![None; aggregates.len()],
509524
Arc::new(input),
510525
Arc::clone(&schema),
511526
)?;
@@ -559,4 +574,66 @@ mod tests {
559574
assert_eq!(plan.group_clustering_mode(), &GroupClusteringMode::Full);
560575
assert_completed_groups_emitted_under_limit(&plan).await
561576
}
577+
578+
/// Like [`completed_groups_under_memory_pressure`], but with accumulators
579+
/// that keep per-batch scratch. For example, each `APPROX_DISTINCT` keeps
580+
/// the hashes of the last batch (64 KB for 8192 rows), and the
581+
/// `GroupsAccumulatorAdapter` of `ANY_VALUE` keeps its per-batch indices
582+
/// (~160 KB). After the completed groups are emitted, the accumulators must
583+
/// release that scratch too, or reserving the table fails the limit
584+
/// although the active group needs a few KB. The output must not change.
585+
#[tokio::test]
586+
async fn accumulator_scratch_under_memory_pressure() -> Result<()> {
587+
type Aggregates = fn(&SchemaRef) -> Result<Vec<Arc<AggregateFunctionExpr>>>;
588+
fn build(
589+
udaf: Arc<AggregateUDF>,
590+
args: &[&str],
591+
schema: &SchemaRef,
592+
) -> Result<Arc<AggregateFunctionExpr>> {
593+
let alias = format!("{}({})", udaf.name(), args.join(", "));
594+
let args = args
595+
.iter()
596+
.map(|name| col(name, schema))
597+
.collect::<Result<Vec<_>>>()?;
598+
Ok(Arc::new(
599+
AggregateExprBuilder::new(udaf, args)
600+
.schema(Arc::clone(schema))
601+
.alias(alias)
602+
.build()?,
603+
))
604+
}
605+
let cases: [(&str, Aggregates); 2] = [
606+
// `HllGroupsAccumulator`
607+
("approx_distinct", |schema| {
608+
Ok(vec![
609+
build(approx_distinct_udaf(), &["k0"], schema)?,
610+
build(approx_distinct_udaf(), &["v"], schema)?,
611+
])
612+
}),
613+
// `GroupsAccumulatorAdapter`
614+
("any_value", |schema| {
615+
Ok(vec![build(any_value_udaf(), &["v"], schema)?])
616+
}),
617+
];
618+
for (name, aggregates) in cases {
619+
let input =
620+
|| batches(&[DataType::Int64], &[0..8193, 8192..16385, 16384..24576]);
621+
let schema = input()?[0].schema();
622+
let expected = run_aggregates(
623+
input()?,
624+
aggregates(&schema)?,
625+
Arc::new(UnboundedMemoryPool::default()),
626+
)
627+
.await?;
628+
let actual = run_aggregates(
629+
input()?,
630+
aggregates(&schema)?,
631+
Arc::new(GreedyMemoryPool::new(100 * 1024)),
632+
)
633+
.await
634+
.map_err(|e| e.context(format!("aggregates {name}")))?;
635+
assert_eq!(batches_to_string(&actual), batches_to_string(&expected));
636+
}
637+
Ok(())
638+
}
562639
}

0 commit comments

Comments
 (0)