Skip to content

Commit 6924c6a

Browse files
authored
fix: reserve initial table memory in aggregate streams (#25900)
## Which issue does this PR close? None ## Rationale for this change pre allocated memory and held memory should be counted ## What changes are included in this PR? reserve the hash table memory on creation, similar to what was already done in: - #24692 ## What is the testing strategy for this PR? added unit tests ## Are there any user-facing changes? No
1 parent 060ac92 commit 6924c6a

8 files changed

Lines changed: 301 additions & 101 deletions

File tree

‎datafusion/core/tests/memory_limit/mod.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,10 @@ async fn group_by_row_hash() {
111111
"Resources exhausted: Additional allocation failed",
112112
"for FinalHashAggregateStream[0]",
113113
])
114-
.with_memory_limit(2_000)
114+
// Every aggregate stream reserves its initial table (~4 KB) when created, so
115+
// pin the partition count and fit the two partial and two final tables.
116+
.with_config(SessionConfig::new().with_target_partitions(2))
117+
.with_memory_limit(20_000)
115118
.run()
116119
.await
117120
}

‎datafusion/physical-plan/src/aggregates/hash_stream.rs‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,9 @@ impl PartialHashAggregateStream {
268268
.with_can_spill(true)
269269
.register(context.memory_pool());
270270

271+
// Reserve memory for the initial hash table. that we hold for the lifetime of the stream.
272+
reservation.try_grow(hash_table.memory_size())?;
273+
271274
Ok(Self {
272275
schema,
273276
input,
@@ -617,6 +620,9 @@ impl FinalHashAggregateStream {
617620
.with_can_spill(can_spill)
618621
.register(context.memory_pool());
619622

623+
// Reserve memory for the initial hash table. that we hold for the lifetime of the stream.
624+
reservation.try_grow(hash_table.memory_size())?;
625+
620626
Ok(Self {
621627
schema,
622628
input,
@@ -926,7 +932,7 @@ mod tests {
926932

927933
// Create constrained memory to trigger early emission but not completely fail
928934
let runtime = RuntimeEnvBuilder::default()
929-
.with_memory_limit(1024, 1.0) // small enough to start but will trigger pressure
935+
.with_memory_limit(5 * 1024, 1.0) // fits the initial table but will trigger pressure
930936
.build_arc()?;
931937

932938
let mut task_ctx = TaskContext::default().with_runtime(runtime);
@@ -1007,7 +1013,7 @@ mod tests {
10071013

10081014
// Disable skip aggregation so the same input is emitted on memory pressure.
10091015
let runtime = RuntimeEnvBuilder::default()
1010-
.with_memory_limit(1024, 1.0)
1016+
.with_memory_limit(5 * 1024, 1.0)
10111017
.build_arc()?;
10121018
let session_config = task_ctx.session_config().clone().set(
10131019
"datafusion.execution.skip_partial_aggregation_probe_ratio_threshold",

0 commit comments

Comments
 (0)