Skip to content

Commit 0da2151

Browse files
authored
feat: add early emit count metric to partial aggregates (#24996)
## Which issue does this PR close? - Closes #24995. ## Why make this change? When DataFusion is working on a partial `GROUP BY` and memory fills up, it sends out the work completed so far and starts collecting again. Today, `EXPLAIN ANALYZE` does not show that this happened. This makes it hard to tell why the partial `GROUP BY` produced more rows than expected. ## What changed? This PR adds an `early_emit_count` number to `EXPLAIN ANALYZE`. It goes up once each time we send out partial results because memory is full. If one set of partial results is sent in several smaller pieces, the number still goes up only once. This change does not affect query results and does not add new logs. ## How was it tested? Tests cover each way DataFusion runs a partial `GROUP BY`. They check that: - the number goes up once for each memory-related send - splitting one send into smaller pieces does not increase it more than once - other ways of sending partial results do not increase it ## Are there any user-facing changes? `EXPLAIN ANALYZE` can now show `early_emit_count` for partial `GROUP BY` steps.
1 parent 4d87baf commit 0da2151

4 files changed

Lines changed: 77 additions & 10 deletions

File tree

‎datafusion/physical-plan/src/aggregates/hash_stream.rs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,9 @@ pub(crate) struct PartialHashAggregateStream {
163163
/// Tracks partial aggregation row reduction, matching `GroupedHashAggregateStream`.
164164
reduction_factor: metrics::RatioMetrics,
165165

166+
/// Number of times accumulated states were emitted due to memory pressure.
167+
early_emit_count: metrics::Count,
168+
166169
/// Tracks whether partial aggregation should switch to direct state conversion.
167170
skip_aggregation_probe: Option<SkipAggregationProbe>,
168171

@@ -389,6 +392,8 @@ impl PartialHashAggregateStream {
389392
let reduction_factor = MetricBuilder::new(&agg.metrics)
390393
.with_type(metrics::MetricType::Summary)
391394
.ratio_metrics("reduction_factor", partition);
395+
let early_emit_count =
396+
MetricBuilder::new(&agg.metrics).counter("early_emit_count", partition);
392397

393398
let hash_table = AggregateHashTable::<PartialMarker>::new(
394399
agg,
@@ -433,6 +438,7 @@ impl PartialHashAggregateStream {
433438
baseline_metrics,
434439
reservation,
435440
reduction_factor,
441+
early_emit_count,
436442
skip_aggregation_probe,
437443
group_values_soft_limit: agg.limit_options().map(|config| config.limit()),
438444
hash_table: Some(hash_table),
@@ -476,6 +482,7 @@ impl PartialHashAggregateStream {
476482
)
477483
})?;
478484

485+
self.early_emit_count.add(1);
479486
timer.done();
480487
self.emit_on_memory_pressure(
481488
materialized_group_states,
@@ -1129,6 +1136,45 @@ mod tests {
11291136
total_output_groups, num_groups,
11301137
"Unexpected number of groups",
11311138
);
1139+
assert_eq!(
1140+
aggregate_exec
1141+
.metrics()
1142+
.unwrap()
1143+
.sum_by_name("early_emit_count")
1144+
.unwrap()
1145+
.as_usize(),
1146+
0
1147+
);
1148+
1149+
// Disable skip aggregation so the same input is emitted on memory pressure.
1150+
let runtime = RuntimeEnvBuilder::default()
1151+
.with_memory_limit(1024, 1.0)
1152+
.build_arc()?;
1153+
let session_config = task_ctx.session_config().clone().set(
1154+
"datafusion.execution.skip_partial_aggregation_probe_ratio_threshold",
1155+
&datafusion_common::ScalarValue::Float64(Some(2.0)),
1156+
);
1157+
let no_skip_task_ctx = Arc::new(
1158+
TaskContext::default()
1159+
.with_runtime(runtime)
1160+
.with_session_config(session_config),
1161+
);
1162+
let mut stream =
1163+
PartialHashAggregateStream::new(&aggregate_exec, &no_skip_task_ctx, 0)?
1164+
.into_stream();
1165+
while let Some(result) = stream.next().await {
1166+
result?;
1167+
}
1168+
1169+
assert_eq!(
1170+
aggregate_exec
1171+
.metrics()
1172+
.unwrap()
1173+
.sum_by_name("early_emit_count")
1174+
.unwrap()
1175+
.as_usize(),
1176+
1
1177+
);
11321178

11331179
Ok(())
11341180
}

‎datafusion/physical-plan/src/aggregates/mod.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4831,6 +4831,16 @@ mod tests {
48314831
let stream: SendableRecordBatchStream = stream.into();
48324832
let output = collect(stream).await?;
48334833

4834+
assert_eq!(
4835+
partial_reduce
4836+
.metrics()
4837+
.unwrap()
4838+
.sum_by_name("early_emit_count")
4839+
.unwrap()
4840+
.as_usize(),
4841+
num_input_batches
4842+
);
4843+
48344844
// The table is flushed after every input batch, so each of the three
48354845
// groups is emitted once per input batch instead of being merged into a
48364846
// single row. Each flush is sliced into batches of 2 and 1 rows.

‎datafusion/physical-plan/src/aggregates/partial_reduce_stream.rs‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ use futures::stream::{Stream, StreamExt};
3030

3131
use super::AggregateExec;
3232
use super::aggregate_hash_table::{AggregateHashTable, PartialReduceMarker};
33-
use crate::metrics::{BaselineMetrics, RecordOutput, SpillMetrics};
33+
use crate::metrics::{BaselineMetrics, Count, MetricBuilder, RecordOutput, SpillMetrics};
3434
use crate::stream::EmptyRecordBatchStream;
3535
use crate::{InputOrderMode, RecordBatchStream, SendableRecordBatchStream};
3636

@@ -88,6 +88,9 @@ pub(crate) struct PartialReduceHashAggregateStream {
8888
/// Memory reservation for group keys and accumulators.
8989
reservation: MemoryReservation,
9090

91+
/// Number of times accumulated states were emitted due to memory pressure.
92+
early_emit_count: Count,
93+
9194
/// Tracks the high-level stream lifecycle. The hash table owns the lower-level
9295
/// state for emitting output batches.
9396
state: Option<PartialReduceHashAggregateState>,
@@ -188,6 +191,8 @@ impl PartialReduceHashAggregateStream {
188191

189192
// Preserve the existing aggregate metric surface for this plan node.
190193
let _spill_metrics = SpillMetrics::new(&agg.metrics, partition);
194+
let early_emit_count =
195+
MetricBuilder::new(&agg.metrics).counter("early_emit_count", partition);
191196

192197
let hash_table = AggregateHashTable::<PartialReduceMarker>::new(
193198
agg,
@@ -210,6 +215,7 @@ impl PartialReduceHashAggregateStream {
210215
batch_size,
211216
baseline_metrics,
212217
reservation,
218+
early_emit_count,
213219
state: Some(PartialReduceHashAggregateState::ReadingInput { hash_table }),
214220
})
215221
}
@@ -315,12 +321,15 @@ impl PartialReduceHashAggregateStream {
315321
let state_batch_result = original_state.hash_table_mut().take_state_batch();
316322

317323
match state_batch_result {
318-
Ok(Some(remaining_groups)) => ControlFlow::Continue(
319-
PartialReduceHashAggregateState::EmittingOnMemoryPressure {
320-
hash_table: original_state.into_hash_table(),
321-
remaining_groups,
322-
},
323-
),
324+
Ok(Some(remaining_groups)) => {
325+
self.early_emit_count.add(1);
326+
ControlFlow::Continue(
327+
PartialReduceHashAggregateState::EmittingOnMemoryPressure {
328+
hash_table: original_state.into_hash_table(),
329+
remaining_groups,
330+
},
331+
)
332+
}
324333
// No accumulated group to emit, so early emission cannot release any
325334
// memory: report the original error.
326335
Ok(None) => Self::break_with_err(oom),

‎datafusion/sqllogictest/test_files/aggregate_memory_spill.slt‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -208,9 +208,9 @@ FROM (
208208
----
209209
100000 5000050000
210210

211-
# Assert spill happened in the final aggregation.
212-
# In multi-partitions configuration, 'spilled_rows' is not deterministic, so assert
213-
# the unit to be 'K'
211+
# Assert spill happened in the final aggregation and the partial aggregation
212+
# reports memory-pressure emissions. Their exact counts are not deterministic,
213+
# so assert only the spilled_rows unit and the presence of early_emit_count.
214214
query TT
215215
EXPLAIN ANALYZE
216216
SELECT count(*), sum(total)
@@ -223,6 +223,8 @@ FROM (
223223
<slt:ignore>
224224
06)----------AggregateExec: mode=FinalPartitioned, gby=[t.v * Int64(7) % Int64(100000)@0 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v)], metrics=[<slt:ignore>spilled_rows=<slt:ignore>K,<slt:ignore>]
225225
<slt:ignore>
226+
08)--------------AggregateExec: mode=Partial, gby=[v@0 * 7 % 100000 as t.v * Int64(7) % Int64(100000)], aggr=[sum(t.v)], metrics=[<slt:ignore>early_emit_count=<slt:ignore>]
227+
<slt:ignore>
226228

227229
# Restore settings to slt runner defaults
228230
statement ok

0 commit comments

Comments
 (0)