Skip to content

Commit 634a59f

Browse files
Rachel IntRachel Int
authored andcommitted
perf: add independent partial flush row and byte thresholds
1 parent 7b1e55d commit 634a59f

5 files changed

Lines changed: 384 additions & 16 deletions

File tree

‎datafusion/common/src/config.rs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1136,6 +1136,22 @@ config_namespace! {
11361136
/// aggregation ratio check and trying to switch to skipping aggregation mode
11371137
pub skip_partial_aggregation_probe_rows_threshold: usize, default = 100_000
11381138

1139+
/// (experimental) Allocated-byte threshold for flushing unordered partial
1140+
/// hash aggregation tables. Checked after each input batch; the table can
1141+
/// exceed this size by one batch. Flushed groups count toward the existing
1142+
/// skip-partial probe. Repeated keys do not disable flushing. Aggregations
1143+
/// with a soft group limit or nested aggregate state are excluded.
1144+
/// Set to 0 to disable this threshold. If both flush thresholds are
1145+
/// enabled, reaching either triggers a flush. Final aggregation is unchanged.
1146+
pub partial_aggregation_flush_bytes: usize, default = 0
1147+
1148+
/// (experimental) Number of distinct group rows above which an unordered
1149+
/// partial hash aggregation table is flushed. This counts groups held in
1150+
/// the table, not input rows. Uses the same eligibility and skip-partial
1151+
/// accounting as partial_aggregation_flush_bytes. Set to 0 to disable this
1152+
/// threshold. If both thresholds are enabled, reaching either triggers a flush.
1153+
pub partial_aggregation_flush_rows: usize, default = 0
1154+
11391155
/// Should DataFusion use row number estimates at the input to decide
11401156
/// whether increasing parallelism is beneficial or not. By default,
11411157
/// only exact row numbers (not estimates) are used for this decision.

‎datafusion/physical-plan/src/aggregates/hash_stream.rs‎

Lines changed: 103 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,9 @@ pub(crate) struct PartialHashAggregateStream {
159159
/// Number of times accumulated states were emitted due to memory pressure.
160160
early_emit_count: metrics::Count,
161161

162+
/// Optional thresholds for emitting partial states before memory pressure.
163+
table_flush: Option<PartialTableFlush>,
164+
162165
/// Tracks whether partial aggregation should switch to direct state conversion.
163166
skip_aggregation_probe: Option<SkipAggregationProbe>,
164167

@@ -207,9 +210,22 @@ enum HandleInputResult {
207210
ReachedLimit,
208211
#[expect(clippy::upper_case_acronyms)]
209212
OOM,
213+
FlushThresholdReached,
210214
SwitchToSkipAggregation,
211215
}
212216

217+
/// Emits partial states at size thresholds while preserving skip-probe counts.
218+
struct PartialTableFlush {
219+
/// Allocated bytes above which the current table is emitted.
220+
byte_threshold: usize,
221+
/// Distinct group rows above which the current table is emitted; zero disables.
222+
group_threshold: usize,
223+
/// Previously emitted groups, included in the skip-partial reduction ratio.
224+
flushed_groups: usize,
225+
/// Number of emissions triggered by either size threshold.
226+
flush_count: metrics::Count,
227+
}
228+
213229
impl PartialHashAggregateStream {
214230
pub fn new(
215231
agg: &AggregateExec,
@@ -232,6 +248,26 @@ impl PartialHashAggregateStream {
232248
let early_emit_count =
233249
MetricBuilder::new(&agg.metrics).counter("early_emit_count", partition);
234250

251+
let execution_options = &context.session_config().options().execution;
252+
let byte_threshold = execution_options.partial_aggregation_flush_bytes;
253+
let group_threshold = execution_options.partial_aggregation_flush_rows;
254+
let group_values_soft_limit = agg.limit_options().map(|config| config.limit());
255+
let has_nested_state = schema
256+
.fields()
257+
.iter()
258+
.skip(agg.group_by().num_group_exprs())
259+
.any(|field| field.data_type().is_nested());
260+
let table_flush = ((byte_threshold > 0 || group_threshold > 0)
261+
&& group_values_soft_limit.is_none()
262+
&& !has_nested_state)
263+
.then(|| PartialTableFlush {
264+
byte_threshold,
265+
group_threshold,
266+
flushed_groups: 0,
267+
flush_count: MetricBuilder::new(&agg.metrics)
268+
.counter("table_flush_count", partition),
269+
});
270+
235271
let hash_table = AggregateHashTable::<PartialMarker>::new(
236272
agg,
237273
partition,
@@ -279,8 +315,9 @@ impl PartialHashAggregateStream {
279315
reservation,
280316
reduction_factor,
281317
early_emit_count,
318+
table_flush,
282319
skip_aggregation_probe,
283-
group_values_soft_limit: agg.limit_options().map(|config| config.limit()),
320+
group_values_soft_limit,
284321
hash_table: Some(hash_table),
285322
})
286323
}
@@ -315,14 +352,23 @@ impl PartialHashAggregateStream {
315352
| HandleInputResult::SwitchToSkipAggregation => {
316353
break;
317354
}
318-
HandleInputResult::OOM => {
319-
let materialized_group_states = hash_table.take_state_batch()?.ok_or_else(|| {
320-
internal_datafusion_err!(
321-
"Partial hash aggregate ran out of memory with no aggregated groups"
322-
)
323-
})?;
324-
325-
self.early_emit_count.add(1);
355+
HandleInputResult::OOM | HandleInputResult::FlushThresholdReached => {
356+
let materialized_group_states =
357+
hash_table.take_state_batch()?.ok_or_else(|| {
358+
internal_datafusion_err!(
359+
"Partial hash aggregate tried to flush an empty table"
360+
)
361+
})?;
362+
363+
if let Some(table_flush) = self.table_flush.as_mut()
364+
&& last_state == HandleInputResult::FlushThresholdReached
365+
{
366+
table_flush.flushed_groups +=
367+
materialized_group_states.num_rows();
368+
table_flush.flush_count.add(1);
369+
} else {
370+
self.early_emit_count.add(1);
371+
}
326372
timer.done();
327373
self.emit_on_memory_pressure(
328374
materialized_group_states,
@@ -410,7 +456,14 @@ impl PartialHashAggregateStream {
410456
// ----------------------------------------------
411457
// Step 3: Skip partial aggregation optimization
412458
// ----------------------------------------------
413-
self.update_skip_aggregation_probe(input_rows, hash_table.building_group_count());
459+
let flushed_groups = self
460+
.table_flush
461+
.as_ref()
462+
.map_or(0, |table_flush| table_flush.flushed_groups);
463+
self.update_skip_aggregation_probe(
464+
input_rows,
465+
flushed_groups + hash_table.building_group_count(),
466+
);
414467

415468
// True branch: a decision has been made to skip partial aggregation.
416469
if self.should_skip_aggregation() {
@@ -420,12 +473,26 @@ impl PartialHashAggregateStream {
420473
// -------------------------------------------------
421474
// Step 4: Larger-than-memory execution (early emit)
422475
// -------------------------------------------------
423-
let resize_result = self.reservation.try_resize(hash_table.memory_size());
476+
let allocated_bytes = hash_table.memory_size();
477+
let resize_result = self.reservation.try_resize(allocated_bytes);
424478
match resize_result {
425-
Ok(()) => Ok(HandleInputResult::ProcessNext),
426-
Err(DataFusionError::ResourcesExhausted(_)) => Ok(HandleInputResult::OOM),
427-
Err(e) => Err(e),
479+
Ok(()) => {}
480+
Err(DataFusionError::ResourcesExhausted(_)) => {
481+
return Ok(HandleInputResult::OOM);
482+
}
483+
Err(e) => return Err(e),
428484
}
485+
486+
// Step 5: Emit partial states once either configured threshold is reached.
487+
if self.table_flush.as_ref().is_some_and(|table_flush| {
488+
(table_flush.byte_threshold > 0
489+
&& allocated_bytes >= table_flush.byte_threshold)
490+
|| (table_flush.group_threshold > 0
491+
&& hash_table.building_group_count() >= table_flush.group_threshold)
492+
}) {
493+
return Ok(HandleInputResult::FlushThresholdReached);
494+
}
495+
Ok(HandleInputResult::ProcessNext)
429496
}
430497

431498
/// Emit a materialized partial-state batch in `batch_size` slices.
@@ -1021,9 +1088,16 @@ mod tests {
10211088
Ok(())
10221089
}
10231090

1091+
#[rstest::rstest]
1092+
#[case(0, 0)]
1093+
#[case(1, 0)]
1094+
#[case(0, 1)]
1095+
#[case(1, 1)]
10241096
#[tokio::test]
1025-
async fn test_partial_hash_stream_skip_aggregation_probe_not_locked_until_skip()
1026-
-> Result<()> {
1097+
async fn test_partial_hash_stream_skip_aggregation_probe_not_locked_until_skip(
1098+
#[case] flush_bytes: usize,
1099+
#[case] flush_rows: usize,
1100+
) -> Result<()> {
10271101
// Test that the probe is not locked until we actually decide to skip.
10281102
// This allows us to continue evaluating the skip condition across multiple batches.
10291103
//
@@ -1108,6 +1182,14 @@ mod tests {
11081182

11091183
// Configure skip aggregation settings
11101184
let mut session_config = task_ctx.session_config().clone();
1185+
session_config
1186+
.options_mut()
1187+
.execution
1188+
.partial_aggregation_flush_bytes = flush_bytes;
1189+
session_config
1190+
.options_mut()
1191+
.execution
1192+
.partial_aggregation_flush_rows = flush_rows;
11111193
session_config = session_config.set(
11121194
"datafusion.execution.skip_partial_aggregation_probe_rows_threshold",
11131195
&datafusion_common::ScalarValue::UInt64(Some(probe_rows_threshold)),
@@ -1166,6 +1248,11 @@ mod tests {
11661248
"Expected batch 3's rows ({batch3_rows}) to be skipped",
11671249
);
11681250

1251+
let flushes = metrics
1252+
.sum_by_name("table_flush_count")
1253+
.map_or(0, |value| value.as_usize());
1254+
assert_eq!(flushes, usize::from(flush_bytes > 0 || flush_rows > 0));
1255+
11691256
Ok(())
11701257
}
11711258

0 commit comments

Comments
 (0)