@@ -159,6 +159,9 @@ pub(crate) struct PartialHashAggregateStream {
159159 /// Number of times accumulated states were emitted due to memory pressure.
160160 early_emit_count : metrics:: Count ,
161161
162+ /// Optional thresholds for emitting partial states before memory pressure.
163+ table_flush : Option < PartialTableFlush > ,
164+
162165 /// Tracks whether partial aggregation should switch to direct state conversion.
163166 skip_aggregation_probe : Option < SkipAggregationProbe > ,
164167
@@ -207,9 +210,22 @@ enum HandleInputResult {
207210 ReachedLimit ,
208211 #[ expect( clippy:: upper_case_acronyms) ]
209212 OOM ,
213+ FlushThresholdReached ,
210214 SwitchToSkipAggregation ,
211215}
212216
217+ /// Emits partial states at size thresholds while preserving skip-probe counts.
218+ struct PartialTableFlush {
219+ /// Allocated bytes above which the current table is emitted.
220+ byte_threshold : usize ,
221+ /// Distinct group rows above which the current table is emitted; zero disables.
222+ group_threshold : usize ,
223+ /// Previously emitted groups, included in the skip-partial reduction ratio.
224+ flushed_groups : usize ,
225+ /// Number of emissions triggered by either size threshold.
226+ flush_count : metrics:: Count ,
227+ }
228+
213229impl PartialHashAggregateStream {
214230 pub fn new (
215231 agg : & AggregateExec ,
@@ -232,6 +248,26 @@ impl PartialHashAggregateStream {
232248 let early_emit_count =
233249 MetricBuilder :: new ( & agg. metrics ) . counter ( "early_emit_count" , partition) ;
234250
251+ let execution_options = & context. session_config ( ) . options ( ) . execution ;
252+ let byte_threshold = execution_options. partial_aggregation_flush_bytes ;
253+ let group_threshold = execution_options. partial_aggregation_flush_rows ;
254+ let group_values_soft_limit = agg. limit_options ( ) . map ( |config| config. limit ( ) ) ;
255+ let has_nested_state = schema
256+ . fields ( )
257+ . iter ( )
258+ . skip ( agg. group_by ( ) . num_group_exprs ( ) )
259+ . any ( |field| field. data_type ( ) . is_nested ( ) ) ;
260+ let table_flush = ( ( byte_threshold > 0 || group_threshold > 0 )
261+ && group_values_soft_limit. is_none ( )
262+ && !has_nested_state)
263+ . then ( || PartialTableFlush {
264+ byte_threshold,
265+ group_threshold,
266+ flushed_groups : 0 ,
267+ flush_count : MetricBuilder :: new ( & agg. metrics )
268+ . counter ( "table_flush_count" , partition) ,
269+ } ) ;
270+
235271 let hash_table = AggregateHashTable :: < PartialMarker > :: new (
236272 agg,
237273 partition,
@@ -279,8 +315,9 @@ impl PartialHashAggregateStream {
279315 reservation,
280316 reduction_factor,
281317 early_emit_count,
318+ table_flush,
282319 skip_aggregation_probe,
283- group_values_soft_limit : agg . limit_options ( ) . map ( |config| config . limit ( ) ) ,
320+ group_values_soft_limit,
284321 hash_table : Some ( hash_table) ,
285322 } )
286323 }
@@ -315,14 +352,23 @@ impl PartialHashAggregateStream {
315352 | HandleInputResult :: SwitchToSkipAggregation => {
316353 break ;
317354 }
318- HandleInputResult :: OOM => {
319- let materialized_group_states = hash_table. take_state_batch ( ) ?. ok_or_else ( || {
320- internal_datafusion_err ! (
321- "Partial hash aggregate ran out of memory with no aggregated groups"
322- )
323- } ) ?;
324-
325- self . early_emit_count . add ( 1 ) ;
355+ HandleInputResult :: OOM | HandleInputResult :: FlushThresholdReached => {
356+ let materialized_group_states =
357+ hash_table. take_state_batch ( ) ?. ok_or_else ( || {
358+ internal_datafusion_err ! (
359+ "Partial hash aggregate tried to flush an empty table"
360+ )
361+ } ) ?;
362+
363+ if let Some ( table_flush) = self . table_flush . as_mut ( )
364+ && last_state == HandleInputResult :: FlushThresholdReached
365+ {
366+ table_flush. flushed_groups +=
367+ materialized_group_states. num_rows ( ) ;
368+ table_flush. flush_count . add ( 1 ) ;
369+ } else {
370+ self . early_emit_count . add ( 1 ) ;
371+ }
326372 timer. done ( ) ;
327373 self . emit_on_memory_pressure (
328374 materialized_group_states,
@@ -410,7 +456,14 @@ impl PartialHashAggregateStream {
410456 // ----------------------------------------------
411457 // Step 3: Skip partial aggregation optimization
412458 // ----------------------------------------------
413- self . update_skip_aggregation_probe ( input_rows, hash_table. building_group_count ( ) ) ;
459+ let flushed_groups = self
460+ . table_flush
461+ . as_ref ( )
462+ . map_or ( 0 , |table_flush| table_flush. flushed_groups ) ;
463+ self . update_skip_aggregation_probe (
464+ input_rows,
465+ flushed_groups + hash_table. building_group_count ( ) ,
466+ ) ;
414467
415468 // True branch: a decision has been made to skip partial aggregation.
416469 if self . should_skip_aggregation ( ) {
@@ -420,12 +473,26 @@ impl PartialHashAggregateStream {
420473 // -------------------------------------------------
421474 // Step 4: Larger-than-memory execution (early emit)
422475 // -------------------------------------------------
423- let resize_result = self . reservation . try_resize ( hash_table. memory_size ( ) ) ;
476+ let allocated_bytes = hash_table. memory_size ( ) ;
477+ let resize_result = self . reservation . try_resize ( allocated_bytes) ;
424478 match resize_result {
425- Ok ( ( ) ) => Ok ( HandleInputResult :: ProcessNext ) ,
426- Err ( DataFusionError :: ResourcesExhausted ( _) ) => Ok ( HandleInputResult :: OOM ) ,
427- Err ( e) => Err ( e) ,
479+ Ok ( ( ) ) => { }
480+ Err ( DataFusionError :: ResourcesExhausted ( _) ) => {
481+ return Ok ( HandleInputResult :: OOM ) ;
482+ }
483+ Err ( e) => return Err ( e) ,
428484 }
485+
486+ // Step 5: Emit partial states once either configured threshold is reached.
487+ if self . table_flush . as_ref ( ) . is_some_and ( |table_flush| {
488+ ( table_flush. byte_threshold > 0
489+ && allocated_bytes >= table_flush. byte_threshold )
490+ || ( table_flush. group_threshold > 0
491+ && hash_table. building_group_count ( ) >= table_flush. group_threshold )
492+ } ) {
493+ return Ok ( HandleInputResult :: FlushThresholdReached ) ;
494+ }
495+ Ok ( HandleInputResult :: ProcessNext )
429496 }
430497
431498 /// Emit a materialized partial-state batch in `batch_size` slices.
@@ -1021,9 +1088,16 @@ mod tests {
10211088 Ok ( ( ) )
10221089 }
10231090
1091+ #[ rstest:: rstest]
1092+ #[ case( 0 , 0 ) ]
1093+ #[ case( 1 , 0 ) ]
1094+ #[ case( 0 , 1 ) ]
1095+ #[ case( 1 , 1 ) ]
10241096 #[ tokio:: test]
1025- async fn test_partial_hash_stream_skip_aggregation_probe_not_locked_until_skip ( )
1026- -> Result < ( ) > {
1097+ async fn test_partial_hash_stream_skip_aggregation_probe_not_locked_until_skip (
1098+ #[ case] flush_bytes : usize ,
1099+ #[ case] flush_rows : usize ,
1100+ ) -> Result < ( ) > {
10271101 // Test that the probe is not locked until we actually decide to skip.
10281102 // This allows us to continue evaluating the skip condition across multiple batches.
10291103 //
@@ -1108,6 +1182,14 @@ mod tests {
11081182
11091183 // Configure skip aggregation settings
11101184 let mut session_config = task_ctx. session_config ( ) . clone ( ) ;
1185+ session_config
1186+ . options_mut ( )
1187+ . execution
1188+ . partial_aggregation_flush_bytes = flush_bytes;
1189+ session_config
1190+ . options_mut ( )
1191+ . execution
1192+ . partial_aggregation_flush_rows = flush_rows;
11111193 session_config = session_config. set (
11121194 "datafusion.execution.skip_partial_aggregation_probe_rows_threshold" ,
11131195 & datafusion_common:: ScalarValue :: UInt64 ( Some ( probe_rows_threshold) ) ,
@@ -1166,6 +1248,11 @@ mod tests {
11661248 "Expected batch 3's rows ({batch3_rows}) to be skipped" ,
11671249 ) ;
11681250
1251+ let flushes = metrics
1252+ . sum_by_name ( "table_flush_count" )
1253+ . map_or ( 0 , |value| value. as_usize ( ) ) ;
1254+ assert_eq ! ( flushes, usize :: from( flush_bytes > 0 || flush_rows > 0 ) ) ;
1255+
11691256 Ok ( ( ) )
11701257 }
11711258
0 commit comments