@@ -434,9 +434,14 @@ mod tests {
434434 GreedyMemoryPool , MemoryPool , UnboundedMemoryPool ,
435435 } ;
436436 use datafusion_execution:: runtime_env:: RuntimeEnvBuilder ;
437+ use datafusion_expr:: AggregateUDF ;
438+ use datafusion_functions_aggregate:: any_value:: any_value_udaf;
439+ use datafusion_functions_aggregate:: approx_distinct:: approx_distinct_udaf;
437440 use datafusion_functions_aggregate:: sum:: sum_udaf;
438441 use datafusion_physical_expr:: PhysicalSortExpr ;
439- use datafusion_physical_expr:: aggregate:: AggregateExprBuilder ;
442+ use datafusion_physical_expr:: aggregate:: {
443+ AggregateExprBuilder , AggregateFunctionExpr ,
444+ } ;
440445 use datafusion_physical_expr:: expressions:: col;
441446 use datafusion_physical_expr_common:: sort_expr:: LexOrdering ;
442447
@@ -484,6 +489,21 @@ mod tests {
484489 async fn run (
485490 batches : Vec < RecordBatch > ,
486491 pool : Arc < dyn MemoryPool > ,
492+ ) -> Result < Vec < RecordBatch > > {
493+ let schema = batches[ 0 ] . schema ( ) ;
494+ let sum = AggregateExprBuilder :: new ( sum_udaf ( ) , vec ! [ col( "v" , & schema) ?] )
495+ . schema ( Arc :: clone ( & schema) )
496+ . alias ( "sum" )
497+ . build ( ) ?;
498+ run_aggregates ( batches, vec ! [ Arc :: new( sum) ] , pool) . await
499+ }
500+
501+ /// Runs a partial aggregate of `aggregates` grouped by `k0, ..` over
502+ /// `batches`, which are sorted on every key.
503+ async fn run_aggregates (
504+ batches : Vec < RecordBatch > ,
505+ aggregates : Vec < Arc < AggregateFunctionExpr > > ,
506+ pool : Arc < dyn MemoryPool > ,
487507 ) -> Result < Vec < RecordBatch > > {
488508 let schema = batches[ 0 ] . schema ( ) ;
489509 let keys = ( 0 ..schema. fields ( ) . len ( ) - 1 )
@@ -499,13 +519,8 @@ mod tests {
499519 let aggregate = AggregateExec :: try_new (
500520 AggregateMode :: Partial ,
501521 PhysicalGroupBy :: new_single ( keys) ,
502- vec ! [ Arc :: new(
503- AggregateExprBuilder :: new( sum_udaf( ) , vec![ col( "v" , & schema) ?] )
504- . schema( Arc :: clone( & schema) )
505- . alias( "sum" )
506- . build( ) ?,
507- ) ] ,
508- vec ! [ None ] ,
522+ aggregates. clone ( ) ,
523+ vec ! [ None ; aggregates. len( ) ] ,
509524 Arc :: new ( input) ,
510525 Arc :: clone ( & schema) ,
511526 ) ?;
@@ -559,4 +574,66 @@ mod tests {
559574 assert_eq ! ( plan. group_clustering_mode( ) , & GroupClusteringMode :: Full ) ;
560575 assert_completed_groups_emitted_under_limit ( & plan) . await
561576 }
577+
578+ /// Like [`completed_groups_under_memory_pressure`], but with accumulators
579+ /// that keep per-batch scratch. For example, each `APPROX_DISTINCT` keeps
580+ /// the hashes of the last batch (64 KB for 8192 rows), and the
581+ /// `GroupsAccumulatorAdapter` of `ANY_VALUE` keeps its per-batch indices
582+ /// (~160 KB). After the completed groups are emitted, the accumulators must
583+ /// release that scratch too, or reserving the table fails the limit
584+ /// although the active group needs a few KB. The output must not change.
585+ #[ tokio:: test]
586+ async fn accumulator_scratch_under_memory_pressure ( ) -> Result < ( ) > {
587+ type Aggregates = fn ( & SchemaRef ) -> Result < Vec < Arc < AggregateFunctionExpr > > > ;
588+ fn build (
589+ udaf : Arc < AggregateUDF > ,
590+ args : & [ & str ] ,
591+ schema : & SchemaRef ,
592+ ) -> Result < Arc < AggregateFunctionExpr > > {
593+ let alias = format ! ( "{}({})" , udaf. name( ) , args. join( ", " ) ) ;
594+ let args = args
595+ . iter ( )
596+ . map ( |name| col ( name, schema) )
597+ . collect :: < Result < Vec < _ > > > ( ) ?;
598+ Ok ( Arc :: new (
599+ AggregateExprBuilder :: new ( udaf, args)
600+ . schema ( Arc :: clone ( schema) )
601+ . alias ( alias)
602+ . build ( ) ?,
603+ ) )
604+ }
605+ let cases: [ ( & str , Aggregates ) ; 2 ] = [
606+ // `HllGroupsAccumulator`
607+ ( "approx_distinct" , |schema| {
608+ Ok ( vec ! [
609+ build( approx_distinct_udaf( ) , & [ "k0" ] , schema) ?,
610+ build( approx_distinct_udaf( ) , & [ "v" ] , schema) ?,
611+ ] )
612+ } ) ,
613+ // `GroupsAccumulatorAdapter`
614+ ( "any_value" , |schema| {
615+ Ok ( vec ! [ build( any_value_udaf( ) , & [ "v" ] , schema) ?] )
616+ } ) ,
617+ ] ;
618+ for ( name, aggregates) in cases {
619+ let input =
620+ || batches ( & [ DataType :: Int64 ] , & [ 0 ..8193 , 8192 ..16385 , 16384 ..24576 ] ) ;
621+ let schema = input ( ) ?[ 0 ] . schema ( ) ;
622+ let expected = run_aggregates (
623+ input ( ) ?,
624+ aggregates ( & schema) ?,
625+ Arc :: new ( UnboundedMemoryPool :: default ( ) ) ,
626+ )
627+ . await ?;
628+ let actual = run_aggregates (
629+ input ( ) ?,
630+ aggregates ( & schema) ?,
631+ Arc :: new ( GreedyMemoryPool :: new ( 100 * 1024 ) ) ,
632+ )
633+ . await
634+ . map_err ( |e| e. context ( format ! ( "aggregates {name}" ) ) ) ?;
635+ assert_eq ! ( batches_to_string( & actual) , batches_to_string( & expected) ) ;
636+ }
637+ Ok ( ( ) )
638+ }
562639}
0 commit comments