@@ -1044,6 +1044,10 @@ impl AggregateExec {
10441044 // states without using group boundaries to recognize completed groups.
10451045 if group_by. has_grouping_set ( ) || mode == AggregateMode :: PartialReduce {
10461046 group_clustering_mode = GroupClusteringMode :: None ;
1047+ } else if num_non_constant_groupby_exprs > 0
1048+ && input_eq_properties. grouping_satisfy ( groupby_exprs. iter ( ) . cloned ( ) ) ?
1049+ {
1050+ group_clustering_mode = GroupClusteringMode :: Full ;
10471051 }
10481052
10491053 // construct a map from the input expression to the output expression of the Aggregation group by
@@ -2457,7 +2461,7 @@ impl ExecutionPlan for AggregateExec {
24572461 metrics : _,
24582462 // Derived at construction from the input ordering and `group_by`.
24592463 required_input_ordering : _,
2460- // Derived at construction from the input ordering and `group_by`.
2464+ // Derived at construction from the input properties and `group_by`.
24612465 group_clustering_mode : _,
24622466 // Derived at construction by `Self::compute_properties`.
24632467 cache : _,
@@ -3381,7 +3385,7 @@ mod tests {
33813385 Int64Array , NullArray , StringArray , StructArray , UInt32Array , UInt64Array ,
33823386 } ;
33833387 use arrow:: compute:: { SortOptions , concat_batches} ;
3384- use arrow:: datatypes:: { Float64Type , Int32Type , Int64Type , UInt32Type } ;
3388+ use arrow:: datatypes:: { Float64Type , Int32Type , Int64Type , TimeUnit , UInt32Type } ;
33853389 use datafusion_common:: test_util:: { batches_to_sort_string, batches_to_string} ;
33863390 use datafusion_common:: { DataFusionError , assert_contains, internal_err} ;
33873391 use datafusion_execution:: config:: SessionConfig ;
@@ -3394,6 +3398,7 @@ mod tests {
33943398 Accumulator , AggregateUDF , AggregateUDFImpl , EmitTo , GroupsAccumulator , Operator ,
33953399 Signature , Volatility ,
33963400 } ;
3401+ use datafusion_functions:: datetime:: date_bin;
33973402 use datafusion_functions_aggregate:: approx_percentile_cont:: approx_percentile_cont_udaf;
33983403 use datafusion_functions_aggregate:: array_agg:: array_agg_udaf;
33993404 use datafusion_functions_aggregate:: average:: avg_udaf;
@@ -3402,10 +3407,9 @@ mod tests {
34023407 use datafusion_functions_aggregate:: median:: median_udaf;
34033408 use datafusion_functions_aggregate:: min_max:: min_udaf;
34043409 use datafusion_functions_aggregate:: sum:: sum_udaf;
3405- use datafusion_physical_expr:: Partitioning ;
3406- use datafusion_physical_expr:: PhysicalSortExpr ;
34073410 use datafusion_physical_expr:: aggregate:: AggregateExprBuilder ;
34083411 use datafusion_physical_expr:: expressions:: { Literal , NotExpr , binary} ;
3412+ use datafusion_physical_expr:: { Partitioning , PhysicalSortExpr , ScalarFunctionExpr } ;
34093413
34103414 use crate :: projection:: ProjectionExec ;
34113415 use crate :: repartition:: RepartitionExec ;
@@ -5961,7 +5965,7 @@ mod tests {
59615965 }
59625966
59635967 #[ tokio:: test]
5964- async fn unsorted_contiguous_groups_use_final_emission ( ) -> Result < ( ) > {
5968+ async fn unsorted_contiguous_groups_use_incremental_emission ( ) -> Result < ( ) > {
59655969 let schema = Arc :: new ( Schema :: new ( vec ! [
59665970 Field :: new( "key" , DataType :: Int32 , false ) ,
59675971 Field :: new( "time_bin" , DataType :: Int64 , false ) ,
@@ -5989,18 +5993,21 @@ mod tests {
59895993 ] ,
59905994 ) ?,
59915995 ] ;
5996+ let key = col ( "key" , & schema) ?;
5997+ let time_bin = col ( "time_bin" , & schema) ?;
59925998 let group_by = PhysicalGroupBy :: new_single ( vec ! [
5993- ( col ( "key" , & schema ) ? , "key" . to_string( ) ) ,
5994- ( col ( "time_bin" , & schema ) ? , "time_bin" . to_string( ) ) ,
5999+ ( Arc :: clone ( & key ) , "key" . to_string( ) ) ,
6000+ ( Arc :: clone ( & time_bin ) , "time_bin" . to_string( ) ) ,
59956001 ] ) ;
59966002 let aggr_expr = Arc :: new (
59976003 AggregateExprBuilder :: new ( sum_udaf ( ) , vec ! [ col( "value" , & schema) ?] )
59986004 . schema ( Arc :: clone ( & schema) )
59996005 . alias ( "SUM(value)" )
60006006 . build ( ) ?,
60016007 ) ;
6002- let input: Arc < dyn ExecutionPlan > =
6003- TestMemoryExec :: try_new_exec ( & [ input_batches] , Arc :: clone ( & schema) , None ) ?;
6008+ let input = TestMemoryExec :: try_new ( & [ input_batches] , Arc :: clone ( & schema) , None ) ?
6009+ . try_with_grouping_information ( vec ! [ vec![ key, time_bin] ] ) ?;
6010+ let input: Arc < dyn ExecutionPlan > = Arc :: new ( input) ;
60046011 assert_eq ! ( input. output_partitioning( ) . partition_count( ) , 1 ) ;
60056012
60066013 let aggregate = AggregateExec :: try_new (
@@ -6014,16 +6021,21 @@ mod tests {
60146021
60156022 assert_eq ! (
60166023 aggregate. group_clustering_mode( ) ,
6017- & GroupClusteringMode :: None
6024+ & GroupClusteringMode :: Full
6025+ ) ;
6026+ assert_eq ! ( aggregate. cache( ) . emission_type, EmissionType :: Incremental ) ;
6027+ assert ! (
6028+ aggregate
6029+ . cache( )
6030+ . equivalence_properties( )
6031+ . geq_class( )
6032+ . is_empty( )
60186033 ) ;
6019- // This captures the behavior before #24438. When the source can declare
6020- // `(key, time_bin)` group-contiguous, the corresponding case can use
6021- // `EmissionType::Incremental`.
6022- assert_eq ! ( aggregate. cache( ) . emission_type, EmissionType :: Final ) ;
6034+ assert ! ( aggregate. cache( ) . output_ordering( ) . is_none( ) ) ;
60236035
60246036 let task_ctx = new_migrated_hash_ctx ( 1024 ) ;
60256037 let stream = aggregate. execute_typed ( 0 , & task_ctx) ?;
6026- assert ! ( matches!( stream, StreamType :: SingleHash ( _) ) ) ;
6038+ assert ! ( matches!( stream, StreamType :: ClusteredSingleAggregate ( _) ) ) ;
60276039 let stream: SendableRecordBatchStream = stream. into ( ) ;
60286040 let output = collect ( stream) . await ?;
60296041 assert_snapshot ! ( batches_to_sort_string( & output) , @r"
@@ -6040,6 +6052,68 @@ mod tests {
60406052 Ok ( ( ) )
60416053 }
60426054
6055+ #[ test]
6056+ fn grouped_date_bin_projects_to_aggregate ( ) -> Result < ( ) > {
6057+ let schema = Arc :: new ( Schema :: new ( vec ! [
6058+ Field :: new( "key" , DataType :: Int32 , false ) ,
6059+ Field :: new( "time" , DataType :: Timestamp ( TimeUnit :: Second , None ) , false ) ,
6060+ ] ) ) ;
6061+ let time_bin_expr = || -> Result < Arc < dyn PhysicalExpr > > {
6062+ Ok ( Arc :: new ( ScalarFunctionExpr :: try_new (
6063+ date_bin ( ) ,
6064+ vec ! [
6065+ lit( ScalarValue :: new_interval_dt( 0 , 10_000 ) ) ,
6066+ col( "time" , & schema) ?,
6067+ ] ,
6068+ & schema,
6069+ Arc :: new ( ConfigOptions :: default ( ) ) ,
6070+ ) ?) )
6071+ } ;
6072+ let input = TestMemoryExec :: try_new ( & [ vec ! [ ] ] , Arc :: clone ( & schema) , None ) ?
6073+ . try_with_grouping_information ( vec ! [ vec![
6074+ col( "key" , & schema) ?,
6075+ time_bin_expr( ) ?,
6076+ ] ] ) ?;
6077+ let projection = ProjectionExec :: try_new (
6078+ [
6079+ ProjectionExpr :: new ( col ( "key" , & schema) ?, "key" ) ,
6080+ // Build this expression independently from the source
6081+ // assertion so the test exercises semantic expression matching.
6082+ ProjectionExpr :: new ( time_bin_expr ( ) ?, "time_bin" ) ,
6083+ ] ,
6084+ Arc :: new ( input) ,
6085+ ) ?;
6086+
6087+ let projected_schema = projection. schema ( ) ;
6088+ let key = col ( "key" , & projected_schema) ?;
6089+ let time_bin = col ( "time_bin" , & projected_schema) ?;
6090+ assert ! (
6091+ projection
6092+ . properties( )
6093+ . equivalence_properties( )
6094+ . grouping_satisfy( [ Arc :: clone( & key) , Arc :: clone( & time_bin) ] ) ?
6095+ ) ;
6096+ let aggregate = AggregateExec :: try_new (
6097+ AggregateMode :: Single ,
6098+ PhysicalGroupBy :: new_single ( vec ! [
6099+ ( key, "key" . to_string( ) ) ,
6100+ ( time_bin, "time_bin" . to_string( ) ) ,
6101+ ] ) ,
6102+ vec ! [ ] ,
6103+ vec ! [ ] ,
6104+ Arc :: new ( projection) ,
6105+ projected_schema,
6106+ ) ?;
6107+
6108+ assert ! ( aggregate. input( ) . output_ordering( ) . is_none( ) ) ;
6109+ assert_eq ! (
6110+ aggregate. group_clustering_mode( ) ,
6111+ & GroupClusteringMode :: Full
6112+ ) ;
6113+ assert_eq ! ( aggregate. cache( ) . emission_type, EmissionType :: Incremental ) ;
6114+ Ok ( ( ) )
6115+ }
6116+
60436117 /// Ensures for ordered input, `ClusteredPartialAggregateStream` is used.
60446118 #[ tokio:: test]
60456119 async fn ordered_partial_aggregate_planning ( ) -> Result < ( ) > {
0 commit comments