@@ -3376,7 +3376,8 @@ mod tests {
33763376 use crate :: test:: TestMemoryExec ;
33773377 use crate :: test:: assert_is_pending;
33783378 use crate :: test:: exec:: {
3379- BlockingExec , PanicExec , StatisticsExec , assert_strong_count_converges_to_zero,
3379+ BarrierExec , BlockingExec , PanicExec , StatisticsExec ,
3380+ assert_strong_count_converges_to_zero,
33803381 } ;
33813382
33823383 use arrow:: array:: AsArray ;
@@ -5569,7 +5570,7 @@ mod tests {
55695570 }
55705571
55715572 #[ test]
5572- fn partial_reduce_does_not_advertise_input_ordering ( ) -> Result < ( ) > {
5573+ fn partial_reduce_does_not_use_group_clustering ( ) -> Result < ( ) > {
55735574 let schema = Arc :: new ( Schema :: new ( vec ! [
55745575 Field :: new( "a" , DataType :: UInt32 , false ) ,
55755576 Field :: new( "b" , DataType :: Float64 , false ) ,
@@ -5578,39 +5579,113 @@ mod tests {
55785579 Column :: new ( "a" , 0 ) ,
55795580 ) ) ] )
55805581 . unwrap ( ) ;
5581- let input = TestMemoryExec :: try_new ( & [ vec ! [ ] ] , Arc :: clone ( & schema) , None ) ?
5582- . try_with_sort_information ( vec ! [ ordering] ) ?;
5583- let input = Arc :: new ( TestMemoryExec :: update_cache ( & Arc :: new ( input) ) ) ;
5584- assert ! (
5585- input. properties( ) . output_ordering( ) . is_some( ) ,
5586- "test setup: the input is ordered by the group key"
5587- ) ;
5582+ let ordered_input =
5583+ TestMemoryExec :: try_new ( & [ vec ! [ ] ] , Arc :: clone ( & schema) , None ) ?
5584+ . try_with_sort_information ( vec ! [ ordering] ) ?;
5585+ let grouped_input =
5586+ TestMemoryExec :: try_new ( & [ vec ! [ ] ] , Arc :: clone ( & schema) , None ) ?
5587+ . try_with_grouping_information ( vec ! [ vec![ col( "a" , & schema) ?] ] ) ?;
55885588
5589- let partial_reduce = AggregateExec :: try_new (
5590- AggregateMode :: PartialReduce ,
5591- PhysicalGroupBy :: new_single ( vec ! [ ( col( "a" , & schema) ?, "a" . to_string( ) ) ] ) ,
5592- vec ! [ Arc :: new(
5593- AggregateExprBuilder :: new( sum_udaf( ) , vec![ col( "b" , & schema) ?] )
5594- . schema( Arc :: clone( & schema) )
5595- . alias( "SUM(b)" )
5596- . build( ) ?,
5597- ) ] ,
5598- vec ! [ None ] ,
5599- input,
5589+ for input in [ ordered_input, grouped_input] {
5590+ assert ! (
5591+ input
5592+ . properties( )
5593+ . equivalence_properties( )
5594+ . grouping_satisfy( [ col( "a" , & schema) ?] ) ?
5595+ ) ;
5596+ let partial_reduce = AggregateExec :: try_new (
5597+ AggregateMode :: PartialReduce ,
5598+ PhysicalGroupBy :: new_single ( vec ! [ ( col( "a" , & schema) ?, "a" . to_string( ) ) ] ) ,
5599+ vec ! [ Arc :: new(
5600+ AggregateExprBuilder :: new( sum_udaf( ) , vec![ col( "b" , & schema) ?] )
5601+ . schema( Arc :: clone( & schema) )
5602+ . alias( "SUM(b)" )
5603+ . build( ) ?,
5604+ ) ] ,
5605+ vec ! [ None ] ,
5606+ Arc :: new ( input) ,
5607+ Arc :: clone ( & schema) ,
5608+ ) ?;
5609+
5610+ assert_eq ! (
5611+ partial_reduce. group_clustering_mode( ) ,
5612+ & GroupClusteringMode :: None
5613+ ) ;
5614+ assert_eq ! ( partial_reduce. maintains_input_order( ) , vec![ false ] ) ;
5615+ assert ! (
5616+ partial_reduce. properties( ) . output_ordering( ) . is_none( ) ,
5617+ "partial reduce advertised an ordering it does not maintain: {:?}" ,
5618+ partial_reduce. properties( ) . output_ordering( )
5619+ ) ;
5620+
5621+ let stream = partial_reduce. execute_typed ( 0 , & new_migrated_hash_ctx ( 1024 ) ) ?;
5622+ assert ! ( matches!( stream, StreamType :: PartialReduceHash ( _) ) ) ;
5623+ }
5624+
5625+ Ok ( ( ) )
5626+ }
5627+
5628+ #[ tokio:: test]
5629+ async fn constant_only_grouping_uses_final_emission ( ) -> Result < ( ) > {
5630+ let schema = Arc :: new ( Schema :: new ( vec ! [
5631+ Field :: new( "key" , DataType :: Int32 , false ) ,
5632+ Field :: new( "value" , DataType :: Int64 , false ) ,
5633+ ] ) ) ;
5634+ let batch = RecordBatch :: try_new (
5635+ Arc :: clone ( & schema) ,
5636+ vec ! [
5637+ Arc :: new( Int32Array :: from( vec![ 1 , 1 ] ) ) ,
5638+ Arc :: new( Int64Array :: from( vec![ 10 , 20 ] ) ) ,
5639+ ] ,
5640+ ) ?;
5641+ let input = TestMemoryExec :: try_new_exec (
5642+ & [ vec ! [ batch. clone( ) , batch] ] ,
56005643 Arc :: clone ( & schema) ,
5644+ None ,
56015645 ) ?;
5646+ let predicate = binary ( col ( "key" , & schema) ?, Operator :: Eq , lit ( 1i32 ) , & schema) ?;
5647+ let input: Arc < dyn ExecutionPlan > =
5648+ Arc :: new ( FilterExecBuilder :: new ( predicate, input) . build ( ) ?) ;
56025649
5603- assert_eq ! (
5604- partial_reduce. group_clustering_mode( ) ,
5605- & GroupClusteringMode :: None
5606- ) ;
5607- assert_eq ! ( partial_reduce. maintains_input_order( ) , vec![ false ] ) ;
5608- assert ! (
5609- partial_reduce. properties( ) . output_ordering( ) . is_none( ) ,
5610- "partial reduce advertised an ordering it does not maintain: {:?}" ,
5611- partial_reduce. properties( ) . output_ordering( )
5612- ) ;
5650+ // Cover both a literal and a column known to be constant after filtering.
5651+ for key in [ lit ( 1i32 ) , col ( "key" , & schema) ?] {
5652+ assert ! (
5653+ input
5654+ . equivalence_properties( )
5655+ . grouping_satisfy( [ Arc :: clone( & key) ] ) ?
5656+ ) ;
5657+ let aggregate = AggregateExec :: try_new (
5658+ AggregateMode :: Single ,
5659+ PhysicalGroupBy :: new_single ( vec ! [ ( key, "key" . to_string( ) ) ] ) ,
5660+ vec ! [ Arc :: new(
5661+ AggregateExprBuilder :: new( sum_udaf( ) , vec![ col( "value" , & schema) ?] )
5662+ . schema( Arc :: clone( & schema) )
5663+ . alias( "SUM(value)" )
5664+ . build( ) ?,
5665+ ) ] ,
5666+ vec ! [ None ] ,
5667+ Arc :: clone ( & input) ,
5668+ Arc :: clone ( & schema) ,
5669+ ) ?;
5670+ assert_eq ! (
5671+ aggregate. group_clustering_mode( ) ,
5672+ & GroupClusteringMode :: None
5673+ ) ;
5674+ assert_eq ! ( aggregate. cache( ) . emission_type, EmissionType :: Final ) ;
56135675
5676+ let stream = aggregate. execute_typed ( 0 , & new_migrated_hash_ctx ( 1024 ) ) ?;
5677+ assert ! ( matches!( stream, StreamType :: SingleHash ( _) ) ) ;
5678+ let output = collect ( stream. into ( ) ) . await ?;
5679+ allow_duplicates ! {
5680+ assert_snapshot!( batches_to_string( & output) , @r"
5681+ +-----+------------+
5682+ | key | SUM(value) |
5683+ +-----+------------+
5684+ | 1 | 60 |
5685+ +-----+------------+
5686+ " ) ;
5687+ }
5688+ }
56145689 Ok ( ( ) )
56155690 }
56165691
@@ -5971,25 +6046,24 @@ mod tests {
59716046 Field :: new( "time_bin" , DataType :: Int64 , false ) ,
59726047 Field :: new( "value" , DataType :: Int64 , false ) ,
59736048 ] ) ) ;
5974- // Two sorted logical runs are emitted as batches in one DataFusion
5975- // partition. Every distinct grouping tuple occupies one contiguous range,
5976- // but tuple order resets at the batch boundary, so (key, time_bin) is not
5977- // globally sorted.
6049+ // Every grouping tuple occupies one contiguous range in a single
6050+ // partition, with (2, 20) spanning both batches. Tuple order resets
6051+ // within the second batch, so (key, time_bin) is not globally sorted.
59786052 let input_batches = vec ! [
59796053 RecordBatch :: try_new(
59806054 Arc :: clone( & schema) ,
59816055 vec![
5982- Arc :: new( Int32Array :: from( vec![ 1 , 1 , 2 , 2 ] ) ) ,
5983- Arc :: new( Int64Array :: from( vec![ 20 , 20 , 20 , 20 ] ) ) ,
5984- Arc :: new( Int64Array :: from( vec![ 10 , 20 , 30 , 40 ] ) ) ,
6056+ Arc :: new( Int32Array :: from( vec![ 1 , 1 , 2 ] ) ) ,
6057+ Arc :: new( Int64Array :: from( vec![ 20 , 20 , 20 ] ) ) ,
6058+ Arc :: new( Int64Array :: from( vec![ 10 , 20 , 30 ] ) ) ,
59856059 ] ,
59866060 ) ?,
59876061 RecordBatch :: try_new(
59886062 Arc :: clone( & schema) ,
59896063 vec![
5990- Arc :: new( Int32Array :: from( vec![ 1 , 1 , 2 , 2 ] ) ) ,
5991- Arc :: new( Int64Array :: from( vec![ 0 , 0 , 0 , 0 ] ) ) ,
5992- Arc :: new( Int64Array :: from( vec![ 50 , 60 , 70 , 80 ] ) ) ,
6064+ Arc :: new( Int32Array :: from( vec![ 2 , 1 , 1 , 2 , 2 ] ) ) ,
6065+ Arc :: new( Int64Array :: from( vec![ 20 , 0 , 0 , 0 , 0 ] ) ) ,
6066+ Arc :: new( Int64Array :: from( vec![ 40 , 50 , 60 , 70 , 80 ] ) ) ,
59936067 ] ,
59946068 ) ?,
59956069 ] ;
@@ -6005,17 +6079,26 @@ mod tests {
60056079 . alias ( "SUM(value)" )
60066080 . build ( ) ?,
60076081 ) ;
6008- let input = TestMemoryExec :: try_new ( & [ input_batches] , Arc :: clone ( & schema) , None ) ?
6009- . try_with_grouping_information ( vec ! [ vec![ key, time_bin] ] ) ?;
6010- let input: Arc < dyn ExecutionPlan > = Arc :: new ( input) ;
6011- assert_eq ! ( input. output_partitioning( ) . partition_count( ) , 1 ) ;
6082+ let mut eq_properties = EquivalenceProperties :: new ( Arc :: clone ( & schema) ) ;
6083+ eq_properties. add_groupings ( [ vec ! [ key, time_bin] ] ) ;
6084+ let input = Arc :: new (
6085+ BarrierExec :: new ( vec ! [ input_batches] , Arc :: clone ( & schema) )
6086+ . without_start_barrier ( )
6087+ . with_batch_barrier ( )
6088+ . with_equivalence_properties ( eq_properties)
6089+ . with_log ( false ) ,
6090+ ) ;
6091+ assert_eq ! (
6092+ input. properties( ) . output_partitioning( ) . partition_count( ) ,
6093+ 1
6094+ ) ;
60126095
60136096 let aggregate = AggregateExec :: try_new (
60146097 AggregateMode :: Single ,
60156098 group_by,
60166099 vec ! [ aggr_expr] ,
60176100 vec ! [ None ] ,
6018- input,
6101+ Arc :: clone ( & input) as Arc < dyn ExecutionPlan > ,
60196102 schema,
60206103 ) ?;
60216104
@@ -6036,8 +6119,25 @@ mod tests {
60366119 let task_ctx = new_migrated_hash_ctx ( 1024 ) ;
60376120 let stream = aggregate. execute_typed ( 0 , & task_ctx) ?;
60386121 assert ! ( matches!( stream, StreamType :: ClusteredSingleAggregate ( _) ) ) ;
6039- let stream: SendableRecordBatchStream = stream. into ( ) ;
6040- let output = collect ( stream) . await ?;
6122+ let mut stream: SendableRecordBatchStream = stream. into ( ) ;
6123+ input. wait_batch ( ) . await ;
6124+ // The second batch and EOF remain blocked until the first completed
6125+ // group has been emitted.
6126+ let first =
6127+ tokio:: time:: timeout ( std:: time:: Duration :: from_secs ( 5 ) , stream. next ( ) )
6128+ . await
6129+ . expect ( "completed group was not emitted before the next input batch" )
6130+ . expect ( "stream ended before the next input batch" ) ?;
6131+ assert_snapshot ! ( batches_to_string( std:: slice:: from_ref( & first) ) , @r"
6132+ +-----+----------+------------+
6133+ | key | time_bin | SUM(value) |
6134+ +-----+----------+------------+
6135+ | 1 | 20 | 30 |
6136+ +-----+----------+------------+
6137+ " ) ;
6138+ input. wait_batch ( ) . await ;
6139+ let mut output = vec ! [ first] ;
6140+ output. extend ( collect ( stream) . await ?) ;
60416141 assert_snapshot ! ( batches_to_sort_string( & output) , @r"
60426142+-----+----------+------------+
60436143| key | time_bin | SUM(value) |
0 commit comments