@@ -30,7 +30,6 @@ use super::metrics::{self, ExecutionPlanMetricsSet, MetricBuilder, MetricsSet};
3030use super :: {
3131 DisplayAs , ExecutionPlanProperties , RecordBatchStream , SendableRecordBatchStream ,
3232} ;
33- use crate :: coalesce:: LimitedBatchCoalescer ;
3433use crate :: execution_plan:: { CardinalityEffect , EvaluationType , SchedulingType } ;
3534use crate :: hash_utils:: create_hashes;
3635use crate :: metrics:: { BaselineMetrics , SpillMetrics } ;
@@ -999,7 +998,6 @@ impl ExecutionPlan for RepartitionExec {
999998 spill_stream,
1000999 1 , // Each receiver handles one input partition
10011000 BaselineMetrics :: new ( & metrics, partition) ,
1002- None , // subsequent merge sort already does batching https://github.com/apache/datafusion/blob/e4dcf0c85611ad0bd291f03a8e03fe56d773eb16/datafusion/physical-plan/src/sorts/merge.rs#L286
10031001 ) ) as SendableRecordBatchStream
10041002 } )
10051003 . collect :: < Vec < _ > > ( ) ;
@@ -1038,7 +1036,6 @@ impl ExecutionPlan for RepartitionExec {
10381036 spill_stream,
10391037 num_input_partitions,
10401038 BaselineMetrics :: new ( & metrics, partition) ,
1041- Some ( context. session_config ( ) . batch_size ( ) ) ,
10421039 ) ) as SendableRecordBatchStream )
10431040 }
10441041 } )
@@ -1531,13 +1528,9 @@ struct PerPartitionStream {
15311528
15321529 /// Execution metrics
15331530 baseline_metrics : BaselineMetrics ,
1534-
1535- /// None for sort preserving variant (merge sort already does coalescing)
1536- batch_coalescer : Option < LimitedBatchCoalescer > ,
15371531}
15381532
15391533impl PerPartitionStream {
1540- #[ expect( clippy:: too_many_arguments) ]
15411534 fn new (
15421535 schema : SchemaRef ,
15431536 receiver : DistributionReceiver < MaybeBatch > ,
@@ -1546,10 +1539,7 @@ impl PerPartitionStream {
15461539 spill_stream : SendableRecordBatchStream ,
15471540 num_input_partitions : usize ,
15481541 baseline_metrics : BaselineMetrics ,
1549- batch_size : Option < usize > ,
15501542 ) -> Self {
1551- let batch_coalescer =
1552- batch_size. map ( |s| LimitedBatchCoalescer :: new ( Arc :: clone ( & schema) , s, None ) ) ;
15531543 Self {
15541544 schema,
15551545 receiver,
@@ -1559,7 +1549,6 @@ impl PerPartitionStream {
15591549 state : StreamState :: ReadingMemory ,
15601550 remaining_partitions : num_input_partitions,
15611551 baseline_metrics,
1562- batch_coalescer,
15631552 }
15641553 }
15651554
@@ -1643,43 +1632,6 @@ impl PerPartitionStream {
16431632 }
16441633 }
16451634 }
1646-
1647- fn poll_next_and_coalesce (
1648- self : & mut Pin < & mut Self > ,
1649- cx : & mut Context < ' _ > ,
1650- coalescer : & mut LimitedBatchCoalescer ,
1651- ) -> Poll < Option < Result < RecordBatch > > > {
1652- let cloned_time = self . baseline_metrics . elapsed_compute ( ) . clone ( ) ;
1653- let mut completed = false ;
1654-
1655- loop {
1656- if let Some ( batch) = coalescer. next_completed_batch ( ) {
1657- return Poll :: Ready ( Some ( Ok ( batch) ) ) ;
1658- }
1659- if completed {
1660- return Poll :: Ready ( None ) ;
1661- }
1662-
1663- match ready ! ( self . poll_next_inner( cx) ) {
1664- Some ( Ok ( batch) ) => {
1665- let _timer = cloned_time. timer ( ) ;
1666- if let Err ( err) = coalescer. push_batch ( batch) {
1667- return Poll :: Ready ( Some ( Err ( err) ) ) ;
1668- }
1669- }
1670- Some ( err) => {
1671- return Poll :: Ready ( Some ( err) ) ;
1672- }
1673- None => {
1674- completed = true ;
1675- let _timer = cloned_time. timer ( ) ;
1676- if let Err ( err) = coalescer. finish ( ) {
1677- return Poll :: Ready ( Some ( Err ( err) ) ) ;
1678- }
1679- }
1680- }
1681- }
1682- }
16831635}
16841636
16851637impl Stream for PerPartitionStream {
@@ -1689,13 +1641,7 @@ impl Stream for PerPartitionStream {
16891641 mut self : Pin < & mut Self > ,
16901642 cx : & mut Context < ' _ > ,
16911643 ) -> Poll < Option < Self :: Item > > {
1692- let poll;
1693- if let Some ( mut coalescer) = self . batch_coalescer . take ( ) {
1694- poll = self . poll_next_and_coalesce ( cx, & mut coalescer) ;
1695- self . batch_coalescer = Some ( coalescer) ;
1696- } else {
1697- poll = self . poll_next_inner ( cx) ;
1698- }
1644+ let poll = self . poll_next_inner ( cx) ;
16991645 self . baseline_metrics . record_poll ( poll)
17001646 }
17011647}
@@ -1730,9 +1676,9 @@ mod tests {
17301676 use datafusion_common:: exec_err;
17311677 use datafusion_common:: test_util:: batches_to_sort_string;
17321678 use datafusion_common_runtime:: JoinSet ;
1733- use datafusion_execution:: config:: SessionConfig ;
17341679 use datafusion_execution:: runtime_env:: RuntimeEnvBuilder ;
17351680 use insta:: assert_snapshot;
1681+ use itertools:: Itertools ;
17361682
17371683 #[ tokio:: test]
17381684 async fn one_to_many_round_robin ( ) -> Result < ( ) > {
@@ -1746,13 +1692,10 @@ mod tests {
17461692 repartition ( & schema, partitions, Partitioning :: RoundRobinBatch ( 4 ) ) . await ?;
17471693
17481694 assert_eq ! ( 4 , output_partitions. len( ) ) ;
1749- for partition in & output_partitions {
1750- assert_eq ! ( 1 , partition. len( ) ) ;
1751- }
1752- assert_eq ! ( 13 * 8 , output_partitions[ 0 ] [ 0 ] . num_rows( ) ) ;
1753- assert_eq ! ( 13 * 8 , output_partitions[ 1 ] [ 0 ] . num_rows( ) ) ;
1754- assert_eq ! ( 12 * 8 , output_partitions[ 2 ] [ 0 ] . num_rows( ) ) ;
1755- assert_eq ! ( 12 * 8 , output_partitions[ 3 ] [ 0 ] . num_rows( ) ) ;
1695+ assert_eq ! ( 13 , output_partitions[ 0 ] . len( ) ) ;
1696+ assert_eq ! ( 13 , output_partitions[ 1 ] . len( ) ) ;
1697+ assert_eq ! ( 12 , output_partitions[ 2 ] . len( ) ) ;
1698+ assert_eq ! ( 12 , output_partitions[ 3 ] . len( ) ) ;
17561699
17571700 Ok ( ( ) )
17581701 }
@@ -1769,7 +1712,7 @@ mod tests {
17691712 repartition ( & schema, partitions, Partitioning :: RoundRobinBatch ( 1 ) ) . await ?;
17701713
17711714 assert_eq ! ( 1 , output_partitions. len( ) ) ;
1772- assert_eq ! ( 150 * 8 , output_partitions[ 0 ] [ 0 ] . num_rows ( ) ) ;
1715+ assert_eq ! ( 150 , output_partitions[ 0 ] . len ( ) ) ;
17731716
17741717 Ok ( ( ) )
17751718 }
@@ -1785,12 +1728,12 @@ mod tests {
17851728 let output_partitions =
17861729 repartition ( & schema, partitions, Partitioning :: RoundRobinBatch ( 5 ) ) . await ?;
17871730
1788- let total_rows_per_partition = 8 * 50 * 3 / 5 ;
17891731 assert_eq ! ( 5 , output_partitions. len( ) ) ;
1790- for partition in output_partitions {
1791- assert_eq ! ( 1 , partition. len( ) ) ;
1792- assert_eq ! ( total_rows_per_partition, partition[ 0 ] . num_rows( ) ) ;
1793- }
1732+ assert_eq ! ( 30 , output_partitions[ 0 ] . len( ) ) ;
1733+ assert_eq ! ( 30 , output_partitions[ 1 ] . len( ) ) ;
1734+ assert_eq ! ( 30 , output_partitions[ 2 ] . len( ) ) ;
1735+ assert_eq ! ( 30 , output_partitions[ 3 ] . len( ) ) ;
1736+ assert_eq ! ( 30 , output_partitions[ 4 ] . len( ) ) ;
17941737
17951738 Ok ( ( ) )
17961739 }
@@ -1820,32 +1763,6 @@ mod tests {
18201763 Ok ( ( ) )
18211764 }
18221765
1823- #[ tokio:: test]
1824- async fn test_repartition_with_coalescing ( ) -> Result < ( ) > {
1825- let schema = test_schema ( ) ;
1826- // create 50 batches, each having 8 rows
1827- let partition = create_vec_batches ( 50 ) ;
1828- let partitions = vec ! [ partition. clone( ) , partition. clone( ) ] ;
1829- let partitioning = Partitioning :: RoundRobinBatch ( 1 ) ;
1830-
1831- let session_config = SessionConfig :: new ( ) . with_batch_size ( 200 ) ;
1832- let task_ctx = TaskContext :: default ( ) . with_session_config ( session_config) ;
1833- let task_ctx = Arc :: new ( task_ctx) ;
1834-
1835- // create physical plan
1836- let exec = TestMemoryExec :: try_new_exec ( & partitions, Arc :: clone ( & schema) , None ) ?;
1837- let exec = RepartitionExec :: try_new ( exec, partitioning) ?;
1838-
1839- for i in 0 ..exec. partitioning ( ) . partition_count ( ) {
1840- let mut stream = exec. execute ( i, Arc :: clone ( & task_ctx) ) ?;
1841- while let Some ( result) = stream. next ( ) . await {
1842- let batch = result?;
1843- assert_eq ! ( 200 , batch. num_rows( ) ) ;
1844- }
1845- }
1846- Ok ( ( ) )
1847- }
1848-
18491766 fn test_schema ( ) -> Arc < Schema > {
18501767 Arc :: new ( Schema :: new ( vec ! [ Field :: new( "c0" , DataType :: UInt32 , false ) ] ) )
18511768 }
@@ -1891,12 +1808,12 @@ mod tests {
18911808
18921809 let output_partitions = handle. join ( ) . await . unwrap ( ) . unwrap ( ) ;
18931810
1894- let total_rows_per_partition = 8 * 50 * 3 / 5 ;
18951811 assert_eq ! ( 5 , output_partitions. len( ) ) ;
1896- for partition in output_partitions {
1897- assert_eq ! ( 1 , partition. len( ) ) ;
1898- assert_eq ! ( total_rows_per_partition, partition[ 0 ] . num_rows( ) ) ;
1899- }
1812+ assert_eq ! ( 30 , output_partitions[ 0 ] . len( ) ) ;
1813+ assert_eq ! ( 30 , output_partitions[ 1 ] . len( ) ) ;
1814+ assert_eq ! ( 30 , output_partitions[ 2 ] . len( ) ) ;
1815+ assert_eq ! ( 30 , output_partitions[ 3 ] . len( ) ) ;
1816+ assert_eq ! ( 30 , output_partitions[ 4 ] . len( ) ) ;
19001817
19011818 Ok ( ( ) )
19021819 }
@@ -2134,13 +2051,14 @@ mod tests {
21342051 } ) ;
21352052 let batches_with_drop = crate :: common:: collect ( output_stream1) . await . unwrap ( ) ;
21362053
2137- let items_vec_with_drop = str_batches_to_vec ( & batches_with_drop) ;
2138- let items_set_with_drop: HashSet < & str > =
2139- items_vec_with_drop. iter ( ) . copied ( ) . collect ( ) ;
2140- assert_eq ! (
2141- items_set_with_drop. symmetric_difference( & items_set) . count( ) ,
2142- 0
2143- ) ;
2054+ fn sort ( batch : Vec < RecordBatch > ) -> Vec < RecordBatch > {
2055+ batch
2056+ . into_iter ( )
2057+ . sorted_by_key ( |b| format ! ( "{b:?}" ) )
2058+ . collect ( )
2059+ }
2060+
2061+ assert_eq ! ( sort( batches_without_drop) , sort( batches_with_drop) ) ;
21442062 }
21452063
21462064 fn str_batches_to_vec ( batches : & [ RecordBatch ] ) -> Vec < & str > {
0 commit comments