@@ -1431,6 +1431,7 @@ struct PerPartitionStream {
14311431 /// Execution metrics
14321432 baseline_metrics : BaselineMetrics ,
14331433
1434+ /// None for sort preserving variant (merge sort already does coalescing)
14341435 batch_coalescer : Option < LimitedBatchCoalescer > ,
14351436}
14361437
@@ -1541,6 +1542,46 @@ impl PerPartitionStream {
15411542 }
15421543 }
15431544 }
1545+
1546+ fn poll_next_and_coalesce (
1547+ self : & mut Pin < & mut Self > ,
1548+ cx : & mut Context < ' _ > ,
1549+ coalescer : & mut LimitedBatchCoalescer ,
1550+ ) -> Poll < Option < Result < RecordBatch > > > {
1551+ let cloned_time = self . baseline_metrics . elapsed_compute ( ) . clone ( ) ;
1552+ let mut completed = false ;
1553+
1554+ loop {
1555+ if let Some ( batch) = coalescer. next_completed_batch ( ) {
1556+ return Poll :: Ready ( Some ( Ok ( batch) ) ) ;
1557+ }
1558+ if completed {
1559+ return Poll :: Ready ( None ) ;
1560+ }
1561+ let inner_poll = self . poll_next_inner ( cx) ;
1562+ let _timer = cloned_time. timer ( ) ;
1563+
1564+ match inner_poll {
1565+ Poll :: Pending => {
1566+ return Poll :: Pending ;
1567+ }
1568+ Poll :: Ready ( None ) => {
1569+ completed = true ;
1570+ if let Err ( err) = coalescer. finish ( ) {
1571+ return Poll :: Ready ( Some ( Err ( err) ) ) ;
1572+ }
1573+ }
1574+ Poll :: Ready ( Some ( Ok ( batch) ) ) => {
1575+ if let Err ( err) = coalescer. push_batch ( batch) {
1576+ return Poll :: Ready ( Some ( Err ( err) ) ) ;
1577+ }
1578+ }
1579+ Poll :: Ready ( Some ( err) ) => {
1580+ return Poll :: Ready ( Some ( err) ) ;
1581+ }
1582+ }
1583+ }
1584+ }
15441585}
15451586
15461587impl Stream for PerPartitionStream {
@@ -1550,52 +1591,13 @@ impl Stream for PerPartitionStream {
15501591 mut self : Pin < & mut Self > ,
15511592 cx : & mut Context < ' _ > ,
15521593 ) -> Poll < Option < Self :: Item > > {
1553- let poll = match self . batch_coalescer . take ( ) {
1554- Some ( mut coalescer) => {
1555- let cloned_time = self . baseline_metrics . elapsed_compute ( ) . clone ( ) ;
1556- let mut completed = false ;
1557- let poll;
1558- loop {
1559- if let Some ( batch) = coalescer. next_completed_batch ( ) {
1560- poll = Poll :: Ready ( Some ( Ok ( batch) ) ) ;
1561- break ;
1562- }
1563- if completed {
1564- poll = Poll :: Ready ( None ) ;
1565- break ;
1566- }
1567- let inner_poll = self . poll_next_inner ( cx) ;
1568- let _timer = cloned_time. timer ( ) ;
1569-
1570- match inner_poll {
1571- Poll :: Pending => {
1572- poll = Poll :: Pending ;
1573- break ;
1574- }
1575- Poll :: Ready ( None ) => {
1576- completed = true ;
1577- if let Err ( err) = coalescer. finish ( ) {
1578- poll = Poll :: Ready ( Some ( Err ( err) ) ) ;
1579- break ;
1580- }
1581- }
1582- Poll :: Ready ( Some ( Ok ( batch) ) ) => {
1583- if let Err ( err) = coalescer. push_batch ( batch) {
1584- poll = Poll :: Ready ( Some ( Err ( err) ) ) ;
1585- break ;
1586- }
1587- }
1588- Poll :: Ready ( Some ( err) ) => {
1589- poll = Poll :: Ready ( Some ( err) ) ;
1590- break ;
1591- }
1592- }
1593- }
1594- self . batch_coalescer = Some ( coalescer) ;
1595- poll
1596- }
1597- None => self . poll_next_inner ( cx) ,
1598- } ;
1594+ let poll;
1595+ if let Some ( mut coalescer) = self . batch_coalescer . take ( ) {
1596+ poll = self . poll_next_and_coalesce ( cx, & mut coalescer) ;
1597+ self . batch_coalescer = Some ( coalescer) ;
1598+ } else {
1599+ poll = self . poll_next_inner ( cx) ;
1600+ }
15991601 self . baseline_metrics . record_poll ( poll)
16001602 }
16011603}
@@ -1630,6 +1632,7 @@ mod tests {
16301632 use datafusion_common:: exec_err;
16311633 use datafusion_common:: test_util:: batches_to_sort_string;
16321634 use datafusion_common_runtime:: JoinSet ;
1635+ use datafusion_execution:: config:: SessionConfig ;
16331636 use datafusion_execution:: runtime_env:: RuntimeEnvBuilder ;
16341637 use insta:: assert_snapshot;
16351638
@@ -1719,6 +1722,32 @@ mod tests {
17191722 Ok ( ( ) )
17201723 }
17211724
1725+ #[ tokio:: test]
1726+ async fn test_repartition_with_coalescing ( ) -> Result < ( ) > {
1727+ let schema = test_schema ( ) ;
1728+ // create 50 batches, each having 8 rows
1729+ let partition = create_vec_batches ( 50 ) ;
1730+ let partitions = vec ! [ partition. clone( ) , partition. clone( ) ] ;
1731+ let partitioning = Partitioning :: RoundRobinBatch ( 1 ) ;
1732+
1733+ let session_config = SessionConfig :: new ( ) . with_batch_size ( 200 ) ;
1734+ let task_ctx = TaskContext :: default ( ) . with_session_config ( session_config) ;
1735+ let task_ctx = Arc :: new ( task_ctx) ;
1736+
1737+ // create physical plan
1738+ let exec = TestMemoryExec :: try_new_exec ( & partitions, Arc :: clone ( & schema) , None ) ?;
1739+ let exec = RepartitionExec :: try_new ( exec, partitioning) ?;
1740+
1741+ for i in 0 ..exec. partitioning ( ) . partition_count ( ) {
1742+ let mut stream = exec. execute ( i, Arc :: clone ( & task_ctx) ) ?;
1743+ while let Some ( result) = stream. next ( ) . await {
1744+ let batch = result?;
1745+ assert_eq ! ( 200 , batch. num_rows( ) ) ;
1746+ }
1747+ }
1748+ Ok ( ( ) )
1749+ }
1750+
17221751 fn test_schema ( ) -> Arc < Schema > {
17231752 Arc :: new ( Schema :: new ( vec ! [ Field :: new( "c0" , DataType :: UInt32 , false ) ] ) )
17241753 }
0 commit comments