Skip to content

Commit 0ee62ef

Browse files
committed
refactor poll_next for PerPartitionStream and add test
1 parent 9b7f8ce commit 0ee62ef

1 file changed

Lines changed: 75 additions & 46 deletions

File tree

  • datafusion/physical-plan/src/repartition

‎datafusion/physical-plan/src/repartition/mod.rs‎

Lines changed: 75 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1431,6 +1431,7 @@ struct PerPartitionStream {
14311431
/// Execution metrics
14321432
baseline_metrics: BaselineMetrics,
14331433

1434+
/// None for sort preserving variant (merge sort already does coalescing)
14341435
batch_coalescer: Option<LimitedBatchCoalescer>,
14351436
}
14361437

@@ -1541,6 +1542,46 @@ impl PerPartitionStream {
15411542
}
15421543
}
15431544
}
1545+
1546+
fn poll_next_and_coalesce(
1547+
self: &mut Pin<&mut Self>,
1548+
cx: &mut Context<'_>,
1549+
coalescer: &mut LimitedBatchCoalescer,
1550+
) -> Poll<Option<Result<RecordBatch>>> {
1551+
let cloned_time = self.baseline_metrics.elapsed_compute().clone();
1552+
let mut completed = false;
1553+
1554+
loop {
1555+
if let Some(batch) = coalescer.next_completed_batch() {
1556+
return Poll::Ready(Some(Ok(batch)));
1557+
}
1558+
if completed {
1559+
return Poll::Ready(None);
1560+
}
1561+
let inner_poll = self.poll_next_inner(cx);
1562+
let _timer = cloned_time.timer();
1563+
1564+
match inner_poll {
1565+
Poll::Pending => {
1566+
return Poll::Pending;
1567+
}
1568+
Poll::Ready(None) => {
1569+
completed = true;
1570+
if let Err(err) = coalescer.finish() {
1571+
return Poll::Ready(Some(Err(err)));
1572+
}
1573+
}
1574+
Poll::Ready(Some(Ok(batch))) => {
1575+
if let Err(err) = coalescer.push_batch(batch) {
1576+
return Poll::Ready(Some(Err(err)));
1577+
}
1578+
}
1579+
Poll::Ready(Some(err)) => {
1580+
return Poll::Ready(Some(err));
1581+
}
1582+
}
1583+
}
1584+
}
15441585
}
15451586

15461587
impl Stream for PerPartitionStream {
@@ -1550,52 +1591,13 @@ impl Stream for PerPartitionStream {
15501591
mut self: Pin<&mut Self>,
15511592
cx: &mut Context<'_>,
15521593
) -> Poll<Option<Self::Item>> {
1553-
let poll = match self.batch_coalescer.take() {
1554-
Some(mut coalescer) => {
1555-
let cloned_time = self.baseline_metrics.elapsed_compute().clone();
1556-
let mut completed = false;
1557-
let poll;
1558-
loop {
1559-
if let Some(batch) = coalescer.next_completed_batch() {
1560-
poll = Poll::Ready(Some(Ok(batch)));
1561-
break;
1562-
}
1563-
if completed {
1564-
poll = Poll::Ready(None);
1565-
break;
1566-
}
1567-
let inner_poll = self.poll_next_inner(cx);
1568-
let _timer = cloned_time.timer();
1569-
1570-
match inner_poll {
1571-
Poll::Pending => {
1572-
poll = Poll::Pending;
1573-
break;
1574-
}
1575-
Poll::Ready(None) => {
1576-
completed = true;
1577-
if let Err(err) = coalescer.finish() {
1578-
poll = Poll::Ready(Some(Err(err)));
1579-
break;
1580-
}
1581-
}
1582-
Poll::Ready(Some(Ok(batch))) => {
1583-
if let Err(err) = coalescer.push_batch(batch) {
1584-
poll = Poll::Ready(Some(Err(err)));
1585-
break;
1586-
}
1587-
}
1588-
Poll::Ready(Some(err)) => {
1589-
poll = Poll::Ready(Some(err));
1590-
break;
1591-
}
1592-
}
1593-
}
1594-
self.batch_coalescer = Some(coalescer);
1595-
poll
1596-
}
1597-
None => self.poll_next_inner(cx),
1598-
};
1594+
let poll;
1595+
if let Some(mut coalescer) = self.batch_coalescer.take() {
1596+
poll = self.poll_next_and_coalesce(cx, &mut coalescer);
1597+
self.batch_coalescer = Some(coalescer);
1598+
} else {
1599+
poll = self.poll_next_inner(cx);
1600+
}
15991601
self.baseline_metrics.record_poll(poll)
16001602
}
16011603
}
@@ -1630,6 +1632,7 @@ mod tests {
16301632
use datafusion_common::exec_err;
16311633
use datafusion_common::test_util::batches_to_sort_string;
16321634
use datafusion_common_runtime::JoinSet;
1635+
use datafusion_execution::config::SessionConfig;
16331636
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
16341637
use insta::assert_snapshot;
16351638

@@ -1719,6 +1722,32 @@ mod tests {
17191722
Ok(())
17201723
}
17211724

1725+
#[tokio::test]
1726+
async fn test_repartition_with_coalescing() -> Result<()> {
1727+
let schema = test_schema();
1728+
// create 50 batches, each having 8 rows
1729+
let partition = create_vec_batches(50);
1730+
let partitions = vec![partition.clone(), partition.clone()];
1731+
let partitioning = Partitioning::RoundRobinBatch(1);
1732+
1733+
let session_config = SessionConfig::new().with_batch_size(200);
1734+
let task_ctx = TaskContext::default().with_session_config(session_config);
1735+
let task_ctx = Arc::new(task_ctx);
1736+
1737+
// create physical plan
1738+
let exec = TestMemoryExec::try_new_exec(&partitions, Arc::clone(&schema), None)?;
1739+
let exec = RepartitionExec::try_new(exec, partitioning)?;
1740+
1741+
for i in 0..exec.partitioning().partition_count() {
1742+
let mut stream = exec.execute(i, Arc::clone(&task_ctx))?;
1743+
while let Some(result) = stream.next().await {
1744+
let batch = result?;
1745+
assert_eq!(200, batch.num_rows());
1746+
}
1747+
}
1748+
Ok(())
1749+
}
1750+
17221751
fn test_schema() -> Arc<Schema> {
17231752
Arc::new(Schema::new(vec![Field::new("c0", DataType::UInt32, false)]))
17241753
}

0 commit comments

Comments
 (0)