Skip to content

Commit 3532755

Browse files
committed
Revert "feat: integrate batch coalescer with repartition exec (apache#19002)"
This reverts commit 20870c1. # Conflicts: # datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs # datafusion/physical-plan/src/repartition/mod.rs
1 parent 77306e9 commit 3532755

2 files changed

Lines changed: 26 additions & 118 deletions

File tree

‎datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -134,19 +134,9 @@ impl ExecutionPlan for CustomPlan {
134134
_partition: usize,
135135
_context: Arc<TaskContext>,
136136
) -> Result<SendableRecordBatchStream> {
137-
let schema_captured = self.schema().clone();
138137
Ok(Box::pin(RecordBatchStreamAdapter::new(
139138
self.schema(),
140-
futures::stream::iter(self.batches.clone().into_iter().map(move |batch| {
141-
let projection: Vec<usize> = schema_captured
142-
.fields()
143-
.iter()
144-
.filter_map(|field| batch.schema().index_of(field.name()).ok())
145-
.collect();
146-
batch
147-
.project(&projection)
148-
.map_err(|e| DataFusionError::ArrowError(Box::new(e), None))
149-
})),
139+
futures::stream::iter(self.batches.clone().into_iter().map(Ok)),
150140
)))
151141
}
152142

‎datafusion/physical-plan/src/repartition/mod.rs‎

Lines changed: 25 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ use super::metrics::{self, ExecutionPlanMetricsSet, MetricBuilder, MetricsSet};
3030
use super::{
3131
DisplayAs, ExecutionPlanProperties, RecordBatchStream, SendableRecordBatchStream,
3232
};
33-
use crate::coalesce::LimitedBatchCoalescer;
3433
use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType};
3534
use crate::hash_utils::create_hashes;
3635
use crate::metrics::{BaselineMetrics, SpillMetrics};
@@ -999,7 +998,6 @@ impl ExecutionPlan for RepartitionExec {
999998
spill_stream,
1000999
1, // Each receiver handles one input partition
10011000
BaselineMetrics::new(&metrics, partition),
1002-
None, // subsequent merge sort already does batching https://github.com/apache/datafusion/blob/e4dcf0c85611ad0bd291f03a8e03fe56d773eb16/datafusion/physical-plan/src/sorts/merge.rs#L286
10031001
)) as SendableRecordBatchStream
10041002
})
10051003
.collect::<Vec<_>>();
@@ -1038,7 +1036,6 @@ impl ExecutionPlan for RepartitionExec {
10381036
spill_stream,
10391037
num_input_partitions,
10401038
BaselineMetrics::new(&metrics, partition),
1041-
Some(context.session_config().batch_size()),
10421039
)) as SendableRecordBatchStream)
10431040
}
10441041
})
@@ -1531,13 +1528,9 @@ struct PerPartitionStream {
15311528

15321529
/// Execution metrics
15331530
baseline_metrics: BaselineMetrics,
1534-
1535-
/// None for sort preserving variant (merge sort already does coalescing)
1536-
batch_coalescer: Option<LimitedBatchCoalescer>,
15371531
}
15381532

15391533
impl PerPartitionStream {
1540-
#[expect(clippy::too_many_arguments)]
15411534
fn new(
15421535
schema: SchemaRef,
15431536
receiver: DistributionReceiver<MaybeBatch>,
@@ -1546,10 +1539,7 @@ impl PerPartitionStream {
15461539
spill_stream: SendableRecordBatchStream,
15471540
num_input_partitions: usize,
15481541
baseline_metrics: BaselineMetrics,
1549-
batch_size: Option<usize>,
15501542
) -> Self {
1551-
let batch_coalescer =
1552-
batch_size.map(|s| LimitedBatchCoalescer::new(Arc::clone(&schema), s, None));
15531543
Self {
15541544
schema,
15551545
receiver,
@@ -1559,7 +1549,6 @@ impl PerPartitionStream {
15591549
state: StreamState::ReadingMemory,
15601550
remaining_partitions: num_input_partitions,
15611551
baseline_metrics,
1562-
batch_coalescer,
15631552
}
15641553
}
15651554

@@ -1643,43 +1632,6 @@ impl PerPartitionStream {
16431632
}
16441633
}
16451634
}
1646-
1647-
fn poll_next_and_coalesce(
1648-
self: &mut Pin<&mut Self>,
1649-
cx: &mut Context<'_>,
1650-
coalescer: &mut LimitedBatchCoalescer,
1651-
) -> Poll<Option<Result<RecordBatch>>> {
1652-
let cloned_time = self.baseline_metrics.elapsed_compute().clone();
1653-
let mut completed = false;
1654-
1655-
loop {
1656-
if let Some(batch) = coalescer.next_completed_batch() {
1657-
return Poll::Ready(Some(Ok(batch)));
1658-
}
1659-
if completed {
1660-
return Poll::Ready(None);
1661-
}
1662-
1663-
match ready!(self.poll_next_inner(cx)) {
1664-
Some(Ok(batch)) => {
1665-
let _timer = cloned_time.timer();
1666-
if let Err(err) = coalescer.push_batch(batch) {
1667-
return Poll::Ready(Some(Err(err)));
1668-
}
1669-
}
1670-
Some(err) => {
1671-
return Poll::Ready(Some(err));
1672-
}
1673-
None => {
1674-
completed = true;
1675-
let _timer = cloned_time.timer();
1676-
if let Err(err) = coalescer.finish() {
1677-
return Poll::Ready(Some(Err(err)));
1678-
}
1679-
}
1680-
}
1681-
}
1682-
}
16831635
}
16841636

16851637
impl Stream for PerPartitionStream {
@@ -1689,13 +1641,7 @@ impl Stream for PerPartitionStream {
16891641
mut self: Pin<&mut Self>,
16901642
cx: &mut Context<'_>,
16911643
) -> Poll<Option<Self::Item>> {
1692-
let poll;
1693-
if let Some(mut coalescer) = self.batch_coalescer.take() {
1694-
poll = self.poll_next_and_coalesce(cx, &mut coalescer);
1695-
self.batch_coalescer = Some(coalescer);
1696-
} else {
1697-
poll = self.poll_next_inner(cx);
1698-
}
1644+
let poll = self.poll_next_inner(cx);
16991645
self.baseline_metrics.record_poll(poll)
17001646
}
17011647
}
@@ -1730,9 +1676,9 @@ mod tests {
17301676
use datafusion_common::exec_err;
17311677
use datafusion_common::test_util::batches_to_sort_string;
17321678
use datafusion_common_runtime::JoinSet;
1733-
use datafusion_execution::config::SessionConfig;
17341679
use datafusion_execution::runtime_env::RuntimeEnvBuilder;
17351680
use insta::assert_snapshot;
1681+
use itertools::Itertools;
17361682

17371683
#[tokio::test]
17381684
async fn one_to_many_round_robin() -> Result<()> {
@@ -1746,13 +1692,10 @@ mod tests {
17461692
repartition(&schema, partitions, Partitioning::RoundRobinBatch(4)).await?;
17471693

17481694
assert_eq!(4, output_partitions.len());
1749-
for partition in &output_partitions {
1750-
assert_eq!(1, partition.len());
1751-
}
1752-
assert_eq!(13 * 8, output_partitions[0][0].num_rows());
1753-
assert_eq!(13 * 8, output_partitions[1][0].num_rows());
1754-
assert_eq!(12 * 8, output_partitions[2][0].num_rows());
1755-
assert_eq!(12 * 8, output_partitions[3][0].num_rows());
1695+
assert_eq!(13, output_partitions[0].len());
1696+
assert_eq!(13, output_partitions[1].len());
1697+
assert_eq!(12, output_partitions[2].len());
1698+
assert_eq!(12, output_partitions[3].len());
17561699

17571700
Ok(())
17581701
}
@@ -1769,7 +1712,7 @@ mod tests {
17691712
repartition(&schema, partitions, Partitioning::RoundRobinBatch(1)).await?;
17701713

17711714
assert_eq!(1, output_partitions.len());
1772-
assert_eq!(150 * 8, output_partitions[0][0].num_rows());
1715+
assert_eq!(150, output_partitions[0].len());
17731716

17741717
Ok(())
17751718
}
@@ -1785,12 +1728,12 @@ mod tests {
17851728
let output_partitions =
17861729
repartition(&schema, partitions, Partitioning::RoundRobinBatch(5)).await?;
17871730

1788-
let total_rows_per_partition = 8 * 50 * 3 / 5;
17891731
assert_eq!(5, output_partitions.len());
1790-
for partition in output_partitions {
1791-
assert_eq!(1, partition.len());
1792-
assert_eq!(total_rows_per_partition, partition[0].num_rows());
1793-
}
1732+
assert_eq!(30, output_partitions[0].len());
1733+
assert_eq!(30, output_partitions[1].len());
1734+
assert_eq!(30, output_partitions[2].len());
1735+
assert_eq!(30, output_partitions[3].len());
1736+
assert_eq!(30, output_partitions[4].len());
17941737

17951738
Ok(())
17961739
}
@@ -1820,32 +1763,6 @@ mod tests {
18201763
Ok(())
18211764
}
18221765

1823-
#[tokio::test]
1824-
async fn test_repartition_with_coalescing() -> Result<()> {
1825-
let schema = test_schema();
1826-
// create 50 batches, each having 8 rows
1827-
let partition = create_vec_batches(50);
1828-
let partitions = vec![partition.clone(), partition.clone()];
1829-
let partitioning = Partitioning::RoundRobinBatch(1);
1830-
1831-
let session_config = SessionConfig::new().with_batch_size(200);
1832-
let task_ctx = TaskContext::default().with_session_config(session_config);
1833-
let task_ctx = Arc::new(task_ctx);
1834-
1835-
// create physical plan
1836-
let exec = TestMemoryExec::try_new_exec(&partitions, Arc::clone(&schema), None)?;
1837-
let exec = RepartitionExec::try_new(exec, partitioning)?;
1838-
1839-
for i in 0..exec.partitioning().partition_count() {
1840-
let mut stream = exec.execute(i, Arc::clone(&task_ctx))?;
1841-
while let Some(result) = stream.next().await {
1842-
let batch = result?;
1843-
assert_eq!(200, batch.num_rows());
1844-
}
1845-
}
1846-
Ok(())
1847-
}
1848-
18491766
fn test_schema() -> Arc<Schema> {
18501767
Arc::new(Schema::new(vec![Field::new("c0", DataType::UInt32, false)]))
18511768
}
@@ -1891,12 +1808,12 @@ mod tests {
18911808

18921809
let output_partitions = handle.join().await.unwrap().unwrap();
18931810

1894-
let total_rows_per_partition = 8 * 50 * 3 / 5;
18951811
assert_eq!(5, output_partitions.len());
1896-
for partition in output_partitions {
1897-
assert_eq!(1, partition.len());
1898-
assert_eq!(total_rows_per_partition, partition[0].num_rows());
1899-
}
1812+
assert_eq!(30, output_partitions[0].len());
1813+
assert_eq!(30, output_partitions[1].len());
1814+
assert_eq!(30, output_partitions[2].len());
1815+
assert_eq!(30, output_partitions[3].len());
1816+
assert_eq!(30, output_partitions[4].len());
19001817

19011818
Ok(())
19021819
}
@@ -2134,13 +2051,14 @@ mod tests {
21342051
});
21352052
let batches_with_drop = crate::common::collect(output_stream1).await.unwrap();
21362053

2137-
let items_vec_with_drop = str_batches_to_vec(&batches_with_drop);
2138-
let items_set_with_drop: HashSet<&str> =
2139-
items_vec_with_drop.iter().copied().collect();
2140-
assert_eq!(
2141-
items_set_with_drop.symmetric_difference(&items_set).count(),
2142-
0
2143-
);
2054+
fn sort(batch: Vec<RecordBatch>) -> Vec<RecordBatch> {
2055+
batch
2056+
.into_iter()
2057+
.sorted_by_key(|b| format!("{b:?}"))
2058+
.collect()
2059+
}
2060+
2061+
assert_eq!(sort(batches_without_drop), sort(batches_with_drop));
21442062
}
21452063

21462064
fn str_batches_to_vec(batches: &[RecordBatch]) -> Vec<&str> {

0 commit comments

Comments
 (0)