Skip to content

Commit cc5d4b8

Browse files
committed
fix: remove partition aware union logic
`UnionExec` shall be a simple node and not perform some optimizations on its own. If repartition / partition concatenation is desired, then there should be an optimizer pass for that. Fixes apache#5970.
1 parent 2c57310 commit cc5d4b8

3 files changed

Lines changed: 98 additions & 176 deletions

File tree

‎datafusion/core/src/dataframe.rs‎

Lines changed: 0 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1929,52 +1929,6 @@ mod tests {
19291929
Ok(())
19301930
}
19311931

1932-
#[tokio::test]
1933-
async fn partition_aware_union() -> Result<()> {
1934-
let left = test_table().await?.select_columns(&["c1", "c2"])?;
1935-
let right = test_table_with_name("c2")
1936-
.await?
1937-
.select_columns(&["c1", "c3"])?
1938-
.with_column_renamed("c2.c1", "c2_c1")?;
1939-
1940-
let left_rows = left.clone().collect().await?;
1941-
let right_rows = right.clone().collect().await?;
1942-
let join1 = left.clone().join(
1943-
right.clone(),
1944-
JoinType::Inner,
1945-
&["c1"],
1946-
&["c2_c1"],
1947-
None,
1948-
)?;
1949-
let join2 = left.join(right, JoinType::Inner, &["c1"], &["c2_c1"], None)?;
1950-
1951-
let union = join1.union(join2)?;
1952-
1953-
let union_rows = union.clone().collect().await?;
1954-
1955-
assert_eq!(100, left_rows.iter().map(|x| x.num_rows()).sum::<usize>());
1956-
assert_eq!(100, right_rows.iter().map(|x| x.num_rows()).sum::<usize>());
1957-
assert_eq!(4016, union_rows.iter().map(|x| x.num_rows()).sum::<usize>());
1958-
1959-
let physical_plan = union.create_physical_plan().await?;
1960-
let default_partition_count = SessionConfig::new().target_partitions();
1961-
1962-
// For partition aware union, the output partition count should not be changed.
1963-
assert_eq!(
1964-
physical_plan.output_partitioning().partition_count(),
1965-
default_partition_count
1966-
);
1967-
// For partition aware union, the output partition is the same with the union's inputs
1968-
for child in physical_plan.children() {
1969-
assert_eq!(
1970-
physical_plan.output_partitioning(),
1971-
child.output_partitioning()
1972-
);
1973-
}
1974-
1975-
Ok(())
1976-
}
1977-
19781932
#[tokio::test]
19791933
async fn non_partition_aware_union() -> Result<()> {
19801934
let left = test_table().await?.select_columns(&["c1", "c2"])?;

‎datafusion/core/src/physical_plan/union.rs‎

Lines changed: 17 additions & 130 deletions
Original file line numberDiff line numberDiff line change
@@ -21,16 +21,14 @@
2121

2222
//! The Union operator combines multiple inputs with the same schema
2323
24-
use std::pin::Pin;
25-
use std::task::{Context, Poll};
2624
use std::{any::Any, sync::Arc};
2725

2826
use arrow::{
2927
datatypes::{Field, Schema, SchemaRef},
3028
record_batch::RecordBatch,
3129
};
3230
use datafusion_common::{DFSchemaRef, DataFusionError};
33-
use futures::{Stream, StreamExt};
31+
use futures::StreamExt;
3432
use itertools::Itertools;
3533
use log::debug;
3634
use log::warn;
@@ -47,7 +45,6 @@ use crate::{
4745
error::Result,
4846
physical_plan::{expressions, metrics::BaselineMetrics},
4947
};
50-
use tokio::macros::support::thread_rng_n;
5148

5249
/// `UnionExec`: `UNION ALL` execution plan.
5350
///
@@ -94,8 +91,6 @@ pub struct UnionExec {
9491
metrics: ExecutionPlanMetricsSet,
9592
/// Schema of Union
9693
schema: SchemaRef,
97-
/// Partition aware Union
98-
partition_aware: bool,
9994
}
10095

10196
impl UnionExec {
@@ -154,24 +149,10 @@ impl UnionExec {
154149
inputs[0].schema().metadata().clone(),
155150
));
156151

157-
// If all the input partitions have the same Hash partition spec with the first_input_partition
158-
// The UnionExec is partition aware.
159-
//
160-
// It might be too strict here in the case that the input partition specs are compatible but not exactly the same.
161-
// For example one input partition has the partition spec Hash('a','b','c') and
162-
// other has the partition spec Hash('a'), It is safe to derive the out partition with the spec Hash('a','b','c').
163-
let first_input_partition = inputs[0].output_partitioning();
164-
let partition_aware = matches!(first_input_partition, Partitioning::Hash(_, _))
165-
&& inputs
166-
.iter()
167-
.map(|plan| plan.output_partitioning())
168-
.all(|partition| partition == first_input_partition);
169-
170152
UnionExec {
171153
inputs,
172154
metrics: ExecutionPlanMetricsSet::new(),
173155
schema,
174-
partition_aware,
175156
}
176157
}
177158

@@ -204,28 +185,20 @@ impl ExecutionPlan for UnionExec {
204185

205186
/// Output of the union is the combination of all output partitions of the inputs
206187
fn output_partitioning(&self) -> Partitioning {
207-
if self.partition_aware {
208-
self.inputs[0].output_partitioning()
209-
} else {
210-
// Output the combination of all output partitions of the inputs if the Union is not partition aware
211-
let num_partitions = self
212-
.inputs
213-
.iter()
214-
.map(|plan| plan.output_partitioning().partition_count())
215-
.sum();
188+
// Output the combination of all output partitions of the inputs if the Union is not partition aware
189+
let num_partitions = self
190+
.inputs
191+
.iter()
192+
.map(|plan| plan.output_partitioning().partition_count())
193+
.sum();
216194

217-
Partitioning::UnknownPartitioning(num_partitions)
218-
}
195+
Partitioning::UnknownPartitioning(num_partitions)
219196
}
220197

221198
fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> {
222-
// If the Union is partition aware, there is no output ordering.
223-
// Otherwise, the output ordering is the "meet" of its input orderings.
199+
// The output ordering is the "meet" of its input orderings.
224200
// The meet is the finest ordering that satisfied by all the input
225201
// orderings, see https://en.wikipedia.org/wiki/Join_and_meet.
226-
if self.partition_aware {
227-
return None;
228-
}
229202
get_meet_of_orderings(&self.inputs)
230203
}
231204

@@ -273,34 +246,15 @@ impl ExecutionPlan for UnionExec {
273246
let elapsed_compute = baseline_metrics.elapsed_compute().clone();
274247
let _timer = elapsed_compute.timer(); // record on drop
275248

276-
if self.partition_aware {
277-
let mut input_stream_vec = vec![];
278-
for input in self.inputs.iter() {
279-
if partition < input.output_partitioning().partition_count() {
280-
input_stream_vec.push(input.execute(partition, context.clone())?);
281-
} else {
282-
// Do not find a partition to execute
283-
break;
284-
}
285-
}
286-
if input_stream_vec.len() == self.inputs.len() {
287-
let stream = Box::pin(CombinedRecordBatchStream::new(
288-
self.schema(),
289-
input_stream_vec,
290-
));
249+
// find partition to execute
250+
for input in self.inputs.iter() {
251+
// Calculate whether partition belongs to the current partition
252+
if partition < input.output_partitioning().partition_count() {
253+
let stream = input.execute(partition, context)?;
254+
debug!("Found a Union partition to execute");
291255
return Ok(Box::pin(ObservedStream::new(stream, baseline_metrics)));
292-
}
293-
} else {
294-
// find partition to execute
295-
for input in self.inputs.iter() {
296-
// Calculate whether partition belongs to the current partition
297-
if partition < input.output_partitioning().partition_count() {
298-
let stream = input.execute(partition, context)?;
299-
debug!("Found a Union partition to execute");
300-
return Ok(Box::pin(ObservedStream::new(stream, baseline_metrics)));
301-
} else {
302-
partition -= input.output_partitioning().partition_count();
303-
}
256+
} else {
257+
partition -= input.output_partitioning().partition_count();
304258
}
305259
}
306260

@@ -340,73 +294,6 @@ impl ExecutionPlan for UnionExec {
340294
}
341295
}
342296

343-
/// CombinedRecordBatchStream can be used to combine a Vec of SendableRecordBatchStreams into one
344-
pub struct CombinedRecordBatchStream {
345-
/// Schema wrapped by Arc
346-
schema: SchemaRef,
347-
/// Stream entries
348-
entries: Vec<SendableRecordBatchStream>,
349-
}
350-
351-
impl CombinedRecordBatchStream {
352-
/// Create an CombinedRecordBatchStream
353-
pub fn new(schema: SchemaRef, entries: Vec<SendableRecordBatchStream>) -> Self {
354-
Self { schema, entries }
355-
}
356-
}
357-
358-
impl RecordBatchStream for CombinedRecordBatchStream {
359-
fn schema(&self) -> SchemaRef {
360-
self.schema.clone()
361-
}
362-
}
363-
364-
impl Stream for CombinedRecordBatchStream {
365-
type Item = Result<RecordBatch>;
366-
367-
fn poll_next(
368-
mut self: Pin<&mut Self>,
369-
cx: &mut Context<'_>,
370-
) -> Poll<Option<Self::Item>> {
371-
use Poll::*;
372-
373-
let start = thread_rng_n(self.entries.len() as u32) as usize;
374-
let mut idx = start;
375-
376-
for _ in 0..self.entries.len() {
377-
let stream = self.entries.get_mut(idx).unwrap();
378-
379-
match Pin::new(stream).poll_next(cx) {
380-
Ready(Some(val)) => return Ready(Some(val)),
381-
Ready(None) => {
382-
// Remove the entry
383-
self.entries.swap_remove(idx);
384-
385-
// Check if this was the last entry, if so the cursor needs
386-
// to wrap
387-
if idx == self.entries.len() {
388-
idx = 0;
389-
} else if idx < start && start <= self.entries.len() {
390-
// The stream being swapped into the current index has
391-
// already been polled, so skip it.
392-
idx = idx.wrapping_add(1) % self.entries.len();
393-
}
394-
}
395-
Pending => {
396-
idx = idx.wrapping_add(1) % self.entries.len();
397-
}
398-
}
399-
}
400-
401-
// If the map is empty, then the stream is complete.
402-
if self.entries.is_empty() {
403-
Ready(None)
404-
} else {
405-
Pending
406-
}
407-
}
408-
}
409-
410297
/// Stream wrapper that records `BaselineMetrics` for a particular
411298
/// partition
412299
struct ObservedStream {

‎datafusion/core/tests/sql/order.rs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,3 +175,84 @@ async fn sort_with_duplicate_sort_exprs() -> Result<()> {
175175

176176
Ok(())
177177
}
178+
179+
/// Minimal test case for https://github.com/apache/arrow-datafusion/issues/5970
180+
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
181+
async fn test_issue5970_mini() -> Result<()> {
182+
let config = SessionConfig::new()
183+
.with_target_partitions(2)
184+
.with_repartition_sorts(true);
185+
let ctx = SessionContext::with_config(config);
186+
let sql = "
187+
WITH
188+
m0(t) AS (
189+
VALUES (0), (1), (2)),
190+
191+
m1(t) AS (
192+
VALUES (0), (1)),
193+
194+
u AS (
195+
SELECT 0 as m, t FROM m0 GROUP BY 1, 2),
196+
197+
v AS (
198+
SELECT 1 as m, t FROM m1 GROUP BY 1, 2)
199+
SELECT * FROM u
200+
UNION ALL
201+
SELECT * FROM v
202+
ORDER BY 1, 2;
203+
";
204+
205+
// check phys. plan
206+
let dataframe = ctx.sql(sql).await.unwrap();
207+
let plan = dataframe.into_optimized_plan().unwrap();
208+
let plan = ctx.state().create_physical_plan(&plan).await.unwrap();
209+
let expected = vec![
210+
"SortPreservingMergeExec: [m@0 ASC NULLS LAST,t@1 ASC NULLS LAST]",
211+
" UnionExec",
212+
" SortExec: expr=[m@0 ASC NULLS LAST,t@1 ASC NULLS LAST]",
213+
" ProjectionExec: expr=[Int64(0)@0 as m, t@1 as t]",
214+
" AggregateExec: mode=FinalPartitioned, gby=[Int64(0)@0 as Int64(0), t@1 as t], aggr=[]",
215+
" CoalesceBatchesExec: target_batch_size=8192",
216+
" RepartitionExec: partitioning=Hash([Column { name: \"Int64(0)\", index: 0 }, Column { name: \"t\", index: 1 }], 2), input_partitions=2",
217+
" AggregateExec: mode=Partial, gby=[0 as Int64(0), t@0 as t], aggr=[]",
218+
" RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1",
219+
" ProjectionExec: expr=[column1@0 as t]",
220+
" ValuesExec",
221+
" SortExec: expr=[m@0 ASC NULLS LAST,t@1 ASC NULLS LAST]",
222+
" ProjectionExec: expr=[Int64(1)@0 as m, t@1 as t]",
223+
" AggregateExec: mode=FinalPartitioned, gby=[Int64(1)@0 as Int64(1), t@1 as t], aggr=[]",
224+
" CoalesceBatchesExec: target_batch_size=8192",
225+
" RepartitionExec: partitioning=Hash([Column { name: \"Int64(1)\", index: 0 }, Column { name: \"t\", index: 1 }], 2), input_partitions=2",
226+
" AggregateExec: mode=Partial, gby=[1 as Int64(1), t@0 as t], aggr=[]",
227+
" RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1",
228+
" ProjectionExec: expr=[column1@0 as t]",
229+
" ValuesExec",
230+
];
231+
let formatted = displayable(plan.as_ref()).indent().to_string();
232+
let actual: Vec<&str> = formatted.trim().lines().collect();
233+
assert_eq!(
234+
expected, actual,
235+
"\n\nexpected:\n\n{expected:#?}\nactual:\n\n{actual:#?}\n\n"
236+
);
237+
238+
// sometimes it "just works"
239+
for i in 0..10 {
240+
println!("run: {i}");
241+
let actual = execute_to_batches(&ctx, sql).await;
242+
243+
// in https://github.com/apache/arrow-datafusion/issues/5970 the order of the output was sometimes not right
244+
let expected = vec![
245+
"+---+---+",
246+
"| m | t |",
247+
"+---+---+",
248+
"| 0 | 0 |",
249+
"| 0 | 1 |",
250+
"| 0 | 2 |",
251+
"| 1 | 0 |",
252+
"| 1 | 1 |",
253+
"+---+---+",
254+
];
255+
assert_batches_eq!(expected, &actual);
256+
}
257+
Ok(())
258+
}

0 commit comments

Comments
 (0)