2121
2222//! The Union operator combines multiple inputs with the same schema
2323
24- use std:: pin:: Pin ;
25- use std:: task:: { Context , Poll } ;
2624use std:: { any:: Any , sync:: Arc } ;
2725
2826use arrow:: {
2927 datatypes:: { Field , Schema , SchemaRef } ,
3028 record_batch:: RecordBatch ,
3129} ;
3230use datafusion_common:: { DFSchemaRef , DataFusionError } ;
33- use futures:: { Stream , StreamExt } ;
31+ use futures:: StreamExt ;
3432use itertools:: Itertools ;
3533use log:: debug;
3634use log:: warn;
@@ -47,7 +45,6 @@ use crate::{
4745 error:: Result ,
4846 physical_plan:: { expressions, metrics:: BaselineMetrics } ,
4947} ;
50- use tokio:: macros:: support:: thread_rng_n;
5148
5249/// `UnionExec`: `UNION ALL` execution plan.
5350///
@@ -94,8 +91,6 @@ pub struct UnionExec {
9491 metrics : ExecutionPlanMetricsSet ,
9592 /// Schema of Union
9693 schema : SchemaRef ,
97- /// Partition aware Union
98- partition_aware : bool ,
9994}
10095
10196impl UnionExec {
@@ -154,24 +149,10 @@ impl UnionExec {
154149 inputs[ 0 ] . schema ( ) . metadata ( ) . clone ( ) ,
155150 ) ) ;
156151
157- // If all the input partitions have the same Hash partition spec with the first_input_partition
158- // The UnionExec is partition aware.
159- //
160- // It might be too strict here in the case that the input partition specs are compatible but not exactly the same.
161- // For example one input partition has the partition spec Hash('a','b','c') and
162- // other has the partition spec Hash('a'), It is safe to derive the out partition with the spec Hash('a','b','c').
163- let first_input_partition = inputs[ 0 ] . output_partitioning ( ) ;
164- let partition_aware = matches ! ( first_input_partition, Partitioning :: Hash ( _, _) )
165- && inputs
166- . iter ( )
167- . map ( |plan| plan. output_partitioning ( ) )
168- . all ( |partition| partition == first_input_partition) ;
169-
170152 UnionExec {
171153 inputs,
172154 metrics : ExecutionPlanMetricsSet :: new ( ) ,
173155 schema,
174- partition_aware,
175156 }
176157 }
177158
@@ -204,28 +185,20 @@ impl ExecutionPlan for UnionExec {
204185
205186 /// Output of the union is the combination of all output partitions of the inputs
206187 fn output_partitioning ( & self ) -> Partitioning {
207- if self . partition_aware {
208- self . inputs [ 0 ] . output_partitioning ( )
209- } else {
210- // Output the combination of all output partitions of the inputs if the Union is not partition aware
211- let num_partitions = self
212- . inputs
213- . iter ( )
214- . map ( |plan| plan. output_partitioning ( ) . partition_count ( ) )
215- . sum ( ) ;
188+ // Output the combination of all output partitions of the inputs if the Union is not partition aware
189+ let num_partitions = self
190+ . inputs
191+ . iter ( )
192+ . map ( |plan| plan. output_partitioning ( ) . partition_count ( ) )
193+ . sum ( ) ;
216194
217- Partitioning :: UnknownPartitioning ( num_partitions)
218- }
195+ Partitioning :: UnknownPartitioning ( num_partitions)
219196 }
220197
221198 fn output_ordering ( & self ) -> Option < & [ PhysicalSortExpr ] > {
222- // If the Union is partition aware, there is no output ordering.
223- // Otherwise, the output ordering is the "meet" of its input orderings.
199+ // The output ordering is the "meet" of its input orderings.
224200 // The meet is the finest ordering that satisfied by all the input
225201 // orderings, see https://en.wikipedia.org/wiki/Join_and_meet.
226- if self . partition_aware {
227- return None ;
228- }
229202 get_meet_of_orderings ( & self . inputs )
230203 }
231204
@@ -273,34 +246,15 @@ impl ExecutionPlan for UnionExec {
273246 let elapsed_compute = baseline_metrics. elapsed_compute ( ) . clone ( ) ;
274247 let _timer = elapsed_compute. timer ( ) ; // record on drop
275248
276- if self . partition_aware {
277- let mut input_stream_vec = vec ! [ ] ;
278- for input in self . inputs . iter ( ) {
279- if partition < input. output_partitioning ( ) . partition_count ( ) {
280- input_stream_vec. push ( input. execute ( partition, context. clone ( ) ) ?) ;
281- } else {
282- // Do not find a partition to execute
283- break ;
284- }
285- }
286- if input_stream_vec. len ( ) == self . inputs . len ( ) {
287- let stream = Box :: pin ( CombinedRecordBatchStream :: new (
288- self . schema ( ) ,
289- input_stream_vec,
290- ) ) ;
249+ // find partition to execute
250+ for input in self . inputs . iter ( ) {
251+ // Calculate whether partition belongs to the current partition
252+ if partition < input. output_partitioning ( ) . partition_count ( ) {
253+ let stream = input. execute ( partition, context) ?;
254+ debug ! ( "Found a Union partition to execute" ) ;
291255 return Ok ( Box :: pin ( ObservedStream :: new ( stream, baseline_metrics) ) ) ;
292- }
293- } else {
294- // find partition to execute
295- for input in self . inputs . iter ( ) {
296- // Calculate whether partition belongs to the current partition
297- if partition < input. output_partitioning ( ) . partition_count ( ) {
298- let stream = input. execute ( partition, context) ?;
299- debug ! ( "Found a Union partition to execute" ) ;
300- return Ok ( Box :: pin ( ObservedStream :: new ( stream, baseline_metrics) ) ) ;
301- } else {
302- partition -= input. output_partitioning ( ) . partition_count ( ) ;
303- }
256+ } else {
257+ partition -= input. output_partitioning ( ) . partition_count ( ) ;
304258 }
305259 }
306260
@@ -340,73 +294,6 @@ impl ExecutionPlan for UnionExec {
340294 }
341295}
342296
343- /// CombinedRecordBatchStream can be used to combine a Vec of SendableRecordBatchStreams into one
344- pub struct CombinedRecordBatchStream {
345- /// Schema wrapped by Arc
346- schema : SchemaRef ,
347- /// Stream entries
348- entries : Vec < SendableRecordBatchStream > ,
349- }
350-
351- impl CombinedRecordBatchStream {
352- /// Create an CombinedRecordBatchStream
353- pub fn new ( schema : SchemaRef , entries : Vec < SendableRecordBatchStream > ) -> Self {
354- Self { schema, entries }
355- }
356- }
357-
358- impl RecordBatchStream for CombinedRecordBatchStream {
359- fn schema ( & self ) -> SchemaRef {
360- self . schema . clone ( )
361- }
362- }
363-
364- impl Stream for CombinedRecordBatchStream {
365- type Item = Result < RecordBatch > ;
366-
367- fn poll_next (
368- mut self : Pin < & mut Self > ,
369- cx : & mut Context < ' _ > ,
370- ) -> Poll < Option < Self :: Item > > {
371- use Poll :: * ;
372-
373- let start = thread_rng_n ( self . entries . len ( ) as u32 ) as usize ;
374- let mut idx = start;
375-
376- for _ in 0 ..self . entries . len ( ) {
377- let stream = self . entries . get_mut ( idx) . unwrap ( ) ;
378-
379- match Pin :: new ( stream) . poll_next ( cx) {
380- Ready ( Some ( val) ) => return Ready ( Some ( val) ) ,
381- Ready ( None ) => {
382- // Remove the entry
383- self . entries . swap_remove ( idx) ;
384-
385- // Check if this was the last entry, if so the cursor needs
386- // to wrap
387- if idx == self . entries . len ( ) {
388- idx = 0 ;
389- } else if idx < start && start <= self . entries . len ( ) {
390- // The stream being swapped into the current index has
391- // already been polled, so skip it.
392- idx = idx. wrapping_add ( 1 ) % self . entries . len ( ) ;
393- }
394- }
395- Pending => {
396- idx = idx. wrapping_add ( 1 ) % self . entries . len ( ) ;
397- }
398- }
399- }
400-
401- // If the map is empty, then the stream is complete.
402- if self . entries . is_empty ( ) {
403- Ready ( None )
404- } else {
405- Pending
406- }
407- }
408- }
409-
410297/// Stream wrapper that records `BaselineMetrics` for a particular
411298/// partition
412299struct ObservedStream {
0 commit comments