@@ -842,6 +842,8 @@ mod tests {
842842 use super :: * ;
843843 use crate :: execution:: options:: { CsvReadOptions , ParquetReadOptions } ;
844844 use crate :: physical_plan:: ColumnarValue ;
845+ use crate :: physical_plan:: Partitioning ;
846+ use crate :: physical_plan:: PhysicalExpr ;
845847 use crate :: test_util;
846848 use crate :: test_util:: parquet_test_data;
847849 use crate :: { assert_batches_sorted_eq, execution:: context:: SessionContext } ;
@@ -851,6 +853,7 @@ mod tests {
851853 avg, cast, count, count_distinct, create_udf, lit, max, min, sum,
852854 BuiltInWindowFunction , ScalarFunctionImplementation , Volatility , WindowFunction ,
853855 } ;
856+ use datafusion_physical_expr:: expressions:: Column ;
854857
855858 #[ tokio:: test]
856859 async fn select_columns ( ) -> Result < ( ) > {
@@ -1515,4 +1518,163 @@ mod tests {
15151518
15161519 Ok ( ( ) )
15171520 }
1521+
1522+ #[ tokio:: test]
1523+ async fn partition_aware_union ( ) -> Result < ( ) > {
1524+ let left = test_table ( ) . await ?. select_columns ( & [ "c1" , "c2" ] ) ?;
1525+ let right = test_table_with_name ( "c2" )
1526+ . await ?
1527+ . select_columns ( & [ "c1" , "c3" ] ) ?
1528+ . with_column_renamed ( "c2.c1" , "c2_c1" ) ?;
1529+
1530+ let left_rows = left. collect ( ) . await ?;
1531+ let right_rows = right. collect ( ) . await ?;
1532+ let join1 =
1533+ left. join ( right. clone ( ) , JoinType :: Inner , & [ "c1" ] , & [ "c2_c1" ] , None ) ?;
1534+ let join2 = left. join ( right, JoinType :: Inner , & [ "c1" ] , & [ "c2_c1" ] , None ) ?;
1535+
1536+ let union = join1. union ( join2) ?;
1537+
1538+ let union_rows = union. collect ( ) . await ?;
1539+
1540+ assert_eq ! ( 100 , left_rows. iter( ) . map( |x| x. num_rows( ) ) . sum:: <usize >( ) ) ;
1541+ assert_eq ! ( 100 , right_rows. iter( ) . map( |x| x. num_rows( ) ) . sum:: <usize >( ) ) ;
1542+ assert_eq ! ( 4016 , union_rows. iter( ) . map( |x| x. num_rows( ) ) . sum:: <usize >( ) ) ;
1543+
1544+ let physical_plan = union. create_physical_plan ( ) . await ?;
1545+ let default_partition_count =
1546+ SessionContext :: new ( ) . copied_config ( ) . target_partitions ;
1547+
1548+ // For partition aware union, the output partition count should not be changed.
1549+ assert_eq ! (
1550+ physical_plan. output_partitioning( ) . partition_count( ) ,
1551+ default_partition_count
1552+ ) ;
1553+ // For partition aware union, the output partition is the same with the union's inputs
1554+ for child in physical_plan. children ( ) {
1555+ assert_eq ! (
1556+ physical_plan. output_partitioning( ) ,
1557+ child. output_partitioning( )
1558+ ) ;
1559+ }
1560+
1561+ Ok ( ( ) )
1562+ }
1563+
1564+ #[ tokio:: test]
1565+ async fn non_partition_aware_union ( ) -> Result < ( ) > {
1566+ let left = test_table ( ) . await ?. select_columns ( & [ "c1" , "c2" ] ) ?;
1567+ let right = test_table_with_name ( "c2" )
1568+ . await ?
1569+ . select_columns ( & [ "c1" , "c2" ] ) ?
1570+ . with_column_renamed ( "c2.c1" , "c2_c1" ) ?
1571+ . with_column_renamed ( "c2.c2" , "c2_c2" ) ?;
1572+
1573+ let left_rows = left. collect ( ) . await ?;
1574+ let right_rows = right. collect ( ) . await ?;
1575+ let join1 = left. join (
1576+ right. clone ( ) ,
1577+ JoinType :: Inner ,
1578+ & [ "c1" , "c2" ] ,
1579+ & [ "c2_c1" , "c2_c2" ] ,
1580+ None ,
1581+ ) ?;
1582+
1583+ // join key ordering is different
1584+ let join2 = left. join (
1585+ right,
1586+ JoinType :: Inner ,
1587+ & [ "c2" , "c1" ] ,
1588+ & [ "c2_c2" , "c2_c1" ] ,
1589+ None ,
1590+ ) ?;
1591+
1592+ let union = join1. union ( join2) ?;
1593+
1594+ let union_rows = union. collect ( ) . await ?;
1595+
1596+ assert_eq ! ( 100 , left_rows. iter( ) . map( |x| x. num_rows( ) ) . sum:: <usize >( ) ) ;
1597+ assert_eq ! ( 100 , right_rows. iter( ) . map( |x| x. num_rows( ) ) . sum:: <usize >( ) ) ;
1598+ assert_eq ! ( 916 , union_rows. iter( ) . map( |x| x. num_rows( ) ) . sum:: <usize >( ) ) ;
1599+
1600+ let physical_plan = union. create_physical_plan ( ) . await ?;
1601+ let default_partition_count =
1602+ SessionContext :: new ( ) . copied_config ( ) . target_partitions ;
1603+
1604+ // For non-partition aware union, the output partitioning count should be the combination of all output partitions count
1605+ assert ! ( matches!(
1606+ physical_plan. output_partitioning( ) ,
1607+ Partitioning :: UnknownPartitioning ( partition_count) if partition_count == default_partition_count * 2 ) ) ;
1608+ Ok ( ( ) )
1609+ }
1610+
1611+ #[ tokio:: test]
1612+ async fn verify_join_output_partitioning ( ) -> Result < ( ) > {
1613+ let left = test_table ( ) . await ?. select_columns ( & [ "c1" , "c2" ] ) ?;
1614+ let right = test_table_with_name ( "c2" )
1615+ . await ?
1616+ . select_columns ( & [ "c1" , "c2" ] ) ?
1617+ . with_column_renamed ( "c2.c1" , "c2_c1" ) ?
1618+ . with_column_renamed ( "c2.c2" , "c2_c2" ) ?;
1619+
1620+ let all_join_types = vec ! [
1621+ JoinType :: Inner ,
1622+ JoinType :: Left ,
1623+ JoinType :: Right ,
1624+ JoinType :: Full ,
1625+ JoinType :: LeftSemi ,
1626+ JoinType :: RightSemi ,
1627+ JoinType :: LeftAnti ,
1628+ JoinType :: RightAnti ,
1629+ ] ;
1630+
1631+ let default_partition_count =
1632+ SessionContext :: new ( ) . copied_config ( ) . target_partitions ;
1633+
1634+ for join_type in all_join_types {
1635+ let join = left. join (
1636+ right. clone ( ) ,
1637+ join_type,
1638+ & [ "c1" , "c2" ] ,
1639+ & [ "c2_c1" , "c2_c2" ] ,
1640+ None ,
1641+ ) ?;
1642+ let physical_plan = join. create_physical_plan ( ) . await ?;
1643+ let out_partitioning = physical_plan. output_partitioning ( ) ;
1644+ let join_schema = physical_plan. schema ( ) ;
1645+
1646+ match join_type {
1647+ JoinType :: Inner
1648+ | JoinType :: Left
1649+ | JoinType :: LeftSemi
1650+ | JoinType :: LeftAnti => {
1651+ let left_exprs: Vec < Arc < dyn PhysicalExpr > > = vec ! [
1652+ Arc :: new( Column :: new_with_schema( "c1" , & join_schema) . unwrap( ) ) ,
1653+ Arc :: new( Column :: new_with_schema( "c2" , & join_schema) . unwrap( ) ) ,
1654+ ] ;
1655+ assert_eq ! (
1656+ out_partitioning,
1657+ Partitioning :: Hash ( left_exprs, default_partition_count)
1658+ ) ;
1659+ }
1660+ JoinType :: Right | JoinType :: RightSemi | JoinType :: RightAnti => {
1661+ let right_exprs: Vec < Arc < dyn PhysicalExpr > > = vec ! [
1662+ Arc :: new( Column :: new_with_schema( "c2_c1" , & join_schema) . unwrap( ) ) ,
1663+ Arc :: new( Column :: new_with_schema( "c2_c2" , & join_schema) . unwrap( ) ) ,
1664+ ] ;
1665+ assert_eq ! (
1666+ out_partitioning,
1667+ Partitioning :: Hash ( right_exprs, default_partition_count)
1668+ ) ;
1669+ }
1670+ JoinType :: Full => {
1671+ assert ! ( matches!(
1672+ out_partitioning,
1673+ Partitioning :: UnknownPartitioning ( partition_count) if partition_count == default_partition_count) ) ;
1674+ }
1675+ }
1676+ }
1677+
1678+ Ok ( ( ) )
1679+ }
15181680}
0 commit comments