@@ -73,7 +73,8 @@ use crate::physical_optimizer::repartition::Repartition;
7373
7474use crate :: config:: {
7575 ConfigOptions , OPT_BATCH_SIZE , OPT_COALESCE_BATCHES , OPT_COALESCE_TARGET_BATCH_SIZE ,
76- OPT_FILTER_NULL_JOIN_KEYS , OPT_OPTIMIZER_MAX_PASSES , OPT_OPTIMIZER_SKIP_FAILED_RULES ,
76+ OPT_ENABLE_ROUND_ROBIN_REPARTITION , OPT_FILTER_NULL_JOIN_KEYS ,
77+ OPT_OPTIMIZER_MAX_PASSES , OPT_OPTIMIZER_SKIP_FAILED_RULES ,
7778} ;
7879use crate :: execution:: { runtime_env:: RuntimeEnv , FunctionRegistry } ;
7980use crate :: physical_optimizer:: enforcement:: BasicEnforcement ;
@@ -98,6 +99,7 @@ use crate::catalog::information_schema::{InformationSchemaProvider, INFORMATION_
9899use crate :: catalog:: listing_schema:: ListingSchemaProvider ;
99100use crate :: datasource:: object_store:: ObjectStoreUrl ;
100101use crate :: execution:: memory_pool:: MemoryPool ;
102+ use crate :: physical_optimizer:: global_sort_selection:: GlobalSortSelection ;
101103use crate :: physical_optimizer:: optimize_sorts:: OptimizeSorts ;
102104use uuid:: Uuid ;
103105
@@ -1530,11 +1532,47 @@ impl SessionState {
15301532 ) ;
15311533 }
15321534
1533- let mut physical_optimizers: Vec < Arc < dyn PhysicalOptimizerRule + Sync + Send > > = vec ! [
1534- Arc :: new( AggregateStatistics :: new( ) ) ,
1535- Arc :: new( JoinSelection :: new( ) ) ,
1536- ] ;
1535+ // We need to take care of the rule ordering. They may influence each other.
1536+ let mut physical_optimizers: Vec < Arc < dyn PhysicalOptimizerRule + Sync + Send > > =
1537+ vec ! [ Arc :: new( AggregateStatistics :: new( ) ) ] ;
1538+ // - In order to increase the parallelism, it will change the output partitioning
1539+ // of some operators in the plan tree, which will influence other rules.
1540+ // Therefore, it should be run as soon as possible.
1541+ // - The reason to make it optional is
1542+ // - it's not used for the distributed engine, Ballista.
1543+ // - it's conflicted with some parts of the BasicEnforcement, since it will
1544+ // introduce additional repartitioning while the BasicEnforcement aims at
1545+ // reducing unnecessary repartitioning.
1546+ if config
1547+ . config_options
1548+ . get_bool ( OPT_ENABLE_ROUND_ROBIN_REPARTITION )
1549+ . unwrap_or_default ( )
1550+ {
1551+ physical_optimizers. push ( Arc :: new ( Repartition :: new ( ) ) ) ;
1552+ }
1553+ //- Currently it will depend on the partition number to decide whether to change the
1554+ // single node sort to parallel local sort and merge. Therefore, it should be run
1555+ // after the Repartition.
1556+ // - Since it will change the output ordering of some operators, it should be run
1557+ // before JoinSelection and BasicEnforcement, which may depend on that.
1558+ physical_optimizers. push ( Arc :: new ( GlobalSortSelection :: new ( ) ) ) ;
1559+ // Statistics-base join selection will change the Auto mode to real join implementation,
1560+ // like collect left, or hash join, or future sort merge join, which will
1561+ // influence the BasicEnforcement to decide whether to add additional repartition
1562+ // and local sort to meet the distribution and ordering requirements.
1563+ // Therefore, it should be run before BasicEnforcement
1564+ physical_optimizers. push ( Arc :: new ( JoinSelection :: new ( ) ) ) ;
1565+ // It's for adding essential repartition and local sorting operator to satisfy the
1566+ // required distribution and local sort.
1567+ // Please make sure that the whole plan tree is determined.
15371568 physical_optimizers. push ( Arc :: new ( BasicEnforcement :: new ( ) ) ) ;
1569+ // `BasicEnforcement` stage conservatively inserts `SortExec`s to satisfy ordering requirements.
1570+ // However, a deeper analysis may sometimes reveal that such a `SortExec` is actually unnecessary.
1571+ // These cases typically arise when we have reversible `WindowAggExec`s or deep subqueries. The
1572+ // rule below performs this analysis and removes unnecessary `SortExec`s.
1573+ physical_optimizers. push ( Arc :: new ( OptimizeSorts :: new ( ) ) ) ;
1574+ // It will not influence the distribution and ordering of the whole plan tree.
1575+ // Therefore, to avoid influencing other rules, it should be run at last.
15381576 if config
15391577 . config_options
15401578 . get_bool ( OPT_COALESCE_BATCHES )
@@ -1549,16 +1587,6 @@ impl SessionState {
15491587 . unwrap ( ) ,
15501588 ) ) ) ;
15511589 }
1552- physical_optimizers. push ( Arc :: new ( Repartition :: new ( ) ) ) ;
1553- // Repartition rule could introduce additional RepartitionExec with RoundRobin partitioning.
1554- // To make sure the SinglePartition is satisfied, run the BasicEnforcement again, originally it was the AddCoalescePartitionsExec here.
1555- physical_optimizers. push ( Arc :: new ( BasicEnforcement :: new ( ) ) ) ;
1556-
1557- // `BasicEnforcement` stage conservatively inserts `SortExec`s to satisfy ordering requirements.
1558- // However, a deeper analysis may sometimes reveal that such a `SortExec` is actually unnecessary.
1559- // These cases typically arise when we have reversible `WindowAggExec`s or deep subqueries. The
1560- // rule below performs this analysis and removes unnecessary `SortExec`s.
1561- physical_optimizers. push ( Arc :: new ( OptimizeSorts :: new ( ) ) ) ;
15621590
15631591 SessionState {
15641592 session_id,
0 commit comments