Skip to content

Commit fdcd277

Browse files
committed
feat: split physical enforcement into a PhysicalAnalyzerRule phase
Mirror the logical AnalyzerRule/OptimizerRule split on the physical side. - Add a `PhysicalAnalyzerRule` trait and `PhysicalAnalyzer` rule list, plumbed through Session / SessionState / SessionStateBuilder the same way physical optimizer rules are. - Split `EnsureRequirements` into `enforce_distribution_requirements` (Phases 0-2a), `enforce_requirements` (Phases 0-2), and `optimize_sorts` (Phase 3). It now runs as the analyzer doing *distribution enforcement only*; a new `OptimizeSorts` optimizer rule re-enforces and runs the sort optimizations exactly once, at the former `EnsureRequirements` position. The `PhysicalOptimizerRule` impl is retained (running both halves) so downstream chains that register it keep working. - The planner runs the analyzer phase first, then the optimizer phase. - `LimitedDistinctAggregation` now looks through `RepartitionExec` / `CoalescePartitionsExec`, so the distribution-first analyzer no longer drops its pushed-down limit hint. Full sqllogictest suite passes with no plan changes (only EXPLAIN VERBOSE pass-trace churn in explain.slt, regenerated). Prerequisite for the convergence-loop work in apache#25572.
1 parent 424cfd8 commit fdcd277

13 files changed

Lines changed: 754 additions & 150 deletions

File tree

‎datafusion/core/src/execution/session_state.rs‎

Lines changed: 82 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,10 +73,13 @@ use datafusion_optimizer::{
7373
};
7474
use datafusion_physical_expr::create_physical_expr;
7575
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
76+
use datafusion_physical_optimizer::analyzer::PhysicalAnalyzer;
7677
use datafusion_physical_optimizer::optimizer::PhysicalOptimizer;
7778
use datafusion_physical_plan::ExecutionPlan;
7879
use datafusion_physical_plan::operator_statistics::StatisticsRegistry;
79-
use datafusion_session::{PhysicalOptimizerContext, PhysicalOptimizerRule, Session};
80+
use datafusion_session::{
81+
PhysicalAnalyzerRule, PhysicalOptimizerContext, PhysicalOptimizerRule, Session,
82+
};
8083
#[cfg(feature = "sql")]
8184
use datafusion_sql::{
8285
parser::{DFParserBuilder, Statement},
@@ -166,6 +169,9 @@ struct SessionStateInner {
166169
type_planner: Option<Arc<dyn TypePlanner>>,
167170
/// Responsible for optimizing a logical plan
168171
optimizer: Optimizer,
172+
/// Responsible for enforcing invariants on a physical execution plan
173+
/// (distribution, ordering) before optimization
174+
physical_analyzers: PhysicalAnalyzer,
169175
/// Responsible for optimizing a physical execution plan
170176
physical_optimizers: PhysicalOptimizer,
171177
/// Responsible for planning `LogicalPlan`s, and `ExecutionPlan`
@@ -259,6 +265,7 @@ impl Debug for SessionState {
259265
ret.field("query_planners", &self.inner.query_planner)
260266
.field("analyzer", &self.inner.analyzer)
261267
.field("optimizer", &self.inner.optimizer)
268+
.field("physical_analyzers", &self.inner.physical_analyzers)
262269
.field("physical_optimizers", &self.inner.physical_optimizers)
263270
.field("table_functions", &self.inner.table_functions)
264271
.field("scalar_functions", &self.inner.scalar_functions)
@@ -299,6 +306,10 @@ impl Session for SessionState {
299306
SessionState::physical_optimizers(self)
300307
}
301308

309+
fn physical_analyzers(&self) -> &[Arc<dyn PhysicalAnalyzerRule + Send + Sync>] {
310+
SessionState::physical_analyzers(self)
311+
}
312+
302313
fn statistics_registry(&self) -> Option<&StatisticsRegistry> {
303314
SessionState::statistics_registry(self)
304315
}
@@ -441,6 +452,19 @@ impl SessionState {
441452
self
442453
}
443454

455+
/// Add `physical_analyzer_rule` to the end of the list of
456+
/// [`PhysicalAnalyzerRule`]s run before physical optimization.
457+
pub fn add_physical_analyzer_rule(
458+
&mut self,
459+
physical_analyzer_rule: Arc<dyn PhysicalAnalyzerRule + Send + Sync>,
460+
) -> &Self {
461+
Arc::make_mut(&mut self.inner)
462+
.physical_analyzers
463+
.rules
464+
.push(physical_analyzer_rule);
465+
self
466+
}
467+
444468
// the add_optimizer_rule takes an owned reference
445469
// it should probably be renamed to `with_optimizer_rule` to follow builder style
446470
// and `add_optimizer_rule` that takes &mut self added instead of this
@@ -921,6 +945,11 @@ impl SessionState {
921945
&self.inner.physical_optimizers.rules
922946
}
923947

948+
/// Return the physical analyzers
949+
pub fn physical_analyzers(&self) -> &[Arc<dyn PhysicalAnalyzerRule + Send + Sync>] {
950+
&self.inner.physical_analyzers.rules
951+
}
952+
924953
/// return the configuration options
925954
pub fn config_options(&self) -> &Arc<ConfigOptions> {
926955
self.inner.config.options()
@@ -1119,6 +1148,7 @@ pub struct SessionStateBuilder {
11191148
#[cfg(feature = "sql")]
11201149
type_planner: Option<Arc<dyn TypePlanner>>,
11211150
optimizer: Option<Optimizer>,
1151+
physical_analyzers: Option<PhysicalAnalyzer>,
11221152
physical_optimizers: Option<PhysicalOptimizer>,
11231153
query_planner: Option<Arc<dyn QueryPlanner + Send + Sync>>,
11241154
catalog_list: Option<Arc<dyn CatalogProviderList>>,
@@ -1141,6 +1171,7 @@ pub struct SessionStateBuilder {
11411171
// fields to support convenience functions
11421172
analyzer_rules: Option<Vec<Arc<dyn AnalyzerRule + Send + Sync>>>,
11431173
optimizer_rules: Option<Vec<Arc<dyn OptimizerRule + Send + Sync>>>,
1174+
physical_analyzer_rules: Option<Vec<Arc<dyn PhysicalAnalyzerRule + Send + Sync>>>,
11441175
physical_optimizer_rules: Option<Vec<Arc<dyn PhysicalOptimizerRule + Send + Sync>>>,
11451176
}
11461177

@@ -1162,6 +1193,7 @@ impl SessionStateBuilder {
11621193
#[cfg(feature = "sql")]
11631194
type_planner: None,
11641195
optimizer: None,
1196+
physical_analyzers: None,
11651197
physical_optimizers: None,
11661198
query_planner: None,
11671199
catalog_list: None,
@@ -1184,6 +1216,7 @@ impl SessionStateBuilder {
11841216
// fields to support convenience functions
11851217
analyzer_rules: None,
11861218
optimizer_rules: None,
1219+
physical_analyzer_rules: None,
11871220
physical_optimizer_rules: None,
11881221
}
11891222
}
@@ -1223,6 +1256,7 @@ impl SessionStateBuilder {
12231256
#[cfg(feature = "sql")]
12241257
type_planner: existing.type_planner,
12251258
optimizer: Some(existing.optimizer),
1259+
physical_analyzers: Some(existing.physical_analyzers),
12261260
physical_optimizers: Some(existing.physical_optimizers),
12271261
query_planner: Some(existing.query_planner),
12281262
catalog_list: Some(existing.catalog_list),
@@ -1249,6 +1283,7 @@ impl SessionStateBuilder {
12491283
// fields to support convenience functions
12501284
analyzer_rules: None,
12511285
optimizer_rules: None,
1286+
physical_analyzer_rules: None,
12521287
physical_optimizer_rules: None,
12531288
}
12541289
}
@@ -1414,6 +1449,27 @@ impl SessionStateBuilder {
14141449
self
14151450
}
14161451

1452+
/// Set the [`PhysicalAnalyzerRule`]s run before physical optimization.
1453+
pub fn with_physical_analyzer_rules(
1454+
mut self,
1455+
physical_analyzers: Vec<Arc<dyn PhysicalAnalyzerRule + Send + Sync>>,
1456+
) -> Self {
1457+
self.physical_analyzers = Some(PhysicalAnalyzer::with_rules(physical_analyzers));
1458+
self
1459+
}
1460+
1461+
/// Add `physical_analyzer_rule` to the end of the list of
1462+
/// [`PhysicalAnalyzerRule`]s run before physical optimization.
1463+
pub fn with_physical_analyzer_rule(
1464+
mut self,
1465+
physical_analyzer_rule: Arc<dyn PhysicalAnalyzerRule + Send + Sync>,
1466+
) -> Self {
1467+
let mut rules = self.physical_analyzer_rules.unwrap_or_default();
1468+
rules.push(physical_analyzer_rule);
1469+
self.physical_analyzer_rules = Some(rules);
1470+
self
1471+
}
1472+
14171473
/// Set the [`QueryPlanner`]
14181474
pub fn with_query_planner(
14191475
mut self,
@@ -1640,6 +1696,7 @@ impl SessionStateBuilder {
16401696
#[cfg(feature = "sql")]
16411697
type_planner,
16421698
optimizer,
1699+
physical_analyzers,
16431700
physical_optimizers,
16441701
query_planner,
16451702
catalog_list,
@@ -1661,6 +1718,7 @@ impl SessionStateBuilder {
16611718
statistics_registry,
16621719
analyzer_rules,
16631720
optimizer_rules,
1721+
physical_analyzer_rules,
16641722
physical_optimizer_rules,
16651723
} = self;
16661724

@@ -1676,6 +1734,7 @@ impl SessionStateBuilder {
16761734
#[cfg(feature = "sql")]
16771735
type_planner,
16781736
optimizer: optimizer.unwrap_or_default(),
1737+
physical_analyzers: physical_analyzers.unwrap_or_default(),
16791738
physical_optimizers: physical_optimizers.unwrap_or_default(),
16801739
query_planner: query_planner
16811740
.unwrap_or_else(|| Arc::new(DefaultQueryPlanner {})),
@@ -1820,6 +1879,14 @@ impl SessionStateBuilder {
18201879
}
18211880
}
18221881

1882+
if let Some(physical_analyzer_rules) = physical_analyzer_rules {
1883+
let physical_analyzers =
1884+
&mut Arc::make_mut(&mut state.inner).physical_analyzers;
1885+
for physical_analyzer_rule in physical_analyzer_rules {
1886+
physical_analyzers.rules.push(physical_analyzer_rule);
1887+
}
1888+
}
1889+
18231890
if let Some(physical_optimizer_rules) = physical_optimizer_rules {
18241891
let physical_optimizers =
18251892
&mut Arc::make_mut(&mut state.inner).physical_optimizers;
@@ -1868,6 +1935,11 @@ impl SessionStateBuilder {
18681935
&mut self.physical_optimizers
18691936
}
18701937

1938+
/// Returns the current physical_analyzers value
1939+
pub fn physical_analyzers(&mut self) -> &mut Option<PhysicalAnalyzer> {
1940+
&mut self.physical_analyzers
1941+
}
1942+
18711943
/// Returns the current query_planner value
18721944
pub fn query_planner(&mut self) -> &mut Option<Arc<dyn QueryPlanner + Send + Sync>> {
18731945
&mut self.query_planner
@@ -1972,6 +2044,13 @@ impl SessionStateBuilder {
19722044
) -> &mut Option<Vec<Arc<dyn PhysicalOptimizerRule + Send + Sync>>> {
19732045
&mut self.physical_optimizer_rules
19742046
}
2047+
2048+
/// Returns the current physical_analyzer_rules value
2049+
pub fn physical_analyzer_rules(
2050+
&mut self,
2051+
) -> &mut Option<Vec<Arc<dyn PhysicalAnalyzerRule + Send + Sync>>> {
2052+
&mut self.physical_analyzer_rules
2053+
}
19752054
}
19762055

19772056
impl Debug for SessionStateBuilder {
@@ -1999,6 +2078,8 @@ impl Debug for SessionStateBuilder {
19992078
.field("analyzer", &self.analyzer)
20002079
.field("optimizer_rules", &self.optimizer_rules)
20012080
.field("optimizer", &self.optimizer)
2081+
.field("physical_analyzer_rules", &self.physical_analyzer_rules)
2082+
.field("physical_analyzers", &self.physical_analyzers)
20022083
.field("physical_optimizer_rules", &self.physical_optimizer_rules)
20032084
.field("physical_optimizers", &self.physical_optimizers)
20042085
.field("table_functions", &self.table_functions)

0 commit comments

Comments
 (0)