Skip to content

Commit e618a80

Browse files
committed
feat: extract enforcement into a PhysicalAnalyzerRule phase
Mirror the logical layer's AnalyzerRule/OptimizerRule split on the physical side: an analyzer phase makes the plan valid (enforcing the distribution and ordering requirements every operator declares) and the optimizer phase makes the already-valid plan faster. - Add a `PhysicalAnalyzerRule` trait and a `PhysicalAnalyzer` rule list, plumbed through `Session` / `SessionState` / `SessionStateBuilder` the same way physical optimizer rules are. - `EnsureRequirements` now implements `PhysicalAnalyzerRule` and is the sole default analyzer. It is kept as a `PhysicalOptimizerRule` too, so custom rule lists that still register it keep working. - The planner runs the analyzer phase at `EnsureRequirements`' historical position (right before `CombinePartialFinalAggregate`), so the default pipeline order and every plan are unchanged. This is a prerequisite for the convergence-loop work in apache#25572: once enforcement is its own phase, the optimizer rules can be re-run to a fixed point without re-running the enforcement passes.
1 parent 0576a0b commit e618a80

11 files changed

Lines changed: 584 additions & 48 deletions

File tree

‎datafusion/core/src/execution/session_state.rs‎

Lines changed: 82 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,10 +73,13 @@ use datafusion_optimizer::{
7373
};
7474
use datafusion_physical_expr::create_physical_expr;
7575
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
76+
use datafusion_physical_optimizer::analyzer::PhysicalAnalyzer;
7677
use datafusion_physical_optimizer::optimizer::PhysicalOptimizer;
7778
use datafusion_physical_plan::ExecutionPlan;
7879
use datafusion_physical_plan::operator_statistics::StatisticsRegistry;
79-
use datafusion_session::{PhysicalOptimizerContext, PhysicalOptimizerRule, Session};
80+
use datafusion_session::{
81+
PhysicalAnalyzerRule, PhysicalOptimizerContext, PhysicalOptimizerRule, Session,
82+
};
8083
#[cfg(feature = "sql")]
8184
use datafusion_sql::{
8285
parser::{DFParserBuilder, Statement},
@@ -166,6 +169,9 @@ struct SessionStateInner {
166169
type_planner: Option<Arc<dyn TypePlanner>>,
167170
/// Responsible for optimizing a logical plan
168171
optimizer: Optimizer,
172+
/// Responsible for enforcing invariants on a physical execution plan
173+
/// (distribution, ordering) before optimization
174+
physical_analyzers: PhysicalAnalyzer,
169175
/// Responsible for optimizing a physical execution plan
170176
physical_optimizers: PhysicalOptimizer,
171177
/// Responsible for planning `LogicalPlan`s, and `ExecutionPlan`
@@ -259,6 +265,7 @@ impl Debug for SessionState {
259265
ret.field("query_planners", &self.inner.query_planner)
260266
.field("analyzer", &self.inner.analyzer)
261267
.field("optimizer", &self.inner.optimizer)
268+
.field("physical_analyzers", &self.inner.physical_analyzers)
262269
.field("physical_optimizers", &self.inner.physical_optimizers)
263270
.field("table_functions", &self.inner.table_functions)
264271
.field("scalar_functions", &self.inner.scalar_functions)
@@ -299,6 +306,10 @@ impl Session for SessionState {
299306
SessionState::physical_optimizers(self)
300307
}
301308

309+
fn physical_analyzers(&self) -> &[Arc<dyn PhysicalAnalyzerRule + Send + Sync>] {
310+
SessionState::physical_analyzers(self)
311+
}
312+
302313
fn statistics_registry(&self) -> Option<&StatisticsRegistry> {
303314
SessionState::statistics_registry(self)
304315
}
@@ -441,6 +452,19 @@ impl SessionState {
441452
self
442453
}
443454

455+
/// Add `physical_analyzer_rule` to the end of the list of
456+
/// [`PhysicalAnalyzerRule`]s run before physical optimization.
457+
pub fn add_physical_analyzer_rule(
458+
&mut self,
459+
physical_analyzer_rule: Arc<dyn PhysicalAnalyzerRule + Send + Sync>,
460+
) -> &Self {
461+
Arc::make_mut(&mut self.inner)
462+
.physical_analyzers
463+
.rules
464+
.push(physical_analyzer_rule);
465+
self
466+
}
467+
444468
// the add_optimizer_rule takes an owned reference
445469
// it should probably be renamed to `with_optimizer_rule` to follow builder style
446470
// and `add_optimizer_rule` that takes &mut self added instead of this
@@ -921,6 +945,11 @@ impl SessionState {
921945
&self.inner.physical_optimizers.rules
922946
}
923947

948+
/// Return the physical analyzers
949+
pub fn physical_analyzers(&self) -> &[Arc<dyn PhysicalAnalyzerRule + Send + Sync>] {
950+
&self.inner.physical_analyzers.rules
951+
}
952+
924953
/// return the configuration options
925954
pub fn config_options(&self) -> &Arc<ConfigOptions> {
926955
self.inner.config.options()
@@ -1119,6 +1148,7 @@ pub struct SessionStateBuilder {
11191148
#[cfg(feature = "sql")]
11201149
type_planner: Option<Arc<dyn TypePlanner>>,
11211150
optimizer: Option<Optimizer>,
1151+
physical_analyzers: Option<PhysicalAnalyzer>,
11221152
physical_optimizers: Option<PhysicalOptimizer>,
11231153
query_planner: Option<Arc<dyn QueryPlanner + Send + Sync>>,
11241154
catalog_list: Option<Arc<dyn CatalogProviderList>>,
@@ -1141,6 +1171,7 @@ pub struct SessionStateBuilder {
11411171
// fields to support convenience functions
11421172
analyzer_rules: Option<Vec<Arc<dyn AnalyzerRule + Send + Sync>>>,
11431173
optimizer_rules: Option<Vec<Arc<dyn OptimizerRule + Send + Sync>>>,
1174+
physical_analyzer_rules: Option<Vec<Arc<dyn PhysicalAnalyzerRule + Send + Sync>>>,
11441175
physical_optimizer_rules: Option<Vec<Arc<dyn PhysicalOptimizerRule + Send + Sync>>>,
11451176
}
11461177

@@ -1162,6 +1193,7 @@ impl SessionStateBuilder {
11621193
#[cfg(feature = "sql")]
11631194
type_planner: None,
11641195
optimizer: None,
1196+
physical_analyzers: None,
11651197
physical_optimizers: None,
11661198
query_planner: None,
11671199
catalog_list: None,
@@ -1184,6 +1216,7 @@ impl SessionStateBuilder {
11841216
// fields to support convenience functions
11851217
analyzer_rules: None,
11861218
optimizer_rules: None,
1219+
physical_analyzer_rules: None,
11871220
physical_optimizer_rules: None,
11881221
}
11891222
}
@@ -1223,6 +1256,7 @@ impl SessionStateBuilder {
12231256
#[cfg(feature = "sql")]
12241257
type_planner: existing.type_planner,
12251258
optimizer: Some(existing.optimizer),
1259+
physical_analyzers: Some(existing.physical_analyzers),
12261260
physical_optimizers: Some(existing.physical_optimizers),
12271261
query_planner: Some(existing.query_planner),
12281262
catalog_list: Some(existing.catalog_list),
@@ -1249,6 +1283,7 @@ impl SessionStateBuilder {
12491283
// fields to support convenience functions
12501284
analyzer_rules: None,
12511285
optimizer_rules: None,
1286+
physical_analyzer_rules: None,
12521287
physical_optimizer_rules: None,
12531288
}
12541289
}
@@ -1414,6 +1449,27 @@ impl SessionStateBuilder {
14141449
self
14151450
}
14161451

1452+
/// Set the [`PhysicalAnalyzerRule`]s run before physical optimization.
1453+
pub fn with_physical_analyzer_rules(
1454+
mut self,
1455+
physical_analyzers: Vec<Arc<dyn PhysicalAnalyzerRule + Send + Sync>>,
1456+
) -> Self {
1457+
self.physical_analyzers = Some(PhysicalAnalyzer::with_rules(physical_analyzers));
1458+
self
1459+
}
1460+
1461+
/// Add `physical_analyzer_rule` to the end of the list of
1462+
/// [`PhysicalAnalyzerRule`]s run before physical optimization.
1463+
pub fn with_physical_analyzer_rule(
1464+
mut self,
1465+
physical_analyzer_rule: Arc<dyn PhysicalAnalyzerRule + Send + Sync>,
1466+
) -> Self {
1467+
let mut rules = self.physical_analyzer_rules.unwrap_or_default();
1468+
rules.push(physical_analyzer_rule);
1469+
self.physical_analyzer_rules = Some(rules);
1470+
self
1471+
}
1472+
14171473
/// Set the [`QueryPlanner`]
14181474
pub fn with_query_planner(
14191475
mut self,
@@ -1640,6 +1696,7 @@ impl SessionStateBuilder {
16401696
#[cfg(feature = "sql")]
16411697
type_planner,
16421698
optimizer,
1699+
physical_analyzers,
16431700
physical_optimizers,
16441701
query_planner,
16451702
catalog_list,
@@ -1661,6 +1718,7 @@ impl SessionStateBuilder {
16611718
statistics_registry,
16621719
analyzer_rules,
16631720
optimizer_rules,
1721+
physical_analyzer_rules,
16641722
physical_optimizer_rules,
16651723
} = self;
16661724

@@ -1676,6 +1734,7 @@ impl SessionStateBuilder {
16761734
#[cfg(feature = "sql")]
16771735
type_planner,
16781736
optimizer: optimizer.unwrap_or_default(),
1737+
physical_analyzers: physical_analyzers.unwrap_or_default(),
16791738
physical_optimizers: physical_optimizers.unwrap_or_default(),
16801739
query_planner: query_planner
16811740
.unwrap_or_else(|| Arc::new(DefaultQueryPlanner {})),
@@ -1820,6 +1879,14 @@ impl SessionStateBuilder {
18201879
}
18211880
}
18221881

1882+
if let Some(physical_analyzer_rules) = physical_analyzer_rules {
1883+
let physical_analyzers =
1884+
&mut Arc::make_mut(&mut state.inner).physical_analyzers;
1885+
for physical_analyzer_rule in physical_analyzer_rules {
1886+
physical_analyzers.rules.push(physical_analyzer_rule);
1887+
}
1888+
}
1889+
18231890
if let Some(physical_optimizer_rules) = physical_optimizer_rules {
18241891
let physical_optimizers =
18251892
&mut Arc::make_mut(&mut state.inner).physical_optimizers;
@@ -1868,6 +1935,11 @@ impl SessionStateBuilder {
18681935
&mut self.physical_optimizers
18691936
}
18701937

1938+
/// Returns the current physical_analyzers value
1939+
pub fn physical_analyzers(&mut self) -> &mut Option<PhysicalAnalyzer> {
1940+
&mut self.physical_analyzers
1941+
}
1942+
18711943
/// Returns the current query_planner value
18721944
pub fn query_planner(&mut self) -> &mut Option<Arc<dyn QueryPlanner + Send + Sync>> {
18731945
&mut self.query_planner
@@ -1972,6 +2044,13 @@ impl SessionStateBuilder {
19722044
) -> &mut Option<Vec<Arc<dyn PhysicalOptimizerRule + Send + Sync>>> {
19732045
&mut self.physical_optimizer_rules
19742046
}
2047+
2048+
/// Returns the current physical_analyzer_rules value
2049+
pub fn physical_analyzer_rules(
2050+
&mut self,
2051+
) -> &mut Option<Vec<Arc<dyn PhysicalAnalyzerRule + Send + Sync>>> {
2052+
&mut self.physical_analyzer_rules
2053+
}
19752054
}
19762055

19772056
impl Debug for SessionStateBuilder {
@@ -1999,6 +2078,8 @@ impl Debug for SessionStateBuilder {
19992078
.field("analyzer", &self.analyzer)
20002079
.field("optimizer_rules", &self.optimizer_rules)
20012080
.field("optimizer", &self.optimizer)
2081+
.field("physical_analyzer_rules", &self.physical_analyzer_rules)
2082+
.field("physical_analyzers", &self.physical_analyzers)
20022083
.field("physical_optimizer_rules", &self.physical_optimizer_rules)
20032084
.field("physical_optimizers", &self.physical_optimizers)
20042085
.field("table_functions", &self.table_functions)

‎datafusion/core/src/optimizer_rule_reference.md‎

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,16 @@ Rule order matters. The default pipeline may change between releases.
6464
| 25 | `push_down_leaf_projections` | Pushes the helper projections created by leaf extraction toward leaf inputs. |
6565
| 26 | `optimize_projections` | Prunes unused columns and removes unnecessary logical projections. |
6666

67+
### Physical Analyzer Rules
68+
69+
Analyzer rules run before the physical optimizer rules and make the plan
70+
_valid_ by enforcing the distribution and ordering requirements every operator
71+
declares.
72+
73+
| order | rule | summary |
74+
| ----- | -------------------- | --------------------------------------------------------------------- |
75+
| 1 | `EnsureRequirements` | Enforces both distribution and sorting requirements in a single rule. |
76+
6777
### Physical Optimizer Rules
6878

6979
The same rule name may appear more than once when the default pipeline runs it
@@ -77,18 +87,17 @@ in multiple phases.
7787
| 4 | `LimitedDistinctAggregation` | - | Pushes limit hints into grouped distinct-style aggregations when only a small result is needed. |
7888
| 5 | `FilterPushdown` | pre-optimization phase | Pushes supported physical filters down toward data sources before distribution and sorting are enforced. |
7989
| 6 | `WindowTopN` | - | Replaces eligible row-number window and filter patterns with per-partition TopK execution. |
80-
| 7 | `EnsureRequirements` | - | Enforces both distribution and sorting requirements in a single idempotent rule. |
81-
| 8 | `CombinePartialFinalAggregate` | - | Collapses adjacent partial and final aggregates when the distributed shape makes them redundant. |
82-
| 9 | `OptimizeAggregateOrder` | - | Updates aggregate expressions to use the best ordering once sort requirements are known. |
83-
| 10 | `ProjectionPushdown` | early pass | Pushes projections toward inputs before later physical rewrites add more limit and TopK structure. |
84-
| 11 | `OutputRequirements` | remove phase | Removes the temporary output-requirement helper nodes after requirement-sensitive planning is done. |
85-
| 12 | `LimitAggregation` | - | Passes a limit hint into eligible aggregations so they can keep fewer accumulator buckets. |
86-
| 13 | `LimitPushPastWindows` | - | Pushes fetch limits through bounded window operators when doing so keeps the result correct. |
87-
| 14 | `HashJoinBuffering` | - | Adds buffering on the probe side of hash joins so probing can start before build completion. |
88-
| 15 | `LimitPushdown` | - | Moves physical limits into child operators or fetch-enabled variants to cut data early. |
89-
| 16 | `TopKRepartition` | - | Pushes TopK below hash repartition when the partition key is a prefix of the sort key. |
90-
| 17 | `ProjectionPushdown` | late pass | Runs projection pushdown again after limit and TopK rewrites expose new pruning opportunities. |
91-
| 18 | `PushdownSort` | - | Pushes sort requirements into data sources that can already return sorted output. |
92-
| 19 | `EnsureCooperative` | - | Wraps non-cooperative plan parts so long-running tasks yield fairly. |
93-
| 20 | `FilterPushdown(Post)` | post-optimization phase | Pushes dynamic filters at the end of optimization, after plan references stop moving. |
94-
| 21 | `SanityCheckPlan` | - | Validates that the final physical plan meets ordering, distribution, and infinite-input safety requirements. |
90+
| 7 | `CombinePartialFinalAggregate` | - | Collapses adjacent partial and final aggregates when the distributed shape makes them redundant. |
91+
| 8 | `OptimizeAggregateOrder` | - | Updates aggregate expressions to use the best ordering once sort requirements are known. |
92+
| 9 | `ProjectionPushdown` | early pass | Pushes projections toward inputs before later physical rewrites add more limit and TopK structure. |
93+
| 10 | `OutputRequirements` | remove phase | Removes the temporary output-requirement helper nodes after requirement-sensitive planning is done. |
94+
| 11 | `LimitAggregation` | - | Passes a limit hint into eligible aggregations so they can keep fewer accumulator buckets. |
95+
| 12 | `LimitPushPastWindows` | - | Pushes fetch limits through bounded window operators when doing so keeps the result correct. |
96+
| 13 | `HashJoinBuffering` | - | Adds buffering on the probe side of hash joins so probing can start before build completion. |
97+
| 14 | `LimitPushdown` | - | Moves physical limits into child operators or fetch-enabled variants to cut data early. |
98+
| 15 | `TopKRepartition` | - | Pushes TopK below hash repartition when the partition key is a prefix of the sort key. |
99+
| 16 | `ProjectionPushdown` | late pass | Runs projection pushdown again after limit and TopK rewrites expose new pruning opportunities. |
100+
| 17 | `PushdownSort` | - | Pushes sort requirements into data sources that can already return sorted output. |
101+
| 18 | `EnsureCooperative` | - | Wraps non-cooperative plan parts so long-running tasks yield fairly. |
102+
| 19 | `FilterPushdown(Post)` | post-optimization phase | Pushes dynamic filters at the end of optimization, after plan references stop moving. |
103+
| 20 | `SanityCheckPlan` | - | Validates that the final physical plan meets ordering, distribution, and infinite-input safety requirements. |

‎datafusion/core/src/optimizer_rule_reference.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
use datafusion_optimizer::analyzer::Analyzer;
1919
use datafusion_optimizer::optimizer::Optimizer;
20+
use datafusion_physical_optimizer::analyzer::PhysicalAnalyzer;
2021
use datafusion_physical_optimizer::optimizer::PhysicalOptimizer;
2122

2223
const OPTIMIZER_RULE_REFERENCE: &str = include_str!("optimizer_rule_reference.md");
@@ -73,6 +74,17 @@ fn logical_rules_match_documented_order() {
7374
assert_eq!(documented_rules("### Logical Optimizer Rules"), rules);
7475
}
7576

77+
#[test]
78+
fn physical_analyzer_rules_match_documented_order() {
79+
let rules: Vec<_> = PhysicalAnalyzer::new()
80+
.rules
81+
.iter()
82+
.map(|rule| rule.name().to_string())
83+
.collect();
84+
85+
assert_eq!(documented_rules("### Physical Analyzer Rules"), rules);
86+
}
87+
7688
#[test]
7789
fn physical_rules_match_documented_order() {
7890
let rules: Vec<_> = PhysicalOptimizer::new()

0 commit comments

Comments
 (0)