Skip to content

Commit c2f8e47

Browse files
timsaucerclaude
authored andcommitted
refactor: move planning APIs to session crate (apache#23842)
## Which issue does this PR close? - Addresses a portion of apache#23678 - Follow on to apache#23703 ## Rationale for this change This PR unlocks using a query planner across the FFI boundary. ## What changes are included in this PR? Moves these traits to the `datafusion-session` crate: - `QueryPlanner` - `PhysicalPlanner` - `ExtensionPlanner` - `PhysicalOptimizerRule` - `PhysicalOptimizerContext` Additionally changed the method signatures from taking `&SessionState` to `&dyn Session`. Adds these methods on `Session` trait: - `fn query_planner(&self) -> Arc<dyn QueryPlanner + Send + Sync>` - `fn optimize(&self, plan: &LogicalPlan) -> Result<LogicalPlan>` - `fn physical_optimizers(&self) -> &[Arc<dyn PhysicalOptimizerRule + Send + Sync>]` - `fn statistics_registry(&self) -> Option<&StatisticsRegistry>` ## Are these changes tested? Unit tests are added for the new methods. ## Are there any user-facing changes? Yes, users must implement new methods for `query_planner` for their custom `Session`. This is not expected to impact many users, since it is most common to use the existing `SessionState`. --------- Co-authored-by: Claude Opus 4.8 (1M context) <[email protected]>
1 parent 34f922d commit c2f8e47

15 files changed

Lines changed: 698 additions & 276 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion-examples/examples/dataframe/cache_factory.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ use std::sync::{Arc, RwLock};
2323

2424
use arrow::array::RecordBatch;
2525
use async_trait::async_trait;
26+
use datafusion::catalog::Session;
2627
use datafusion::catalog::memory::MemorySourceConfig;
2728
use datafusion::common::DFSchemaRef;
2829
use datafusion::error::Result;
@@ -146,7 +147,7 @@ impl ExtensionPlanner for CacheNodePlanner {
146147
node: &dyn UserDefinedLogicalNode,
147148
logical_inputs: &[&LogicalPlan],
148149
physical_inputs: &[Arc<dyn ExecutionPlan>],
149-
session_state: &SessionState,
150+
session_state: &dyn Session,
150151
_planning_ctx: &PhysicalPlanningContext,
151152
) -> Result<Option<Arc<dyn ExecutionPlan>>> {
152153
if let Some(cache_node) = node.as_any().downcast_ref::<CacheNode>() {
@@ -200,7 +201,7 @@ impl QueryPlanner for CacheNodeQueryPlanner {
200201
async fn create_physical_plan(
201202
&self,
202203
logical_plan: &LogicalPlan,
203-
session_state: &SessionState,
204+
session_state: &dyn Session,
204205
) -> Result<Arc<dyn ExecutionPlan>> {
205206
let physical_planner =
206207
DefaultPhysicalPlanner::with_extension_planners(vec![Arc::new(

‎datafusion-examples/examples/relation_planner/table_sample.rs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,9 +102,10 @@ use tonic::async_trait;
102102

103103
use datafusion::optimizer::simplify_expressions::simplify_literal::parse_literal;
104104
use datafusion::{
105+
catalog::Session,
105106
execution::{
106-
RecordBatchStream, SendableRecordBatchStream, SessionState, SessionStateBuilder,
107-
TaskContext, context::QueryPlanner,
107+
RecordBatchStream, SendableRecordBatchStream, SessionStateBuilder, TaskContext,
108+
context::QueryPlanner,
108109
},
109110
physical_expr::EquivalenceProperties,
110111
physical_plan::{
@@ -565,7 +566,7 @@ impl QueryPlanner for TableSampleQueryPlanner {
565566
async fn create_physical_plan(
566567
&self,
567568
logical_plan: &LogicalPlan,
568-
session_state: &SessionState,
569+
session_state: &dyn Session,
569570
) -> Result<Arc<dyn ExecutionPlan>> {
570571
let planner = DefaultPhysicalPlanner::with_extension_planners(vec![Arc::new(
571572
TableSampleExtensionPlanner,
@@ -587,7 +588,7 @@ impl ExtensionPlanner for TableSampleExtensionPlanner {
587588
node: &dyn UserDefinedLogicalNode,
588589
_logical_inputs: &[&LogicalPlan],
589590
physical_inputs: &[Arc<dyn ExecutionPlan>],
590-
_session_state: &SessionState,
591+
_session_state: &dyn Session,
591592
_planning_ctx: &PhysicalPlanningContext,
592593
) -> Result<Option<Arc<dyn ExecutionPlan>>> {
593594
let Some(sample_node) = node.as_any().downcast_ref::<TableSamplePlanNode>()

‎datafusion/core/src/execution/context/mod.rs‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
//! [`SessionContext`] API for registering data sources and executing queries
1919
20-
use std::any::Any;
2120
use std::collections::HashSet;
2221
use std::fmt::Debug;
2322
use std::sync::{Arc, Weak};
@@ -2181,16 +2180,9 @@ impl From<SessionContext> for SessionStateBuilder {
21812180
}
21822181
}
21832182

2183+
// Re-export from this module for backwards compatibility.
21842184
/// A planner used to add extensions to DataFusion logical and physical plans.
2185-
#[async_trait]
2186-
pub trait QueryPlanner: Any + Debug {
2187-
/// Given a [`LogicalPlan`], create an [`ExecutionPlan`] suitable for execution
2188-
async fn create_physical_plan(
2189-
&self,
2190-
logical_plan: &LogicalPlan,
2191-
session_state: &SessionState,
2192-
) -> Result<Arc<dyn ExecutionPlan>>;
2193-
}
2185+
pub use datafusion_session::{QueryPlanner, UnsupportedQueryPlanner};
21942186

21952187
/// Interface for handling `CREATE FUNCTION` statements and interacting with
21962188
/// [SessionState] to create and register functions ([`ScalarUDF`],
@@ -2390,6 +2382,7 @@ mod tests {
23902382
use crate::physical_planner::PhysicalPlanner;
23912383
use async_trait::async_trait;
23922384
use datafusion_expr::planner::TypePlanner;
2385+
use datafusion_session::Session;
23932386
use sqlparser::ast;
23942387
use tempfile::TempDir;
23952388

@@ -2836,7 +2829,7 @@ mod tests {
28362829
async fn create_physical_plan(
28372830
&self,
28382831
_logical_plan: &LogicalPlan,
2839-
_session_state: &SessionState,
2832+
_session_state: &dyn Session,
28402833
) -> Result<Arc<dyn ExecutionPlan>> {
28412834
not_impl_err!("query not supported")
28422835
}
@@ -2845,7 +2838,7 @@ mod tests {
28452838
&self,
28462839
_expr: &Expr,
28472840
_input_dfschema: &DFSchema,
2848-
_session_state: &SessionState,
2841+
_session_state: &dyn Session,
28492842
_planning_ctx: &PhysicalPlanningContext,
28502843
) -> Result<Arc<dyn PhysicalExpr>> {
28512844
unimplemented!()
@@ -2860,7 +2853,7 @@ mod tests {
28602853
async fn create_physical_plan(
28612854
&self,
28622855
logical_plan: &LogicalPlan,
2863-
session_state: &SessionState,
2856+
session_state: &dyn Session,
28642857
) -> Result<Arc<dyn ExecutionPlan>> {
28652858
let physical_planner = MyPhysicalPlanner {};
28662859
physical_planner

‎datafusion/core/src/execution/session_state.rs‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,12 +73,10 @@ use datafusion_optimizer::{
7373
};
7474
use datafusion_physical_expr::create_physical_expr;
7575
use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
76-
use datafusion_physical_optimizer::PhysicalOptimizerContext;
77-
use datafusion_physical_optimizer::PhysicalOptimizerRule;
7876
use datafusion_physical_optimizer::optimizer::PhysicalOptimizer;
7977
use datafusion_physical_plan::ExecutionPlan;
8078
use datafusion_physical_plan::operator_statistics::StatisticsRegistry;
81-
use datafusion_session::Session;
79+
use datafusion_session::{PhysicalOptimizerContext, PhysicalOptimizerRule, Session};
8280
#[cfg(feature = "sql")]
8381
use datafusion_sql::{
8482
parser::{DFParserBuilder, Statement},
@@ -274,6 +272,25 @@ impl Session for SessionState {
274272
Arc::clone(self.catalog_list())
275273
}
276274

275+
fn query_planner(&self) -> Arc<dyn QueryPlanner + Send + Sync> {
276+
// Disambiguate: `SessionState` has an inherent `query_planner` (returning
277+
// `&Arc<...>`) with the same name as this trait method. The qualified path
278+
// calls the inherent one; a bare `self.query_planner()` would recurse.
279+
Arc::clone(SessionState::query_planner(self))
280+
}
281+
282+
fn optimize(&self, plan: &LogicalPlan) -> datafusion_common::Result<LogicalPlan> {
283+
SessionState::optimize(self, plan)
284+
}
285+
286+
fn physical_optimizers(&self) -> &[Arc<dyn PhysicalOptimizerRule + Send + Sync>] {
287+
SessionState::physical_optimizers(self)
288+
}
289+
290+
fn statistics_registry(&self) -> Option<&StatisticsRegistry> {
291+
SessionState::statistics_registry(self)
292+
}
293+
277294
async fn create_physical_plan(
278295
&self,
279296
logical_plan: &LogicalPlan,
@@ -2322,7 +2339,7 @@ impl QueryPlanner for DefaultQueryPlanner {
23222339
async fn create_physical_plan(
23232340
&self,
23242341
logical_plan: &LogicalPlan,
2325-
session_state: &SessionState,
2342+
session_state: &dyn Session,
23262343
) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {
23272344
let planner = DefaultPhysicalPlanner::default();
23282345
planner

0 commit comments

Comments
 (0)