Skip to content

Commit 61a9ebb

Browse files
authored
Merge branch 'main' into qizhu/help-22384-rebase
2 parents c856062 + 4184b07 commit 61a9ebb

32 files changed

Lines changed: 993 additions & 309 deletions

File tree

‎datafusion-examples/examples/dataframe/cache_factory.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ use datafusion::error::Result;
2929
use datafusion::execution::context::QueryPlanner;
3030
use datafusion::execution::session_state::CacheFactory;
3131
use datafusion::execution::{SessionState, SessionStateBuilder};
32+
use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext;
3233
use datafusion::logical_expr::{
3334
Extension, LogicalPlan, UserDefinedLogicalNode, UserDefinedLogicalNodeCore,
3435
};
@@ -146,6 +147,7 @@ impl ExtensionPlanner for CacheNodePlanner {
146147
logical_inputs: &[&LogicalPlan],
147148
physical_inputs: &[Arc<dyn ExecutionPlan>],
148149
session_state: &SessionState,
150+
_planning_ctx: &PhysicalPlanningContext,
149151
) -> Result<Option<Arc<dyn ExecutionPlan>>> {
150152
if let Some(cache_node) = node.as_any().downcast_ref::<CacheNode>() {
151153
assert_eq!(logical_inputs.len(), 1, "Inconsistent number of inputs");

‎datafusion-examples/examples/query_planning/expr_api.rs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ use datafusion::functions_aggregate::first_last::first_value_udaf;
3333
use datafusion::logical_expr::execution_props::ExecutionProps;
3434
use datafusion::logical_expr::expr::BinaryExpr;
3535
use datafusion::logical_expr::interval_arithmetic::Interval;
36+
use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext;
3637
use datafusion::logical_expr::simplify::SimplifyContext;
3738
use datafusion::logical_expr::{ColumnarValue, ExprFunctionExt, ExprSchemable, Operator};
3839
use datafusion::optimizer::analyzer::type_coercion::TypeCoercionRewriter;
@@ -541,8 +542,12 @@ fn type_coercion_demo() -> Result<()> {
541542

542543
// Evaluation with an expression that has not been type coerced cannot succeed.
543544
let props = ExecutionProps::default();
544-
let physical_expr =
545-
datafusion::physical_expr::create_physical_expr(&expr, &df_schema, &props)?;
545+
let physical_expr = datafusion::physical_expr::create_physical_expr(
546+
&expr,
547+
&df_schema,
548+
&props,
549+
&PhysicalPlanningContext::default(),
550+
)?;
546551
let e = physical_expr.evaluate(&batch).unwrap_err();
547552
assert!(
548553
e.find_root()
@@ -566,6 +571,7 @@ fn type_coercion_demo() -> Result<()> {
566571
&coerced_expr,
567572
&df_schema,
568573
&props,
574+
&PhysicalPlanningContext::default(),
569575
)?;
570576
assert!(physical_expr.evaluate(&batch).is_ok());
571577

@@ -578,6 +584,7 @@ fn type_coercion_demo() -> Result<()> {
578584
&coerced_expr,
579585
&df_schema,
580586
&props,
587+
&PhysicalPlanningContext::default(),
581588
)?;
582589
assert!(physical_expr.evaluate(&batch).is_ok());
583590

@@ -606,6 +613,7 @@ fn type_coercion_demo() -> Result<()> {
606613
&coerced_expr,
607614
&df_schema,
608615
&props,
616+
&PhysicalPlanningContext::default(),
609617
)?;
610618
assert!(physical_expr.evaluate(&batch).is_ok());
611619

‎datafusion-examples/examples/query_planning/pruning.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ use datafusion::common::pruning::PruningStatistics;
2626
use datafusion::common::{DFSchema, ScalarValue};
2727
use datafusion::error::Result;
2828
use datafusion::execution::context::ExecutionProps;
29+
use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext;
2930
use datafusion::physical_expr::create_physical_expr;
3031
use datafusion::physical_optimizer::pruning::PruningPredicate;
3132
use datafusion::prelude::*;
@@ -194,7 +195,13 @@ impl PruningStatistics for MyCatalog {
194195
fn create_pruning_predicate(expr: Expr, schema: &SchemaRef) -> PruningPredicate {
195196
let df_schema = DFSchema::try_from(Arc::clone(schema)).unwrap();
196197
let props = ExecutionProps::new();
197-
let physical_expr = create_physical_expr(&expr, &df_schema, &props).unwrap();
198+
let physical_expr = create_physical_expr(
199+
&expr,
200+
&df_schema,
201+
&props,
202+
&PhysicalPlanningContext::default(),
203+
)
204+
.unwrap();
198205
PruningPredicate::try_new(physical_expr, Arc::clone(schema)).unwrap()
199206
}
200207

‎datafusion-examples/examples/relation_planner/table_sample.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ use datafusion_common::{
119119
DFSchemaRef, DataFusionError, Result, Statistics, internal_err, not_impl_err,
120120
plan_datafusion_err, plan_err,
121121
};
122+
use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
122123
use datafusion_expr::{
123124
UserDefinedLogicalNode, UserDefinedLogicalNodeCore,
124125
logical_plan::{Extension, LogicalPlan, LogicalPlanBuilder},
@@ -587,6 +588,7 @@ impl ExtensionPlanner for TableSampleExtensionPlanner {
587588
_logical_inputs: &[&LogicalPlan],
588589
physical_inputs: &[Arc<dyn ExecutionPlan>],
589590
_session_state: &SessionState,
591+
_planning_ctx: &PhysicalPlanningContext,
590592
) -> Result<Option<Arc<dyn ExecutionPlan>>> {
591593
let Some(sample_node) = node.as_any().downcast_ref::<TableSamplePlanNode>()
592594
else {

‎datafusion/catalog-listing/src/helpers.rs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ use arrow::{
3434
record_batch::RecordBatch,
3535
};
3636
use datafusion_expr::execution_props::ExecutionProps;
37+
use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
3738
use futures::stream::FuturesUnordered;
3839
use futures::{StreamExt, TryStreamExt, stream::BoxStream};
3940
use log::{debug, trace};
@@ -328,7 +329,12 @@ pub fn filter_partitioned_file(
328329

329330
let filter = utils::conjunction(filters.iter().cloned()).unwrap_or_else(|| lit(true));
330331
let props = ExecutionProps::new();
331-
let expr = create_physical_expr(&filter, df_schema, &props)?;
332+
let expr = create_physical_expr(
333+
&filter,
334+
df_schema,
335+
&props,
336+
&PhysicalPlanningContext::default(),
337+
)?;
332338

333339
// Since we're only operating on a single file, our batch and resulting "array" holds only one
334340
// value indicating if the input file matches the provided filters

‎datafusion/catalog-listing/src/table.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ use datafusion_execution::cache::cache_manager::{
4444
};
4545
use datafusion_expr::dml::InsertOp;
4646
use datafusion_expr::execution_props::ExecutionProps;
47+
use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
4748
use datafusion_expr::{
4849
Expr, Partitioning as LogicalPartitioning, TableProviderFilterPushDown, TableType,
4950
};
@@ -614,6 +615,7 @@ impl TableProvider for ListingTable {
614615
output_partitioning,
615616
&df_schema,
616617
state.execution_props(),
618+
&PhysicalPlanningContext::default(),
617619
)?
618620
}
619621
};

‎datafusion/catalog/src/memory/table.rs‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ use datafusion_datasource::memory::{MemSink, MemorySourceConfig};
3636
use datafusion_datasource::sink::DataSinkExec;
3737
use datafusion_datasource::source::DataSourceExec;
3838
use datafusion_expr::dml::InsertOp;
39+
use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
3940
use datafusion_expr::{Expr, SortExpr, TableType};
4041
use datafusion_physical_expr::{
4142
LexOrdering, PhysicalExpr, create_physical_expr, create_physical_sort_exprs,
@@ -209,8 +210,12 @@ impl TableProvider for MemTable {
209210
let eqp = state.execution_props();
210211
let mut file_sort_order = vec![];
211212
for sort_exprs in sort_order.iter() {
212-
let physical_exprs =
213-
create_physical_sort_exprs(sort_exprs, &df_schema, eqp)?;
213+
let physical_exprs = create_physical_sort_exprs(
214+
sort_exprs,
215+
&df_schema,
216+
eqp,
217+
&PhysicalPlanningContext::default(),
218+
)?;
214219
file_sort_order.extend(LexOrdering::new(physical_exprs));
215220
}
216221
source = source.try_with_sort_information(file_sort_order)?;
@@ -356,8 +361,12 @@ impl TableProvider for MemTable {
356361
let physical_assignments: HashMap<String, Arc<dyn PhysicalExpr>> = assignments
357362
.iter()
358363
.map(|(name, expr)| {
359-
let physical_expr =
360-
create_physical_expr(expr, &df_schema, state.execution_props())?;
364+
let physical_expr = create_physical_expr(
365+
expr,
366+
&df_schema,
367+
state.execution_props(),
368+
&PhysicalPlanningContext::default(),
369+
)?;
361370
Ok((name.clone(), physical_expr))
362371
})
363372
.collect::<Result<_>>()?;
@@ -470,8 +479,12 @@ fn evaluate_filters_to_mask(
470479
let mut combined_mask: Option<BooleanArray> = None;
471480

472481
for filter_expr in filters {
473-
let physical_expr =
474-
create_physical_expr(filter_expr, df_schema, execution_props)?;
482+
let physical_expr = create_physical_expr(
483+
filter_expr,
484+
df_schema,
485+
execution_props,
486+
&PhysicalPlanningContext::default(),
487+
)?;
475488

476489
let result = physical_expr.evaluate(batch)?;
477490
let array = result.into_array(batch.num_rows())?;

‎datafusion/catalog/src/streaming.rs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ use std::sync::Arc;
2222
use arrow::datatypes::SchemaRef;
2323
use async_trait::async_trait;
2424
use datafusion_common::{DFSchema, Result, plan_err};
25+
use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
2526
use datafusion_expr::{Expr, SortExpr, TableType};
2627
use datafusion_physical_expr::equivalence::project_ordering;
2728
use datafusion_physical_expr::projection::ProjectionMapping;
@@ -131,8 +132,12 @@ impl TableProvider for StreamingTable {
131132
let df_schema = DFSchema::try_from(Arc::clone(&self.schema))?;
132133
let eqp = state.execution_props();
133134

134-
let original_sort_exprs =
135-
create_physical_sort_exprs(&self.sort_order, &df_schema, eqp)?;
135+
let original_sort_exprs = create_physical_sort_exprs(
136+
&self.sort_order,
137+
&df_schema,
138+
eqp,
139+
&PhysicalPlanningContext::default(),
140+
)?;
136141

137142
if let Some(p) = projection {
138143
// When performing a projection, the output columns will not match

‎datafusion/core/src/execution/context/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2383,6 +2383,7 @@ mod tests {
23832383
use arrow_schema::FieldRef;
23842384
use datafusion_common::DataFusionError;
23852385
use datafusion_common::datatype::DataTypeExt;
2386+
use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
23862387
use std::error::Error;
23872388
use std::path::PathBuf;
23882389

@@ -2851,6 +2852,7 @@ mod tests {
28512852
_expr: &Expr,
28522853
_input_dfschema: &DFSchema,
28532854
_session_state: &SessionState,
2855+
_planning_ctx: &PhysicalPlanningContext,
28542856
) -> Result<Arc<dyn PhysicalExpr>> {
28552857
unimplemented!()
28562858
}

‎datafusion/core/src/execution/session_state.rs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ use datafusion_execution::runtime_env::RuntimeEnv;
5555
use datafusion_expr::TableSource;
5656
use datafusion_expr::execution_props::ExecutionProps;
5757
use datafusion_expr::expr_rewriter::FunctionRewrite;
58+
use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
5859
use datafusion_expr::planner::ExprPlanner;
5960
#[cfg(feature = "sql")]
6061
use datafusion_expr::planner::{RelationPlanner, TypePlanner};
@@ -799,7 +800,12 @@ impl SessionState {
799800
.transform_up(|expr| rewrite.rewrite(expr, df_schema, config_options))?
800801
.data;
801802
}
802-
create_physical_expr(&expr, df_schema, self.execution_props())
803+
create_physical_expr(
804+
&expr,
805+
df_schema,
806+
self.execution_props(),
807+
&PhysicalPlanningContext::default(),
808+
)
803809
}
804810

805811
/// Return the session ID

0 commit comments

Comments
 (0)