Skip to content

Commit b909edf

Browse files
committed
refactor: ParquetExec logical expr. => phys. expr.
Use `Arc<dyn PhysicalExpr>` instead of `Expr` within `ParquetExec` and move lowering from logical to physical expression into plan lowering (e.g. `ListingTable`). This is in line w/ all other physical plan nodes (e.g. `FilterExpr`) and simplifies reasoning within physical optimizer but also allows correct passing of `ExecutionProps` into the conversion. Closes apache#4695.
1 parent bc5d03f commit b909edf

18 files changed

Lines changed: 825 additions & 496 deletions

File tree

‎datafusion/core/src/datasource/file_format/avro.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,13 @@ use std::sync::Arc;
2323
use arrow::datatypes::Schema;
2424
use arrow::{self, datatypes::SchemaRef};
2525
use async_trait::async_trait;
26+
use datafusion_physical_expr::PhysicalExpr;
2627
use object_store::{GetResult, ObjectMeta, ObjectStore};
2728

2829
use super::FileFormat;
2930
use crate::avro_to_arrow::read_avro_schema_from_reader;
3031
use crate::error::Result;
3132
use crate::execution::context::SessionState;
32-
use crate::logical_expr::Expr;
3333
use crate::physical_plan::file_format::{AvroExec, FileScanConfig};
3434
use crate::physical_plan::ExecutionPlan;
3535
use crate::physical_plan::Statistics;
@@ -82,7 +82,7 @@ impl FileFormat for AvroFormat {
8282
&self,
8383
_state: &SessionState,
8484
conf: FileScanConfig,
85-
_filters: &[Expr],
85+
_filters: Option<&Arc<dyn PhysicalExpr>>,
8686
) -> Result<Arc<dyn ExecutionPlan>> {
8787
let exec = AvroExec::new(conf);
8888
Ok(Arc::new(exec))

‎datafusion/core/src/datasource/file_format/csv.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ use bytes::{Buf, Bytes};
2929

3030
use datafusion_common::DataFusionError;
3131

32+
use datafusion_physical_expr::PhysicalExpr;
3233
use futures::{pin_mut, Stream, StreamExt, TryStreamExt};
3334
use object_store::{delimited::newline_delimited_stream, ObjectMeta, ObjectStore};
3435

@@ -37,7 +38,6 @@ use crate::datasource::file_format::file_type::FileCompressionType;
3738
use crate::datasource::file_format::DEFAULT_SCHEMA_INFER_MAX_RECORD;
3839
use crate::error::Result;
3940
use crate::execution::context::SessionState;
40-
use crate::logical_expr::Expr;
4141
use crate::physical_plan::file_format::{CsvExec, FileScanConfig};
4242
use crate::physical_plan::ExecutionPlan;
4343
use crate::physical_plan::Statistics;
@@ -154,7 +154,7 @@ impl FileFormat for CsvFormat {
154154
&self,
155155
_state: &SessionState,
156156
conf: FileScanConfig,
157-
_filters: &[Expr],
157+
_filters: Option<&Arc<dyn PhysicalExpr>>,
158158
) -> Result<Arc<dyn ExecutionPlan>> {
159159
let exec = CsvExec::new(
160160
conf,

‎datafusion/core/src/datasource/file_format/json.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ use arrow::json::reader::ValueIter;
2929
use async_trait::async_trait;
3030
use bytes::Buf;
3131

32+
use datafusion_physical_expr::PhysicalExpr;
3233
use object_store::{GetResult, ObjectMeta, ObjectStore};
3334

3435
use super::FileFormat;
@@ -37,7 +38,6 @@ use crate::datasource::file_format::file_type::FileCompressionType;
3738
use crate::datasource::file_format::DEFAULT_SCHEMA_INFER_MAX_RECORD;
3839
use crate::error::Result;
3940
use crate::execution::context::SessionState;
40-
use crate::logical_expr::Expr;
4141
use crate::physical_plan::file_format::NdJsonExec;
4242
use crate::physical_plan::ExecutionPlan;
4343
use crate::physical_plan::Statistics;
@@ -143,7 +143,7 @@ impl FileFormat for JsonFormat {
143143
&self,
144144
_state: &SessionState,
145145
conf: FileScanConfig,
146-
_filters: &[Expr],
146+
_filters: Option<&Arc<dyn PhysicalExpr>>,
147147
) -> Result<Arc<dyn ExecutionPlan>> {
148148
let exec = NdJsonExec::new(conf, self.file_compression_type.to_owned());
149149
Ok(Arc::new(exec))

‎datafusion/core/src/datasource/file_format/mod.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,12 @@ use std::sync::Arc;
3333

3434
use crate::arrow::datatypes::SchemaRef;
3535
use crate::error::Result;
36-
use crate::logical_expr::Expr;
3736
use crate::physical_plan::file_format::FileScanConfig;
3837
use crate::physical_plan::{ExecutionPlan, Statistics};
3938

4039
use crate::execution::context::SessionState;
4140
use async_trait::async_trait;
41+
use datafusion_physical_expr::PhysicalExpr;
4242
use object_store::{ObjectMeta, ObjectStore};
4343

4444
/// This trait abstracts all the file format specific implementations
@@ -84,7 +84,7 @@ pub trait FileFormat: Send + Sync + fmt::Debug {
8484
&self,
8585
state: &SessionState,
8686
conf: FileScanConfig,
87-
filters: &[Expr],
87+
filters: Option<&Arc<dyn PhysicalExpr>>,
8888
) -> Result<Arc<dyn ExecutionPlan>>;
8989
}
9090

@@ -143,7 +143,7 @@ pub(crate) mod test_util {
143143
output_ordering: None,
144144
infinite_source: false,
145145
},
146-
&[],
146+
None,
147147
)
148148
.await?;
149149
Ok(exec)

‎datafusion/core/src/datasource/file_format/parquet.rs‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ use arrow::datatypes::SchemaRef;
2525
use async_trait::async_trait;
2626
use bytes::{BufMut, BytesMut};
2727
use datafusion_common::DataFusionError;
28-
use datafusion_optimizer::utils::conjunction;
28+
use datafusion_physical_expr::PhysicalExpr;
2929
use hashbrown::HashMap;
3030
use object_store::{ObjectMeta, ObjectStore};
3131
use parquet::arrow::parquet_to_arrow_schema;
@@ -44,7 +44,6 @@ use crate::config::ConfigOptions;
4444
use crate::datasource::{create_max_min_accs, get_col_stats};
4545
use crate::error::Result;
4646
use crate::execution::context::SessionState;
47-
use crate::logical_expr::Expr;
4847
use crate::physical_plan::expressions::{MaxAccumulator, MinAccumulator};
4948
use crate::physical_plan::file_format::{ParquetExec, SchemaAdapter};
5049
use crate::physical_plan::{Accumulator, ExecutionPlan, Statistics};
@@ -189,16 +188,15 @@ impl FileFormat for ParquetFormat {
189188
&self,
190189
state: &SessionState,
191190
conf: FileScanConfig,
192-
filters: &[Expr],
191+
filters: Option<&Arc<dyn PhysicalExpr>>,
193192
) -> Result<Arc<dyn ExecutionPlan>> {
194193
// If enable pruning then combine the filters to build the predicate.
195194
// If disable pruning then set the predicate to None, thus readers
196195
// will not prune data based on the statistics.
197-
let predicate = if self.enable_pruning(state.config_options()) {
198-
conjunction(filters.to_vec())
199-
} else {
200-
None
201-
};
196+
let predicate = self
197+
.enable_pruning(state.config_options())
198+
.then(|| filters.cloned())
199+
.flatten();
202200

203201
Ok(Arc::new(ParquetExec::new(
204202
conf,

‎datafusion/core/src/datasource/listing/table.rs‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,10 @@ use arrow::compute::SortOptions;
2424
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
2525
use async_trait::async_trait;
2626
use dashmap::DashMap;
27+
use datafusion_common::ToDFSchema;
2728
use datafusion_expr::expr::Sort;
28-
use datafusion_physical_expr::PhysicalSortExpr;
29+
use datafusion_optimizer::utils::conjunction;
30+
use datafusion_physical_expr::{create_physical_expr, PhysicalSortExpr};
2931
use futures::{future, stream, StreamExt, TryStreamExt};
3032
use object_store::path::Path;
3133
use object_store::ObjectMeta;
@@ -661,6 +663,20 @@ impl TableProvider for ListingTable {
661663
})
662664
.collect::<Result<Vec<_>>>()?;
663665

666+
let filters = if let Some(expr) = conjunction(filters.to_vec()) {
667+
// NOTE: Use the table schema (NOT file schema) here because `expr` may contain references to partition columns.
668+
let table_df_schema = self.table_schema.as_ref().clone().to_dfschema()?;
669+
let filters = create_physical_expr(
670+
&expr,
671+
&table_df_schema,
672+
&self.table_schema,
673+
state.execution_props(),
674+
)?;
675+
Some(filters)
676+
} else {
677+
None
678+
};
679+
664680
// create the execution plan
665681
self.options
666682
.format
@@ -677,7 +693,7 @@ impl TableProvider for ListingTable {
677693
table_partition_cols,
678694
infinite_source: self.infinite_source,
679695
},
680-
filters,
696+
filters.as_ref(),
681697
)
682698
.await
683699
}

0 commit comments

Comments
 (0)