Skip to content

Commit c7628fb

Browse files
committed
Add partition filters' equivalence classes info to the execution plan if it's DataSourceExec
1 parent a13a6fe commit c7628fb

3 files changed

Lines changed: 38 additions & 4 deletions

File tree

‎datafusion/core/src/datasource/listing/table.rs‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ use crate::datasource::{
3030
physical_plan::FileSinkConfig,
3131
};
3232
use crate::execution::context::SessionState;
33+
use datafusion_catalog::memory::DataSourceExec;
3334
use datafusion_catalog::TableProvider;
3435
use datafusion_common::{config_err, DataFusionError, Result};
3536
use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder};
@@ -941,7 +942,8 @@ impl TableProvider for ListingTable {
941942
};
942943

943944
// create the execution plan
944-
self.options
945+
let mut execution_plan = self
946+
.options
945947
.format
946948
.create_physical_plan(
947949
state,
@@ -960,7 +962,24 @@ impl TableProvider for ListingTable {
960962
.build(),
961963
filters.as_ref(),
962964
)
963-
.await
965+
.await?;
966+
// Add partition filters' equivalence classes info to the execution plan if it's DataSourceExec
967+
if let Some(exec) = execution_plan.as_any().downcast_ref::<DataSourceExec>() {
968+
let partition_filter = conjunction(partition_filters);
969+
if let Some(partition_filter) = partition_filter {
970+
let table_df_schema = self.table_schema.as_ref().clone().to_dfschema()?;
971+
let partition_physical_filter = create_physical_expr(
972+
&partition_filter,
973+
&table_df_schema,
974+
state.execution_props(),
975+
)?;
976+
execution_plan =
977+
Arc::new(exec.clone().add_partition_filter_equivalence_info(
978+
partition_physical_filter,
979+
)?);
980+
}
981+
}
982+
Ok(execution_plan)
964983
}
965984

966985
fn supports_filters_pushdown(

‎datafusion/datasource/src/source.rs‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ use std::fmt::{Debug, Formatter};
2323
use std::sync::Arc;
2424

2525
use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType};
26+
use datafusion_physical_plan::filter::collect_columns_from_predicate;
2627
use datafusion_physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet};
2728
use datafusion_physical_plan::projection::ProjectionExec;
2829
use datafusion_physical_plan::{
@@ -33,7 +34,7 @@ use crate::file_scan_config::FileScanConfig;
3334
use datafusion_common::config::ConfigOptions;
3435
use datafusion_common::{Constraints, Statistics};
3536
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
36-
use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
37+
use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};
3738
use datafusion_physical_expr_common::sort_expr::LexOrdering;
3839

3940
/// Common behaviors in Data Sources for both from Files and Memory.
@@ -237,6 +238,18 @@ impl DataSourceExec {
237238
self
238239
}
239240

241+
/// Add partition filters' equivalence info
242+
pub fn add_partition_filter_equivalence_info(
243+
mut self,
244+
partition_filter: Arc<dyn PhysicalExpr>,
245+
) -> datafusion_common::Result<Self> {
246+
let (equal_pairs, _) = collect_columns_from_predicate(&partition_filter);
247+
for (lhs, rhs) in equal_pairs {
248+
self.cache.eq_properties.add_equal_conditions(lhs, rhs)?
249+
}
250+
Ok(self)
251+
}
252+
240253
fn compute_properties(data_source: Arc<dyn DataSource>) -> PlanProperties {
241254
PlanProperties::new(
242255
data_source.eq_properties(),

‎datafusion/physical-plan/src/filter.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -604,7 +604,9 @@ impl RecordBatchStream for FilterExecStream {
604604
}
605605

606606
/// Return the equals Column-Pairs and Non-equals Column-Pairs
607-
fn collect_columns_from_predicate(predicate: &Arc<dyn PhysicalExpr>) -> EqualAndNonEqual {
607+
pub fn collect_columns_from_predicate(
608+
predicate: &Arc<dyn PhysicalExpr>,
609+
) -> EqualAndNonEqual {
608610
let mut eq_predicate_columns = Vec::<PhysicalExprPairRef>::new();
609611
let mut ne_predicate_columns = Vec::<PhysicalExprPairRef>::new();
610612

0 commit comments

Comments
 (0)