Skip to content

Commit 4e3eca9

Browse files
fix more regressions
1 parent 1b2931d commit 4e3eca9

1 file changed

Lines changed: 35 additions & 9 deletions

File tree

‎datafusion/datasource-parquet/src/row_filter.rs‎

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ use datafusion_common::cast::as_boolean_array;
8383
use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion, TreeNodeVisitor};
8484
use datafusion_physical_expr::ScalarFunctionExpr;
8585
use datafusion_physical_expr::expressions::{Column, Literal};
86-
use datafusion_physical_expr::utils::reassign_expr_columns;
86+
use datafusion_physical_expr::utils::{collect_columns, reassign_expr_columns};
8787
use datafusion_physical_expr::{PhysicalExpr, split_conjunction};
8888

8989
use datafusion_physical_plan::metrics;
@@ -608,9 +608,37 @@ pub(crate) fn build_projection_read_plan(
608608
file_schema: &Schema,
609609
schema_descr: &SchemaDescriptor,
610610
) -> ParquetReadPlan {
611-
// check if any column in the schema is a struct type
612-
// if not, we can skip the PushdownChecker traversal entirely and use
613-
// root-level projection which is significantly faster
611+
// fast path: if every expression is a plain Column reference, skip all
612+
// struct analysis and use root-level projection directly
613+
let exprs = exprs.into_iter().collect::<Vec<_>>();
614+
let all_plain_columns = exprs
615+
.iter()
616+
.all(|e| e.as_any().downcast_ref::<Column>().is_some());
617+
618+
if all_plain_columns {
619+
let mut root_indices: Vec<usize> = exprs
620+
.iter()
621+
.map(|e| e.as_any().downcast_ref::<Column>().unwrap().index())
622+
.collect();
623+
root_indices.sort_unstable();
624+
root_indices.dedup();
625+
626+
let projection_mask =
627+
ProjectionMask::roots(schema_descr, root_indices.iter().copied());
628+
let projected_schema = Arc::new(
629+
file_schema
630+
.project(&root_indices)
631+
.expect("valid column indices"),
632+
);
633+
634+
return ParquetReadPlan {
635+
projection_mask,
636+
projected_schema,
637+
};
638+
}
639+
640+
// secondary fast path: if the schema has no struct columns, we can skip
641+
// PushdownChecker traversal and use root-level projection
614642
let has_struct_columns = file_schema
615643
.fields()
616644
.iter()
@@ -619,17 +647,15 @@ pub(crate) fn build_projection_read_plan(
619647
if !has_struct_columns {
620648
let mut root_indices = exprs
621649
.into_iter()
622-
.flat_map(|e| {
623-
datafusion_physical_expr::utils::collect_columns(&e)
624-
.into_iter()
625-
.map(|col| col.index())
626-
})
650+
.flat_map(|e| collect_columns(&e).into_iter().map(|col| col.index()))
627651
.collect::<Vec<_>>();
652+
628653
root_indices.sort_unstable();
629654
root_indices.dedup();
630655

631656
let projection_mask =
632657
ProjectionMask::roots(schema_descr, root_indices.iter().copied());
658+
633659
let projected_schema = Arc::new(
634660
file_schema
635661
.project(&root_indices)

0 commit comments

Comments
 (0)