Skip to content

Commit 63724fa

Browse files
friendlymatthewadriangb
authored andcommitted
fast-path projection for non-struct columns
1 parent 1723635 commit 63724fa

1 file changed

Lines changed: 55 additions & 1 deletion

File tree

‎datafusion/datasource-parquet/src/row_filter.rs‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -608,6 +608,40 @@ pub(crate) fn build_projection_read_plan(
608608
file_schema: &Schema,
609609
schema_descr: &SchemaDescriptor,
610610
) -> ParquetReadPlan {
611+
// check if any column in the schema is a struct type
612+
// if not, we can skip the PushdownChecker traversal entirely and use
613+
// root-level projection which is significantly faster
614+
let has_struct_columns = file_schema
615+
.fields()
616+
.iter()
617+
.any(|f| matches!(f.data_type(), DataType::Struct(_)));
618+
619+
if !has_struct_columns {
620+
let mut root_indices = exprs
621+
.into_iter()
622+
.flat_map(|e| {
623+
datafusion_physical_expr::utils::collect_columns(&e)
624+
.into_iter()
625+
.map(|col| col.index())
626+
})
627+
.collect::<Vec<_>>();
628+
root_indices.sort_unstable();
629+
root_indices.dedup();
630+
631+
let projection_mask =
632+
ProjectionMask::roots(schema_descr, root_indices.iter().copied());
633+
let projected_schema = Arc::new(
634+
file_schema
635+
.project(&root_indices)
636+
.expect("valid column indices"),
637+
);
638+
639+
return ParquetReadPlan {
640+
projection_mask,
641+
projected_schema,
642+
};
643+
}
644+
611645
let mut all_root_indices = Vec::new();
612646
let mut all_struct_accesses = Vec::new();
613647

@@ -623,6 +657,23 @@ pub(crate) fn build_projection_read_plan(
623657
all_root_indices.sort_unstable();
624658
all_root_indices.dedup();
625659

660+
// when no struct field accesses were found, fall back to root-level projection
661+
// to match the performance of the simple path
662+
if all_struct_accesses.is_empty() {
663+
let projection_mask =
664+
ProjectionMask::roots(schema_descr, all_root_indices.iter().copied());
665+
let projected_schema = Arc::new(
666+
file_schema
667+
.project(&all_root_indices)
668+
.expect("valid column indices"),
669+
);
670+
671+
return ParquetReadPlan {
672+
projection_mask,
673+
projected_schema,
674+
};
675+
}
676+
626677
let leaf_indices = {
627678
let mut out =
628679
leaf_indices_for_roots(all_root_indices.iter().copied(), schema_descr);
@@ -775,7 +826,10 @@ fn build_filter_schema(
775826
})
776827
.collect::<Vec<_>>();
777828

778-
Arc::new(Schema::new_with_metadata(fields, file_schema.metadata().clone()))
829+
Arc::new(Schema::new_with_metadata(
830+
fields,
831+
file_schema.metadata().clone(),
832+
))
779833
}
780834

781835
fn prune_struct_type(dt: &DataType, paths: &[&[String]]) -> DataType {

0 commit comments

Comments
 (0)