2222//! This is shared by the opener's projection handling (via
2323//! [`build_projection_read_plan`]) and row-filter construction (via
2424//! [`crate::row_filter`]), which both need to translate column and struct
25- //! field references into Parquet leaf indices.
25+ //! field references into Parquet leaf indices. [`PushdownChecker`], the
26+ //! expression traversal that discovers those references, lives here as well
27+ //! so that [`crate::row_filter`] depends on this module and not vice versa.
2628
2729use std:: collections:: { BTreeMap , BTreeSet } ;
2830use std:: sync:: Arc ;
@@ -31,12 +33,13 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
3133use parquet:: arrow:: ProjectionMask ;
3234use parquet:: schema:: types:: SchemaDescriptor ;
3335
34- use datafusion_common:: tree_node:: TreeNode ;
35- use datafusion_physical_expr:: PhysicalExpr ;
36- use datafusion_physical_expr:: expressions:: Column ;
36+ use datafusion_common:: Result ;
37+ use datafusion_common:: tree_node:: { TreeNode , TreeNodeRecursion , TreeNodeVisitor } ;
38+ use datafusion_functions:: core:: file_row_index:: FileRowIndexFunc ;
39+ use datafusion_functions:: core:: getfield:: GetFieldFunc ;
40+ use datafusion_physical_expr:: expressions:: { Column , Literal } ;
3741use datafusion_physical_expr:: utils:: collect_columns;
38-
39- use crate :: row_filter:: PushdownChecker ;
42+ use datafusion_physical_expr:: { PhysicalExpr , ScalarFunctionExpr } ;
4043
4144/// The result of resolving which Parquet leaf columns and Arrow schema fields
4245/// are needed to evaluate an expression against a Parquet file
@@ -67,6 +70,272 @@ pub(crate) struct StructFieldAccess {
6770 pub ( crate ) field_path : Vec < String > ,
6871}
6972
73+ /// Traverses a `PhysicalExpr` tree to determine if any column references would
74+ /// prevent the expression from being pushed down to the parquet decoder.
75+ ///
76+ /// An expression cannot be pushed down if it references:
77+ /// - Unsupported nested columns (whole struct references or list fields that are
78+ /// not covered by the supported predicate set)
79+ /// - Columns that don't exist in the file schema
80+ ///
81+ /// Struct field access via `get_field` is supported when the resolved leaf type
82+ /// is primitive (e.g. `get_field(struct_col, 'field') > 5`).
83+ pub ( crate ) struct PushdownChecker < ' schema > {
84+ /// Does the expression require any non-primitive columns (like structs)?
85+ non_primitive_columns : bool ,
86+ /// Does the expression reference any columns not present in the file schema?
87+ projected_columns : bool ,
88+ /// Does the expression references a ScalarUDF that requires some rewrite
89+ /// and therefore can't be pushed down into the row-filter.
90+ has_unpushable_udfs : bool ,
91+ /// Indices into the file schema of columns required to evaluate the expression.
92+ /// Does not include struct columns accessed via `get_field`.
93+ required_columns : Vec < usize > ,
94+ /// Struct field accesses via `get_field`.
95+ struct_field_accesses : Vec < StructFieldAccess > ,
96+ /// Whether nested list columns are supported by the predicate semantics.
97+ allow_list_columns : bool ,
98+ /// The Arrow schema of the parquet file.
99+ file_schema : & ' schema Schema ,
100+ }
101+
102+ impl < ' schema > PushdownChecker < ' schema > {
103+ pub ( crate ) fn new ( file_schema : & ' schema Schema , allow_list_columns : bool ) -> Self {
104+ Self {
105+ non_primitive_columns : false ,
106+ projected_columns : false ,
107+ has_unpushable_udfs : false ,
108+ required_columns : Vec :: new ( ) ,
109+ struct_field_accesses : Vec :: new ( ) ,
110+ allow_list_columns,
111+ file_schema,
112+ }
113+ }
114+
115+ /// Checks whether a struct's root column exists in the file schema and, if so,
116+ /// records its index so the entire struct is decoded for filter evaluation.
117+ ///
118+ /// This is called when we see a `get_field` expression that resolves to a
119+ /// primitive leaf type. We only need the *root* column index because the
120+ /// Parquet reader decodes all leaves of a struct together.
121+ ///
122+ /// # Example
123+ ///
124+ /// Given file schema `{a: Int32, s: Struct(foo: Utf8, bar: Int64)}` and the
125+ /// expression `get_field(s, 'foo') = 'hello'`:
126+ ///
127+ /// - `column_name` = `"s"` (the root struct column)
128+ /// - `file_schema.index_of("s")` returns `1`
129+ /// - We push `1` into `required_columns`
130+ /// - Return `None` (no issue — traversal continues in the caller)
131+ ///
132+ /// If `"s"` is not in the file schema (e.g. a projected-away column), we set
133+ /// `projected_columns = true` and return `Jump` to skip the subtree.
134+ fn check_struct_field_column (
135+ & mut self ,
136+ column_name : & str ,
137+ field_path : Vec < String > ,
138+ ) -> Option < TreeNodeRecursion > {
139+ let Ok ( idx) = self . file_schema . index_of ( column_name) else {
140+ self . projected_columns = true ;
141+ return Some ( TreeNodeRecursion :: Jump ) ;
142+ } ;
143+
144+ self . struct_field_accesses . push ( StructFieldAccess {
145+ root_index : idx,
146+ field_path,
147+ } ) ;
148+
149+ None
150+ }
151+
152+ fn check_single_column ( & mut self , column_name : & str ) -> Option < TreeNodeRecursion > {
153+ let idx = match self . file_schema . index_of ( column_name) {
154+ Ok ( idx) => idx,
155+ Err ( _) => {
156+ // Column does not exist in the file schema, so we can't push this down.
157+ self . projected_columns = true ;
158+ return Some ( TreeNodeRecursion :: Jump ) ;
159+ }
160+ } ;
161+
162+ // Duplicates are handled by dedup() in into_sorted_columns()
163+ self . required_columns . push ( idx) ;
164+ let data_type = self . file_schema . field ( idx) . data_type ( ) ;
165+
166+ if DataType :: is_nested ( data_type) {
167+ self . handle_nested_type ( data_type)
168+ } else {
169+ None
170+ }
171+ }
172+
173+ /// Determines whether a nested data type can be pushed down to Parquet decoding.
174+ ///
175+ /// Returns `Some(TreeNodeRecursion::Jump)` if the nested type prevents pushdown,
176+ /// `None` if the type is supported and pushdown can continue.
177+ fn handle_nested_type ( & mut self , data_type : & DataType ) -> Option < TreeNodeRecursion > {
178+ if self . is_nested_type_supported ( data_type) {
179+ None
180+ } else {
181+ // Block pushdown for unsupported nested types:
182+ // - Structs (regardless of predicate support)
183+ // - Lists without supported predicates
184+ self . non_primitive_columns = true ;
185+ Some ( TreeNodeRecursion :: Jump )
186+ }
187+ }
188+
189+ /// Checks if a nested data type is supported for list column pushdown.
190+ ///
191+ /// List columns are only supported if:
192+ /// 1. The data type is a list variant (List, LargeList, or FixedSizeList)
193+ /// 2. The expression contains supported list predicates (e.g., array_has_all)
194+ fn is_nested_type_supported ( & self , data_type : & DataType ) -> bool {
195+ let is_list = matches ! (
196+ data_type,
197+ DataType :: List ( _) | DataType :: LargeList ( _) | DataType :: FixedSizeList ( _, _)
198+ ) ;
199+ self . allow_list_columns && is_list
200+ }
201+
202+ #[ inline]
203+ pub ( crate ) fn prevents_pushdown ( & self ) -> bool {
204+ self . non_primitive_columns || self . projected_columns || self . has_unpushable_udfs
205+ }
206+
207+ /// Consumes the checker and returns sorted, deduplicated column indices
208+ /// wrapped in a `PushdownColumns` struct.
209+ ///
210+ /// This method sorts the column indices and removes duplicates. The sort
211+ /// is required because downstream code relies on column indices being in
212+ /// ascending order for correct schema projection.
213+ pub ( crate ) fn into_sorted_columns ( mut self ) -> PushdownColumns {
214+ self . required_columns . sort_unstable ( ) ;
215+ self . required_columns . dedup ( ) ;
216+ PushdownColumns {
217+ required_columns : self . required_columns ,
218+ struct_field_accesses : self . struct_field_accesses ,
219+ }
220+ }
221+ }
222+
223+ impl TreeNodeVisitor < ' _ > for PushdownChecker < ' _ > {
224+ type Node = Arc < dyn PhysicalExpr > ;
225+
226+ fn f_down ( & mut self , node : & Self :: Node ) -> Result < TreeNodeRecursion > {
227+ // Handle struct field access like `s['foo']['bar'] > 10`.
228+ //
229+ // DataFusion represents nested field access as `get_field(Column("s"), "foo")`
230+ // (or chained: `get_field(get_field(Column("s"), "foo"), "bar")`).
231+ //
232+ // We intercept the outermost `get_field` on the way *down* the tree so
233+ // the visitor never reaches the raw `Column("s")` node. Without this,
234+ // `check_single_column` would see that `s` is a Struct and reject it.
235+ //
236+ // The strategy:
237+ // 1. Match `get_field` whose first arg is a `Column` (the struct root).
238+ // 2. Check that the *resolved* return type is primitive — meaning we've
239+ // drilled all the way to a leaf (e.g. `s['foo']` → Utf8).
240+ // 3. Record the root column index via `check_struct_field_column` and
241+ // return `Jump` to skip visiting the children (the Column and the
242+ // literal field-name args), since we've already handled them.
243+ //
244+ // If the return type is still nested (e.g. `s['nested_struct']` → Struct),
245+ // we fall through and let normal traversal continue, which will
246+ // eventually reject the expression when it hits the struct Column.
247+ if let Some ( func) =
248+ ScalarFunctionExpr :: try_downcast_func :: < GetFieldFunc > ( node. as_ref ( ) )
249+ {
250+ let args = func. args ( ) ;
251+
252+ if let Some ( column) = args. first ( ) . and_then ( |a| a. downcast_ref :: < Column > ( ) ) {
253+ // for Map columns, get_field performs a runtime key lookup rather than a
254+ // schema-level field access so the entire Map column must be read,
255+ // we skip the struct field optimization and defer to normal Column traversal
256+ let is_map_column = self
257+ . file_schema
258+ . index_of ( column. name ( ) )
259+ . ok ( )
260+ . map ( |idx| {
261+ matches ! (
262+ self . file_schema. field( idx) . data_type( ) ,
263+ DataType :: Map ( _, _)
264+ )
265+ } )
266+ . unwrap_or ( false ) ;
267+
268+ let return_type = func. return_type ( ) ;
269+
270+ if !is_map_column
271+ && ( !DataType :: is_nested ( return_type)
272+ || self . is_nested_type_supported ( return_type) )
273+ {
274+ // try to resolve all field name arguments to strinrg literals
275+ // if any argument is not a string literal, we can not determine the exact
276+ // leaf path so we fall back to reading the entire struct root column
277+ let field_path = args[ 1 ..]
278+ . iter ( )
279+ . map ( |arg| {
280+ arg. downcast_ref :: < Literal > ( ) . and_then ( |lit| {
281+ lit. value ( ) . try_as_str ( ) . flatten ( ) . map ( |s| s. to_string ( ) )
282+ } )
283+ } )
284+ . collect ( ) ;
285+
286+ match field_path {
287+ Some ( path) => {
288+ if let Some ( recursion) =
289+ self . check_struct_field_column ( column. name ( ) , path)
290+ {
291+ return Ok ( recursion) ;
292+ }
293+ }
294+ None => {
295+ // Could not resolve field path — fall back to
296+ // reading the entire struct root column.
297+ if let Some ( recursion) =
298+ self . check_single_column ( column. name ( ) )
299+ {
300+ return Ok ( recursion) ;
301+ }
302+ }
303+ }
304+
305+ return Ok ( TreeNodeRecursion :: Jump ) ;
306+ }
307+ }
308+ }
309+
310+ if let Some ( column) = node. downcast_ref :: < Column > ( )
311+ && let Some ( recursion) = self . check_single_column ( column. name ( ) )
312+ {
313+ return Ok ( recursion) ;
314+ }
315+
316+ if ScalarFunctionExpr :: try_downcast_func :: < FileRowIndexFunc > ( node. as_ref ( ) )
317+ . is_some ( )
318+ {
319+ self . has_unpushable_udfs = true ;
320+ return Ok ( TreeNodeRecursion :: Jump ) ;
321+ }
322+
323+ Ok ( TreeNodeRecursion :: Continue )
324+ }
325+ }
326+
327+ /// Result of checking which columns are required for filter pushdown.
328+ #[ derive( Debug ) ]
329+ pub ( crate ) struct PushdownColumns {
330+ /// Sorted, unique column indices into the file schema required to evaluate
331+ /// the filter expression. Must be in ascending order for correct schema
332+ /// projection matching. Does not include struct columns accessed via `get_field`.
333+ pub ( crate ) required_columns : Vec < usize > ,
334+ /// Struct field accesses via `get_field`. Each entry records the root struct
335+ /// column index and the field path being accessed.
336+ pub ( crate ) struct_field_accesses : Vec < StructFieldAccess > ,
337+ }
338+
70339/// Builds a unified [`ParquetReadPlan`] for a set of projection expressions
71340///
72341/// Unlike [`crate::row_filter::build_parquet_read_plan`] (which is used for
@@ -242,11 +511,7 @@ pub(crate) fn resolve_struct_field_leaves(
242511
243512 // A leaf matches if its path starts with our prefix.
244513 // e.g., prefix=["s", "value"] matches leaf path ["s", "value"]
245- // prefix=["s", "outer"] matches ["s", "outer", "inner"]
246-
247- // a leaf matches if its path starts with our prefix
248- // for example: prefix=["s", "value"] matches leaf path ["s", "value"]
249- // prefix=["s", "outer"] matches ["s", "outer", "inner"]
514+ // prefix=["s", "outer"] matches ["s", "outer", "inner"]
250515 let leaf_matches_path = col_path. len ( ) >= prefix. len ( )
251516 && col_path. iter ( ) . zip ( prefix. iter ( ) ) . all ( |( a, b) | a == b) ;
252517
0 commit comments