Skip to content

Commit b34743b

Browse files
adriangbclaude
andcommitted
refactor: move PushdownChecker/PushdownColumns into projection_read_plan
Makes the imports between row_filter and projection_read_plan one-directional (row_filter -> projection_read_plan) instead of circular, and removes a duplicated comment in resolve_struct_field_leaves. Addresses review feedback. Co-Authored-By: Claude Fable 5 <[email protected]> Claude-Session: https://claude.ai/code/session_016zX5Ga66kQPkscPVf7p1AN
1 parent d4fb0c5 commit b34743b

2 files changed

Lines changed: 282 additions & 290 deletions

File tree

‎datafusion/datasource-parquet/src/projection_read_plan.rs‎

Lines changed: 276 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,9 @@
2222
//! This is shared by the opener's projection handling (via
2323
//! [`build_projection_read_plan`]) and row-filter construction (via
2424
//! [`crate::row_filter`]), which both need to translate column and struct
25-
//! field references into Parquet leaf indices.
25+
//! field references into Parquet leaf indices. [`PushdownChecker`], the
26+
//! expression traversal that discovers those references, lives here as well
27+
//! so that [`crate::row_filter`] depends on this module and not vice versa.
2628
2729
use std::collections::{BTreeMap, BTreeSet};
2830
use std::sync::Arc;
@@ -31,12 +33,13 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
3133
use parquet::arrow::ProjectionMask;
3234
use parquet::schema::types::SchemaDescriptor;
3335

34-
use datafusion_common::tree_node::TreeNode;
35-
use datafusion_physical_expr::PhysicalExpr;
36-
use datafusion_physical_expr::expressions::Column;
36+
use datafusion_common::Result;
37+
use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion, TreeNodeVisitor};
38+
use datafusion_functions::core::file_row_index::FileRowIndexFunc;
39+
use datafusion_functions::core::getfield::GetFieldFunc;
40+
use datafusion_physical_expr::expressions::{Column, Literal};
3741
use datafusion_physical_expr::utils::collect_columns;
38-
39-
use crate::row_filter::PushdownChecker;
42+
use datafusion_physical_expr::{PhysicalExpr, ScalarFunctionExpr};
4043

4144
/// The result of resolving which Parquet leaf columns and Arrow schema fields
4245
/// are needed to evaluate an expression against a Parquet file
@@ -67,6 +70,272 @@ pub(crate) struct StructFieldAccess {
6770
pub(crate) field_path: Vec<String>,
6871
}
6972

73+
/// Traverses a `PhysicalExpr` tree to determine if any column references would
74+
/// prevent the expression from being pushed down to the parquet decoder.
75+
///
76+
/// An expression cannot be pushed down if it references:
77+
/// - Unsupported nested columns (whole struct references or list fields that are
78+
/// not covered by the supported predicate set)
79+
/// - Columns that don't exist in the file schema
80+
///
81+
/// Struct field access via `get_field` is supported when the resolved leaf type
82+
/// is primitive (e.g. `get_field(struct_col, 'field') > 5`).
83+
pub(crate) struct PushdownChecker<'schema> {
84+
/// Does the expression require any non-primitive columns (like structs)?
85+
non_primitive_columns: bool,
86+
/// Does the expression reference any columns not present in the file schema?
87+
projected_columns: bool,
88+
/// Does the expression references a ScalarUDF that requires some rewrite
89+
/// and therefore can't be pushed down into the row-filter.
90+
has_unpushable_udfs: bool,
91+
/// Indices into the file schema of columns required to evaluate the expression.
92+
/// Does not include struct columns accessed via `get_field`.
93+
required_columns: Vec<usize>,
94+
/// Struct field accesses via `get_field`.
95+
struct_field_accesses: Vec<StructFieldAccess>,
96+
/// Whether nested list columns are supported by the predicate semantics.
97+
allow_list_columns: bool,
98+
/// The Arrow schema of the parquet file.
99+
file_schema: &'schema Schema,
100+
}
101+
102+
impl<'schema> PushdownChecker<'schema> {
103+
pub(crate) fn new(file_schema: &'schema Schema, allow_list_columns: bool) -> Self {
104+
Self {
105+
non_primitive_columns: false,
106+
projected_columns: false,
107+
has_unpushable_udfs: false,
108+
required_columns: Vec::new(),
109+
struct_field_accesses: Vec::new(),
110+
allow_list_columns,
111+
file_schema,
112+
}
113+
}
114+
115+
/// Checks whether a struct's root column exists in the file schema and, if so,
116+
/// records its index so the entire struct is decoded for filter evaluation.
117+
///
118+
/// This is called when we see a `get_field` expression that resolves to a
119+
/// primitive leaf type. We only need the *root* column index because the
120+
/// Parquet reader decodes all leaves of a struct together.
121+
///
122+
/// # Example
123+
///
124+
/// Given file schema `{a: Int32, s: Struct(foo: Utf8, bar: Int64)}` and the
125+
/// expression `get_field(s, 'foo') = 'hello'`:
126+
///
127+
/// - `column_name` = `"s"` (the root struct column)
128+
/// - `file_schema.index_of("s")` returns `1`
129+
/// - We push `1` into `required_columns`
130+
/// - Return `None` (no issue — traversal continues in the caller)
131+
///
132+
/// If `"s"` is not in the file schema (e.g. a projected-away column), we set
133+
/// `projected_columns = true` and return `Jump` to skip the subtree.
134+
fn check_struct_field_column(
135+
&mut self,
136+
column_name: &str,
137+
field_path: Vec<String>,
138+
) -> Option<TreeNodeRecursion> {
139+
let Ok(idx) = self.file_schema.index_of(column_name) else {
140+
self.projected_columns = true;
141+
return Some(TreeNodeRecursion::Jump);
142+
};
143+
144+
self.struct_field_accesses.push(StructFieldAccess {
145+
root_index: idx,
146+
field_path,
147+
});
148+
149+
None
150+
}
151+
152+
fn check_single_column(&mut self, column_name: &str) -> Option<TreeNodeRecursion> {
153+
let idx = match self.file_schema.index_of(column_name) {
154+
Ok(idx) => idx,
155+
Err(_) => {
156+
// Column does not exist in the file schema, so we can't push this down.
157+
self.projected_columns = true;
158+
return Some(TreeNodeRecursion::Jump);
159+
}
160+
};
161+
162+
// Duplicates are handled by dedup() in into_sorted_columns()
163+
self.required_columns.push(idx);
164+
let data_type = self.file_schema.field(idx).data_type();
165+
166+
if DataType::is_nested(data_type) {
167+
self.handle_nested_type(data_type)
168+
} else {
169+
None
170+
}
171+
}
172+
173+
/// Determines whether a nested data type can be pushed down to Parquet decoding.
174+
///
175+
/// Returns `Some(TreeNodeRecursion::Jump)` if the nested type prevents pushdown,
176+
/// `None` if the type is supported and pushdown can continue.
177+
fn handle_nested_type(&mut self, data_type: &DataType) -> Option<TreeNodeRecursion> {
178+
if self.is_nested_type_supported(data_type) {
179+
None
180+
} else {
181+
// Block pushdown for unsupported nested types:
182+
// - Structs (regardless of predicate support)
183+
// - Lists without supported predicates
184+
self.non_primitive_columns = true;
185+
Some(TreeNodeRecursion::Jump)
186+
}
187+
}
188+
189+
/// Checks if a nested data type is supported for list column pushdown.
190+
///
191+
/// List columns are only supported if:
192+
/// 1. The data type is a list variant (List, LargeList, or FixedSizeList)
193+
/// 2. The expression contains supported list predicates (e.g., array_has_all)
194+
fn is_nested_type_supported(&self, data_type: &DataType) -> bool {
195+
let is_list = matches!(
196+
data_type,
197+
DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _)
198+
);
199+
self.allow_list_columns && is_list
200+
}
201+
202+
#[inline]
203+
pub(crate) fn prevents_pushdown(&self) -> bool {
204+
self.non_primitive_columns || self.projected_columns || self.has_unpushable_udfs
205+
}
206+
207+
/// Consumes the checker and returns sorted, deduplicated column indices
208+
/// wrapped in a `PushdownColumns` struct.
209+
///
210+
/// This method sorts the column indices and removes duplicates. The sort
211+
/// is required because downstream code relies on column indices being in
212+
/// ascending order for correct schema projection.
213+
pub(crate) fn into_sorted_columns(mut self) -> PushdownColumns {
214+
self.required_columns.sort_unstable();
215+
self.required_columns.dedup();
216+
PushdownColumns {
217+
required_columns: self.required_columns,
218+
struct_field_accesses: self.struct_field_accesses,
219+
}
220+
}
221+
}
222+
223+
impl TreeNodeVisitor<'_> for PushdownChecker<'_> {
224+
type Node = Arc<dyn PhysicalExpr>;
225+
226+
fn f_down(&mut self, node: &Self::Node) -> Result<TreeNodeRecursion> {
227+
// Handle struct field access like `s['foo']['bar'] > 10`.
228+
//
229+
// DataFusion represents nested field access as `get_field(Column("s"), "foo")`
230+
// (or chained: `get_field(get_field(Column("s"), "foo"), "bar")`).
231+
//
232+
// We intercept the outermost `get_field` on the way *down* the tree so
233+
// the visitor never reaches the raw `Column("s")` node. Without this,
234+
// `check_single_column` would see that `s` is a Struct and reject it.
235+
//
236+
// The strategy:
237+
// 1. Match `get_field` whose first arg is a `Column` (the struct root).
238+
// 2. Check that the *resolved* return type is primitive — meaning we've
239+
// drilled all the way to a leaf (e.g. `s['foo']` → Utf8).
240+
// 3. Record the root column index via `check_struct_field_column` and
241+
// return `Jump` to skip visiting the children (the Column and the
242+
// literal field-name args), since we've already handled them.
243+
//
244+
// If the return type is still nested (e.g. `s['nested_struct']` → Struct),
245+
// we fall through and let normal traversal continue, which will
246+
// eventually reject the expression when it hits the struct Column.
247+
if let Some(func) =
248+
ScalarFunctionExpr::try_downcast_func::<GetFieldFunc>(node.as_ref())
249+
{
250+
let args = func.args();
251+
252+
if let Some(column) = args.first().and_then(|a| a.downcast_ref::<Column>()) {
253+
// for Map columns, get_field performs a runtime key lookup rather than a
254+
// schema-level field access so the entire Map column must be read,
255+
// we skip the struct field optimization and defer to normal Column traversal
256+
let is_map_column = self
257+
.file_schema
258+
.index_of(column.name())
259+
.ok()
260+
.map(|idx| {
261+
matches!(
262+
self.file_schema.field(idx).data_type(),
263+
DataType::Map(_, _)
264+
)
265+
})
266+
.unwrap_or(false);
267+
268+
let return_type = func.return_type();
269+
270+
if !is_map_column
271+
&& (!DataType::is_nested(return_type)
272+
|| self.is_nested_type_supported(return_type))
273+
{
274+
// try to resolve all field name arguments to strinrg literals
275+
// if any argument is not a string literal, we can not determine the exact
276+
// leaf path so we fall back to reading the entire struct root column
277+
let field_path = args[1..]
278+
.iter()
279+
.map(|arg| {
280+
arg.downcast_ref::<Literal>().and_then(|lit| {
281+
lit.value().try_as_str().flatten().map(|s| s.to_string())
282+
})
283+
})
284+
.collect();
285+
286+
match field_path {
287+
Some(path) => {
288+
if let Some(recursion) =
289+
self.check_struct_field_column(column.name(), path)
290+
{
291+
return Ok(recursion);
292+
}
293+
}
294+
None => {
295+
// Could not resolve field path — fall back to
296+
// reading the entire struct root column.
297+
if let Some(recursion) =
298+
self.check_single_column(column.name())
299+
{
300+
return Ok(recursion);
301+
}
302+
}
303+
}
304+
305+
return Ok(TreeNodeRecursion::Jump);
306+
}
307+
}
308+
}
309+
310+
if let Some(column) = node.downcast_ref::<Column>()
311+
&& let Some(recursion) = self.check_single_column(column.name())
312+
{
313+
return Ok(recursion);
314+
}
315+
316+
if ScalarFunctionExpr::try_downcast_func::<FileRowIndexFunc>(node.as_ref())
317+
.is_some()
318+
{
319+
self.has_unpushable_udfs = true;
320+
return Ok(TreeNodeRecursion::Jump);
321+
}
322+
323+
Ok(TreeNodeRecursion::Continue)
324+
}
325+
}
326+
327+
/// Result of checking which columns are required for filter pushdown.
328+
#[derive(Debug)]
329+
pub(crate) struct PushdownColumns {
330+
/// Sorted, unique column indices into the file schema required to evaluate
331+
/// the filter expression. Must be in ascending order for correct schema
332+
/// projection matching. Does not include struct columns accessed via `get_field`.
333+
pub(crate) required_columns: Vec<usize>,
334+
/// Struct field accesses via `get_field`. Each entry records the root struct
335+
/// column index and the field path being accessed.
336+
pub(crate) struct_field_accesses: Vec<StructFieldAccess>,
337+
}
338+
70339
/// Builds a unified [`ParquetReadPlan`] for a set of projection expressions
71340
///
72341
/// Unlike [`crate::row_filter::build_parquet_read_plan`] (which is used for
@@ -242,11 +511,7 @@ pub(crate) fn resolve_struct_field_leaves(
242511

243512
// A leaf matches if its path starts with our prefix.
244513
// e.g., prefix=["s", "value"] matches leaf path ["s", "value"]
245-
// prefix=["s", "outer"] matches ["s", "outer", "inner"]
246-
247-
// a leaf matches if its path starts with our prefix
248-
// for example: prefix=["s", "value"] matches leaf path ["s", "value"]
249-
// prefix=["s", "outer"] matches ["s", "outer", "inner"]
514+
// prefix=["s", "outer"] matches ["s", "outer", "inner"]
250515
let leaf_matches_path = col_path.len() >= prefix.len()
251516
&& col_path.iter().zip(prefix.iter()).all(|(a, b)| a == b);
252517

0 commit comments

Comments
 (0)