Skip to content

Commit 5a173c9

Browse files
committed
Reimplement build_row_filter on top of prebuild_row_filter_candidates + row_filter_from_prebuilt
One path now splits conjuncts, orders candidates (sort_unstable_by_key on required_bytes under reorder_predicates), and wires metrics (every predicate shares pushdown_rows_pruned, only the last counts pushdown_rows_matched) — the once-per-file open path and the per-row-group RowFilterContext rebuild path can no longer drift. DatafusionArrowPredicate::try_new becomes a #[cfg(test)] convenience; the production paths construct predicates from prebuilt candidates. Folds the PR 2 piece of @adriangb's decomposition into this PR: on main the prebuilt list would have no consumer (into_builder preserves the RowFilter across runtime-prune rebuilds), so it lands together with the per-RG toggle that consumes it.
1 parent 6bf7ef9 commit 5a173c9

1 file changed

Lines changed: 18 additions & 61 deletions

File tree

‎datafusion/datasource-parquet/src/row_filter.rs‎

Lines changed: 18 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,12 @@ pub(crate) struct DatafusionArrowPredicate {
120120
}
121121

122122
impl DatafusionArrowPredicate {
123-
/// Create a new `DatafusionArrowPredicate` from a `FilterCandidate`
123+
/// Create a new `DatafusionArrowPredicate` from a `FilterCandidate`.
124+
///
125+
/// Production code goes through [`prebuild_row_filter_candidates`] +
126+
/// [`row_filter_from_prebuilt`]; this constructor remains as a test
127+
/// convenience for exercising a single candidate.
128+
#[cfg(test)]
124129
pub fn try_new(
125130
candidate: FilterCandidate,
126131
rows_pruned: metrics::Count,
@@ -401,67 +406,19 @@ pub fn build_row_filter(
401406
reorder_predicates: bool,
402407
file_metrics: &ParquetFileMetrics,
403408
) -> Result<Option<RowFilter>> {
404-
let rows_pruned = &file_metrics.pushdown_rows_pruned;
405-
let rows_matched = &file_metrics.pushdown_rows_matched;
406-
let time = &file_metrics.row_pushdown_eval_time;
407-
408-
// Split into conjuncts:
409-
// `a = 1 AND b = 2 AND c = 3` -> [`a = 1`, `b = 2`, `c = 3`]
410-
let predicates = split_conjunction(expr);
411-
412-
// Determine which conjuncts can be evaluated as ArrowPredicates, if any
413-
let mut candidates: Vec<FilterCandidate> = predicates
414-
.into_iter()
415-
.map(|expr| {
416-
FilterCandidateBuilder::new(Arc::clone(expr), Arc::clone(file_schema))
417-
.build(metadata)
418-
})
419-
.collect::<Result<Vec<_>, _>>()?
420-
.into_iter()
421-
.flatten()
422-
.collect();
423-
424-
// no candidates
425-
if candidates.is_empty() {
409+
// Implemented on top of the prebuild split so there is a single place
410+
// that splits conjuncts, orders candidates, and wires metrics — callers
411+
// that build once per file go through the same code as the per-row-group
412+
// rebuild path in `RowFilterContext`.
413+
let Some(prebuilt) = prebuild_row_filter_candidates(expr, file_schema, metadata)?
414+
else {
426415
return Ok(None);
427-
}
428-
429-
if reorder_predicates {
430-
candidates.sort_unstable_by_key(|c| c.required_bytes);
431-
}
432-
433-
// To avoid double-counting metrics when multiple predicates are used:
434-
// - All predicates should count rows_pruned (cumulative pruned rows)
435-
// - Only the last predicate should count rows_matched (final result)
436-
// This ensures: rows_matched + rows_pruned = total rows processed
437-
let total_candidates = candidates.len();
438-
439-
candidates
440-
.into_iter()
441-
.enumerate()
442-
.map(|(idx, candidate)| {
443-
let is_last = idx == total_candidates - 1;
444-
445-
// All predicates share the pruned counter (cumulative)
446-
let predicate_rows_pruned = rows_pruned.clone();
447-
448-
// Only the last predicate tracks matched rows (final result)
449-
let predicate_rows_matched = if is_last {
450-
rows_matched.clone()
451-
} else {
452-
metrics::Count::new()
453-
};
454-
455-
DatafusionArrowPredicate::try_new(
456-
candidate,
457-
predicate_rows_pruned,
458-
predicate_rows_matched,
459-
time.clone(),
460-
)
461-
.map(|pred| Box::new(pred) as _)
462-
})
463-
.collect::<Result<Vec<_>, _>>()
464-
.map(|filters| Some(RowFilter::new(filters)))
416+
};
417+
Ok(Some(row_filter_from_prebuilt(
418+
&prebuilt,
419+
reorder_predicates,
420+
file_metrics,
421+
)))
465422
}
466423

467424
/// A precomputed [`FilterCandidate`] with its expression column-reassigned to

0 commit comments

Comments
 (0)