Skip to content

Commit fd21f30

Browse files
committed
Keep conjuncts in RowFilter when filter cols are not projected
When a conjunct references columns not in the output projection (e.g. COUNT(*) WHERE col = X), it cannot be evaluated as a batch filter because those columns are absent from the output schema. Keep such conjuncts in the RowFilter to avoid schema errors.
1 parent 700abfb commit fd21f30

1 file changed

Lines changed: 12 additions & 5 deletions

File tree

‎datafusion/datasource-parquet/src/row_filter.rs‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -612,19 +612,26 @@ pub fn build_row_filter(
612612

613613
match candidate {
614614
Some(c) => {
615-
// Check if RowFilter saves column decoding for this conjunct:
616-
// if there are projected columns NOT referenced by this
617-
// conjunct, RowFilter can skip decoding them for non-matching
618-
// rows.
619615
let conjunct_cols: HashSet<usize> = collect_columns(&c.expr)
620616
.iter()
621617
.map(|col| col.index())
622618
.collect();
619+
620+
// Keep in RowFilter when:
621+
// 1. There are projected columns NOT in this conjunct →
622+
// RowFilter can skip decoding them for non-matching rows.
623623
let has_extra_cols = projection_col_indices
624624
.iter()
625625
.any(|idx| !conjunct_cols.contains(idx));
626+
// 2. The conjunct references columns NOT in the output
627+
// projection (e.g. COUNT(*) WHERE col = X) → the
628+
// batch filter can't evaluate it because those
629+
// columns won't be in the output schema.
630+
let needs_non_projected = conjunct_cols
631+
.iter()
632+
.any(|idx| !projection_col_indices.contains(idx));
626633

627-
if has_extra_cols || c.can_use_index {
634+
if has_extra_cols || needs_non_projected || c.can_use_index {
628635
candidates.push(c);
629636
} else {
630637
demoted.push(c.expr);

0 commit comments

Comments
 (0)