Skip to content

Commit ea255a6

Browse files
committed
Clarify null count handling in row group pruning
1 parent 8b002d9 commit ea255a6

1 file changed

Lines changed: 8 additions & 0 deletions

File tree

‎datafusion/datasource-parquet/src/row_group_filter.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,10 @@ impl RowGroupAccessPlanFilter {
274274
parquet_schema,
275275
row_group_metadatas,
276276
arrow_schema,
277+
// Preserve the existing row-group pruning behavior. This path only
278+
// proves whether matching rows may exist, so it uses the
279+
// StatisticsConverter default for older parquet-rs files where a
280+
// missing null count can mean there are zero nulls.
277281
missing_null_counts_as_zero: true,
278282
};
279283

@@ -376,6 +380,10 @@ impl RowGroupAccessPlanFilter {
376380
.map(|&i| &groups[i])
377381
.collect::<Vec<_>>(),
378382
arrow_schema,
383+
// Fully matched row groups require a stronger proof: every row
384+
// must pass the predicate. Missing null counts are unknown here;
385+
// treating them as zero can incorrectly mark nullable row groups as
386+
// fully matched and make limit pruning unsound.
379387
missing_null_counts_as_zero: false,
380388
};
381389

0 commit comments

Comments
 (0)