Skip to content

Commit 0828f1b

Browse files
committed
fix(parquet): split RowGroupPruner errors into creation vs evaluation
Per Copilot review on #22450: `RowGroupPruner` was using a single `predicate_creation_errors` counter for both predicate construction (`build_pruning_predicate`) AND predicate evaluation (`PruningPredicate::prune`) failures. The log message also said "Ignoring error building..." when the failure was during evaluation. This misattributed evaluation failures and made the metric semantics inconsistent with the static row-group pruning path in `RowGroupAccessPlanFilter::prune_by_statistics`, which already separates the two. `RowGroupPruner::new` now takes both counters: - `predicate_creation_errors`: bumped on `build_pruning_predicate` failures. Wired to `prepared.predicate_creation_errors` from the opener — same field the static path uses. - `predicate_evaluation_errors`: bumped on `PruningPredicate::prune` failures. Wired to `prepared.file_metrics.predicate_evaluation_errors` — same field the static `prune_by_statistics` path uses, so the two paths accumulate into a shared counter. The error log message is updated to say "evaluating" so the metric and the log agree.
1 parent f3adbeb commit 0828f1b

2 files changed

Lines changed: 32 additions & 8 deletions

File tree

‎datafusion/datasource-parquet/src/opener/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1266,6 +1266,7 @@ impl RowGroupsPrunedParquetOpen {
12661266
Arc::clone(predicate),
12671267
Arc::clone(&prepared.physical_file_schema),
12681268
Arc::clone(reader_metadata.metadata()),
1269+
prepared.predicate_creation_errors.clone(),
12691270
prepared.file_metrics.predicate_evaluation_errors.clone(),
12701271
)),
12711272
_ => None,

‎datafusion/datasource-parquet/src/push_decoder.rs‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,11 @@ pub(crate) struct RowGroupPruner {
133133
/// current generation (e.g. the predicate has no analyzable bounds);
134134
/// in that case we conservatively don't prune.
135135
pruning_predicate: Option<Arc<PruningPredicate>>,
136-
/// Metric for pruning-predicate construction failures.
136+
/// Metric for `build_pruning_predicate` failures (predicate creation).
137137
predicate_creation_errors: Count,
138+
/// Metric for `PruningPredicate::prune` failures (evaluating an
139+
/// already-built predicate against row-group statistics).
140+
predicate_evaluation_errors: Count,
138141
}
139142

140143
impl RowGroupPruner {
@@ -143,6 +146,7 @@ impl RowGroupPruner {
143146
arrow_schema: SchemaRef,
144147
parquet_metadata: Arc<ParquetMetaData>,
145148
predicate_creation_errors: Count,
149+
predicate_evaluation_errors: Count,
146150
) -> Self {
147151
Self {
148152
predicate,
@@ -151,6 +155,7 @@ impl RowGroupPruner {
151155
last_generation: None,
152156
pruning_predicate: None,
153157
predicate_creation_errors,
158+
predicate_evaluation_errors,
154159
}
155160
}
156161

@@ -203,10 +208,14 @@ impl RowGroupPruner {
203208
// every requested row group is in that state.
204209
Ok(values) => values.iter().all(|&keep| !keep),
205210
Err(e) => {
211+
// The predicate was already built successfully (we hold `pp`);
212+
// this failure is in *evaluating* it against the row-group
213+
// stats, so it belongs in the evaluation-errors counter, not
214+
// creation-errors.
206215
debug!(
207-
"Ignoring error building runtime row-group pruning predicate: {e}"
216+
"Ignoring error evaluating runtime row-group pruning predicate: {e}"
208217
);
209-
self.predicate_creation_errors.add(1);
218+
self.predicate_evaluation_errors.add(1);
210219
false
211220
}
212221
}
@@ -428,9 +437,17 @@ mod tests {
428437
(Arc::new(meta), schema)
429438
}
430439

431-
fn predicate_creation_errors_metric() -> Count {
440+
/// Create a fresh `(creation_errors, evaluation_errors)` counter pair
441+
/// for tests. The names mirror the two metrics
442+
/// [`RowGroupPruner::new`] consumes — predicate construction is
443+
/// accounted separately from per-row-group evaluation.
444+
fn pruner_error_counters() -> (Count, Count) {
432445
let metrics = ExecutionPlanMetricsSet::new();
433-
MetricBuilder::new(&metrics).counter("predicate_evaluation_errors", 0)
446+
let creation =
447+
MetricBuilder::new(&metrics).counter("num_predicate_creation_errors", 0);
448+
let evaluation =
449+
MetricBuilder::new(&metrics).counter("predicate_evaluation_errors", 0);
450+
(creation, evaluation)
434451
}
435452

436453
/// `v > literal` predicate on a single-column schema.
@@ -445,11 +462,13 @@ mod tests {
445462
#[test]
446463
fn row_group_pruner_skips_only_disqualified_row_groups() {
447464
let (meta, schema) = build_three_rg_file();
465+
let (creation, evaluation) = pruner_error_counters();
448466
let mut pruner = RowGroupPruner::new(
449467
gt_predicate(1500),
450468
Arc::clone(&schema),
451469
Arc::clone(&meta),
452-
predicate_creation_errors_metric(),
470+
creation,
471+
evaluation,
453472
);
454473

455474
// RG0 (0..1000) is entirely below threshold → fully prunable.
@@ -474,11 +493,13 @@ mod tests {
474493
vec![Arc::new(Column::new("v", 0))],
475494
gt_predicate(500),
476495
));
496+
let (creation, evaluation) = pruner_error_counters();
477497
let mut pruner = RowGroupPruner::new(
478498
Arc::clone(&dynamic) as Arc<dyn PhysicalExpr>,
479499
Arc::clone(&schema),
480500
Arc::clone(&meta),
481-
predicate_creation_errors_metric(),
501+
creation,
502+
evaluation,
482503
);
483504

484505
// Initial threshold 500 → only the lower half of RG0 fails, so RG0
@@ -517,11 +538,13 @@ mod tests {
517538
// We use `lit(true)` which produces no column references, so
518539
// `build_pruning_predicate` will return None.
519540
let (meta, schema) = build_three_rg_file();
541+
let (creation, evaluation) = pruner_error_counters();
520542
let mut pruner = RowGroupPruner::new(
521543
lit(true) as Arc<dyn PhysicalExpr>,
522544
Arc::clone(&schema),
523545
Arc::clone(&meta),
524-
predicate_creation_errors_metric(),
546+
creation,
547+
evaluation,
525548
);
526549
// No pruning predicate could be built → conservatively keep RGs.
527550
assert!(!pruner.should_prune(&[0]));

0 commit comments

Comments
 (0)