Skip to content

Commit d0b4c30

Browse files
committed
fix: lazily record fully matched page-index skips
1 parent 67a0526 commit d0b4c30

3 files changed

Lines changed: 34 additions & 64 deletions

File tree

‎datafusion/datasource-parquet/src/metrics.rs‎

Lines changed: 15 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ use datafusion_physical_plan::metrics::{
1919
Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, MetricCategory, MetricType,
2020
PruningMetrics, RatioMergeStrategy, RatioMetrics, Time,
2121
};
22-
use std::sync::{Arc, OnceLock};
2322

2423
/// Stores metrics about the parquet execution for a particular parquet file.
2524
///
@@ -68,11 +67,6 @@ pub struct ParquetFileMetrics {
6867
pub page_index_rows_pruned: PruningMetrics,
6968
/// Total pages filtered or matched by parquet page index
7069
pub page_index_pages_pruned: PruningMetrics,
71-
/// Lazily registered counter for pages whose page-index pruning was skipped
72-
/// because the containing row group was fully matched by row-group statistics.
73-
///
74-
/// These pages are still scanned; only page-index predicate evaluation is skipped.
75-
page_index_pages_skipped_by_fully_matched: LazyParquetSummaryCount,
7670
/// Total time spent evaluating parquet page index filters
7771
pub page_index_eval_time: Time,
7872
/// Total time spent reading and parsing metadata from the footer
@@ -210,12 +204,6 @@ impl ParquetFileMetrics {
210204
row_pushdown_eval_time,
211205
page_index_rows_pruned,
212206
page_index_pages_pruned,
213-
page_index_pages_skipped_by_fully_matched: LazyParquetSummaryCount::new(
214-
"page_index_pages_skipped_by_fully_matched",
215-
partition,
216-
filename,
217-
metrics,
218-
),
219207
statistics_eval_time,
220208
bloom_filter_eval_time,
221209
page_index_eval_time,
@@ -228,48 +216,25 @@ impl ParquetFileMetrics {
228216

229217
/// Record pages whose page-index pruning was skipped because the containing
230218
/// row group was fully matched by row-group statistics.
231-
pub(crate) fn add_page_index_pages_skipped_by_fully_matched(&self, n: usize) {
232-
if n == 0 {
233-
return;
234-
}
235-
236-
self.page_index_pages_skipped_by_fully_matched.add(n);
237-
}
238-
}
239-
240-
#[derive(Debug, Clone)]
241-
struct LazyParquetSummaryCount {
242-
count: Arc<OnceLock<Count>>,
243-
name: &'static str,
244-
partition: usize,
245-
filename: String,
246-
metrics: ExecutionPlanMetricsSet,
247-
}
248-
249-
impl LazyParquetSummaryCount {
250-
fn new(
251-
name: &'static str,
219+
///
220+
/// The counter is only registered when there is a non-zero value. This keeps
221+
/// [`ParquetFileMetrics::new`] from cloning the filename and metrics set for
222+
/// files that never use this metric.
223+
pub(crate) fn add_page_index_pages_skipped_by_fully_matched(
224+
metrics: &ExecutionPlanMetricsSet,
252225
partition: usize,
253226
filename: &str,
254-
metrics: &ExecutionPlanMetricsSet,
255-
) -> Self {
256-
Self {
257-
count: Arc::new(OnceLock::new()),
258-
name,
259-
partition,
260-
filename: filename.to_string(),
261-
metrics: metrics.clone(),
227+
n: usize,
228+
) {
229+
if n == 0 {
230+
return;
262231
}
263-
}
264232

265-
fn add(&self, n: usize) {
266-
let count = self.count.get_or_init(|| {
267-
MetricBuilder::new(&self.metrics)
268-
.with_new_label("filename", self.filename.clone())
269-
.with_type(MetricType::Summary)
270-
.with_category(MetricCategory::Rows)
271-
.counter(self.name, self.partition)
272-
});
233+
let count = MetricBuilder::new(metrics)
234+
.with_new_label("filename", filename.to_string())
235+
.with_type(MetricType::Summary)
236+
.with_category(MetricCategory::Rows)
237+
.counter("page_index_pages_skipped_by_fully_matched", partition);
273238
count.add(n);
274239
}
275240
}

‎datafusion/datasource-parquet/src/opener.rs‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1128,12 +1128,20 @@ impl RowGroupsPrunedParquetOpen {
11281128
&& !access_plan.is_empty()
11291129
&& let Some(page_pruning_predicate) = page_pruning_predicate
11301130
{
1131-
access_plan = page_pruning_predicate.prune_plan_with_page_index(
1132-
access_plan,
1133-
&prepared.physical_file_schema,
1134-
reader_metadata.parquet_schema(),
1135-
file_metadata.as_ref(),
1136-
&prepared.file_metrics,
1131+
let (page_pruned_access_plan, pages_skipped_by_fully_matched) =
1132+
page_pruning_predicate.prune_plan_with_page_index(
1133+
access_plan,
1134+
&prepared.physical_file_schema,
1135+
reader_metadata.parquet_schema(),
1136+
file_metadata.as_ref(),
1137+
&prepared.file_metrics,
1138+
);
1139+
access_plan = page_pruned_access_plan;
1140+
ParquetFileMetrics::add_page_index_pages_skipped_by_fully_matched(
1141+
&prepared.metrics,
1142+
prepared.partition_index,
1143+
&prepared.file_name,
1144+
pages_skipped_by_fully_matched,
11371145
);
11381146
}
11391147

‎datafusion/datasource-parquet/src/page_filter.rs‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -160,18 +160,18 @@ impl PagePruningAccessPlanFilter {
160160
parquet_schema: &SchemaDescriptor,
161161
parquet_metadata: &ParquetMetaData,
162162
file_metrics: &ParquetFileMetrics,
163-
) -> ParquetAccessPlan {
163+
) -> (ParquetAccessPlan, usize) {
164164
// scoped timer updates on drop
165165
let _timer_guard = file_metrics.page_index_eval_time.timer();
166166
if self.predicates.is_empty() {
167-
return access_plan;
167+
return (access_plan, 0);
168168
}
169169

170170
let page_index_predicates = &self.predicates;
171171
let groups = parquet_metadata.row_groups();
172172

173173
if groups.is_empty() {
174-
return access_plan;
174+
return (access_plan, 0);
175175
}
176176

177177
if parquet_metadata.offset_index().is_none()
@@ -182,7 +182,7 @@ impl PagePruningAccessPlanFilter {
182182
parquet_metadata.offset_index().is_some(),
183183
parquet_metadata.column_index().is_some()
184184
);
185-
return access_plan;
185+
return (access_plan, 0);
186186
};
187187

188188
// track the total number of rows that should be skipped
@@ -329,10 +329,7 @@ impl PagePruningAccessPlanFilter {
329329
file_metrics
330330
.page_index_pages_pruned
331331
.add_matched(total_pages_select);
332-
file_metrics.add_page_index_pages_skipped_by_fully_matched(
333-
total_pages_skipped_by_fully_matched,
334-
);
335-
access_plan
332+
(access_plan, total_pages_skipped_by_fully_matched)
336333
}
337334

338335
/// Returns the number of filters in the [`PagePruningAccessPlanFilter`]

0 commit comments

Comments
 (0)