Skip to content

Commit 20bea80

Browse files
Vaibhav Bodhaneclaude
andcommitted
fix(parquet): register one set of ParquetFileMetrics per opened file
The parquet opener registered a set of ParquetFileMetrics in prepare_open_file, the reader factory registered another for the reader it built, and the replacement reader created after Bloom filter pruning registered a third. Each set is about 20 filename-labelled metrics that the plan keeps until the query ends. Add ParquetFileReaderFactory::create_reader_with_file_metrics, a provided method whose default calls create_reader, and implement it in the two built-in factories. The opener passes the file's existing metrics, so each file registers one set. Reported names and totals are unchanged. Part of apache#22189. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent d4c2a32 commit 20bea80

2 files changed

Lines changed: 126 additions & 8 deletions

File tree

‎datafusion/datasource-parquet/src/opener/mod.rs‎

Lines changed: 68 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -839,12 +839,16 @@ impl ParquetMorselizer {
839839
.metadata_size_hint
840840
.or(self.metadata_size_hint);
841841

842-
let async_file_reader: Box<dyn AsyncFileReader> =
843-
self.parquet_file_reader_factory.create_reader(
842+
// The reader records into this file's metrics rather than registering
843+
// its own set, so each opened file registers one set of file metrics.
844+
let async_file_reader: Box<dyn AsyncFileReader> = self
845+
.parquet_file_reader_factory
846+
.create_reader_with_file_metrics(
844847
self.partition_index,
845848
partitioned_file.clone(),
846849
metadata_size_hint,
847850
&self.metrics,
851+
file_metrics.clone(),
848852
)?;
849853

850854
// Calculate the output schema from the original projection (before literal replacement)
@@ -1504,12 +1508,15 @@ impl RowGroupsPrunedParquetOpen {
15041508
let reader_metadata = self.prepared.loaded.reader_metadata.clone();
15051509
let replacement_reader = {
15061510
let prepared = &self.prepared.loaded.prepared;
1507-
prepared.parquet_file_reader_factory.create_reader(
1508-
prepared.partition_index,
1509-
prepared.partitioned_file.clone(),
1510-
prepared.metadata_size_hint,
1511-
&prepared.metrics,
1512-
)?
1511+
prepared
1512+
.parquet_file_reader_factory
1513+
.create_reader_with_file_metrics(
1514+
prepared.partition_index,
1515+
prepared.partitioned_file.clone(),
1516+
prepared.metadata_size_hint,
1517+
&prepared.metrics,
1518+
prepared.file_metrics.clone(),
1519+
)?
15131520
};
15141521

15151522
let prepared = &mut self.prepared.loaded.prepared;
@@ -4880,6 +4887,59 @@ mod test {
48804887
assert_eq!(matched, 0);
48814888
}
48824889

4890+
/// Each file the opener reads registers one set of file metrics, shared by
4891+
/// the opener and the readers it creates, including the replacement reader
4892+
/// built after Bloom filter pruning.
4893+
#[tokio::test]
4894+
async fn opening_a_file_registers_one_set_of_file_metrics() {
4895+
let store = Arc::new(InMemory::new()) as Arc<dyn ObjectStore>;
4896+
let batches = vec![
4897+
record_batch!(("a", Int32, vec![1, 1, 1])).unwrap(),
4898+
record_batch!(("a", Int32, vec![0, 1, 2])).unwrap(),
4899+
];
4900+
let schema = batches[0].schema();
4901+
let props = WriterProperties::builder()
4902+
.set_max_row_group_row_count(Some(3))
4903+
.set_bloom_filter_enabled(true)
4904+
.set_statistics_enabled(EnabledStatistics::Chunk)
4905+
.build();
4906+
let data_size = write_parquet_batches(
4907+
Arc::clone(&store),
4908+
"one-set.parquet",
4909+
batches,
4910+
Some(props),
4911+
)
4912+
.await;
4913+
let file = PartitionedFile::new("one-set.parquet".to_string(), data_size as u64);
4914+
let predicate = logical2physical(&col("a").eq(lit(1)), &schema);
4915+
4916+
let metrics = ExecutionPlanMetricsSet::new();
4917+
let morselizer = ParquetMorselizerBuilder::new()
4918+
.with_store(Arc::clone(&store))
4919+
.with_schema(Arc::clone(&schema))
4920+
.with_predicate(predicate)
4921+
.with_pushdown_filters(true)
4922+
.with_row_group_stats_pruning(false)
4923+
.with_enable_bloom_filter(true)
4924+
.with_metrics(metrics.clone())
4925+
.build();
4926+
let stream = open_file(&morselizer, file).await.unwrap();
4927+
assert_eq!(collect_int32_values(stream).await, vec![1, 1, 1, 1]);
4928+
4929+
let registered = |name: &str| {
4930+
metrics
4931+
.clone_inner()
4932+
.iter()
4933+
.filter(|metric| metric.value().name() == name)
4934+
.count()
4935+
};
4936+
// Both row groups are checked against the Bloom filter, so the opener
4937+
// also builds a replacement reader; neither reader registers its own set.
4938+
assert_eq!(registered("bytes_scanned"), 1);
4939+
assert_eq!(registered("metadata_load_time"), 1);
4940+
assert!(counter_metric_value(&metrics, "bytes_scanned") > 0);
4941+
}
4942+
48834943
#[test]
48844944
fn should_load_page_index_with_row_selection() {
48854945
let mut plan = ParquetAccessPlan::new_all(1);

‎datafusion/datasource-parquet/src/reader.rs‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,32 @@ pub trait ParquetFileReaderFactory: Debug + Send + Sync + 'static {
7777
metadata_size_hint: Option<usize>,
7878
metrics: &ExecutionPlanMetricsSet,
7979
) -> datafusion_common::Result<Box<dyn AsyncFileReader + Send>>;
80+
81+
/// Like [`Self::create_reader`], but the reader should record into
82+
/// `file_metrics`, the [`ParquetFileMetrics`] the caller has already
83+
/// registered for this file, rather than registering another set in
84+
/// `metrics`.
85+
///
86+
/// The parquet opener calls this so that each file it opens registers one
87+
/// set of file metrics. The default implementation ignores `file_metrics`
88+
/// and calls [`Self::create_reader`], so existing implementations keep
89+
/// their behavior.
90+
fn create_reader_with_file_metrics(
91+
&self,
92+
partition_index: usize,
93+
partitioned_file: PartitionedFile,
94+
metadata_size_hint: Option<usize>,
95+
metrics: &ExecutionPlanMetricsSet,
96+
file_metrics: ParquetFileMetrics,
97+
) -> datafusion_common::Result<Box<dyn AsyncFileReader + Send>> {
98+
let _ = file_metrics;
99+
self.create_reader(
100+
partition_index,
101+
partitioned_file,
102+
metadata_size_hint,
103+
metrics,
104+
)
105+
}
80106
}
81107

82108
/// Default implementation of [`ParquetFileReaderFactory`]
@@ -110,7 +136,23 @@ impl ParquetFileReaderFactory for DefaultParquetFileReaderFactory {
110136
partitioned_file.object_meta.location.as_ref(),
111137
metrics,
112138
);
139+
self.create_reader_with_file_metrics(
140+
partition_index,
141+
partitioned_file,
142+
metadata_size_hint,
143+
metrics,
144+
file_metrics,
145+
)
146+
}
113147

148+
fn create_reader_with_file_metrics(
149+
&self,
150+
_partition_index: usize,
151+
partitioned_file: PartitionedFile,
152+
metadata_size_hint: Option<usize>,
153+
_metrics: &ExecutionPlanMetricsSet,
154+
file_metrics: ParquetFileMetrics,
155+
) -> datafusion_common::Result<Box<dyn AsyncFileReader + Send>> {
114156
let reader = ParquetFileReader::new(
115157
file_metrics,
116158
Arc::clone(&self.store),
@@ -158,7 +200,23 @@ impl ParquetFileReaderFactory for CachedParquetFileReaderFactory {
158200
partitioned_file.object_meta.location.as_ref(),
159201
metrics,
160202
);
203+
self.create_reader_with_file_metrics(
204+
partition_index,
205+
partitioned_file,
206+
metadata_size_hint,
207+
metrics,
208+
file_metrics,
209+
)
210+
}
161211

212+
fn create_reader_with_file_metrics(
213+
&self,
214+
_partition_index: usize,
215+
partitioned_file: PartitionedFile,
216+
metadata_size_hint: Option<usize>,
217+
_metrics: &ExecutionPlanMetricsSet,
218+
file_metrics: ParquetFileMetrics,
219+
) -> datafusion_common::Result<Box<dyn AsyncFileReader + Send>> {
162220
let reader = ParquetFileReader::new(
163221
file_metrics,
164222
Arc::clone(&self.store),

0 commit comments

Comments
 (0)