Skip to content

Commit b0015d3

Browse files
authored
feat(parquet): Implement scan_efficiency_ratio metric for parquet reading (#18577)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Part of #18195 ## Rationale for this change <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> ## What changes are included in this PR? <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> ## Are these changes tested? <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> Added test ## Are there any user-facing changes? <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> Yes, new metric to view. <!-- If there are any breaking changes to public APIs, please add the `api change` label. -->
1 parent 6cb3d7e commit b0015d3

7 files changed

Lines changed: 180 additions & 13 deletions

File tree

‎datafusion/core/tests/sql/explain_analyze.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -887,6 +887,7 @@ async fn parquet_explain_analyze() {
887887
&formatted,
888888
"row_groups_pruned_statistics=1 total \u{2192} 1 matched"
889889
);
890+
assert_contains!(&formatted, "scan_efficiency_ratio=14% (259/1851)");
890891

891892
// The order of metrics is expected to be the same as the actual pruning order
892893
// (file-> row-group -> page)

‎datafusion/datasource-parquet/src/metrics.rs‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
// under the License.
1717

1818
use datafusion_physical_plan::metrics::{
19-
Count, ExecutionPlanMetricsSet, MetricBuilder, MetricType, PruningMetrics, Time,
19+
Count, ExecutionPlanMetricsSet, MetricBuilder, MetricType, PruningMetrics,
20+
RatioMergeStrategy, RatioMetrics, Time,
2021
};
2122

2223
/// Stores metrics about the parquet execution for a particular parquet file.
@@ -66,6 +67,8 @@ pub struct ParquetFileMetrics {
6667
pub page_index_eval_time: Time,
6768
/// Total time spent reading and parsing metadata from the footer
6869
pub metadata_load_time: Time,
70+
/// Scan Efficiency Ratio, calculated as bytes_scanned / total_file_size
71+
pub scan_efficiency_ratio: RatioMetrics,
6972
/// Predicate Cache: number of records read directly from the inner reader.
7073
/// This is the number of rows decoded while evaluating predicates
7174
pub predicate_cache_inner_records: Count,
@@ -114,6 +117,15 @@ impl ParquetFileMetrics {
114117
.with_type(MetricType::SUMMARY)
115118
.pruning_metrics("files_ranges_pruned_statistics", partition);
116119

120+
let scan_efficiency_ratio = MetricBuilder::new(metrics)
121+
.with_new_label("filename", filename.to_string())
122+
.with_type(MetricType::SUMMARY)
123+
.ratio_metrics_with_strategy(
124+
"scan_efficiency_ratio",
125+
partition,
126+
RatioMergeStrategy::AddPartSetTotal,
127+
);
128+
117129
// -----------------------
118130
// 'dev' level metrics
119131
// -----------------------
@@ -164,6 +176,7 @@ impl ParquetFileMetrics {
164176
bloom_filter_eval_time,
165177
page_index_eval_time,
166178
metadata_load_time,
179+
scan_efficiency_ratio,
167180
predicate_cache_inner_records,
168181
predicate_cache_records,
169182
}

‎datafusion/datasource-parquet/src/reader.rs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ impl DefaultParquetFileReaderFactory {
9797
pub struct ParquetFileReader {
9898
pub file_metrics: ParquetFileMetrics,
9999
pub inner: ParquetObjectReader,
100+
pub partitioned_file: PartitionedFile,
100101
}
101102

102103
impl AsyncFileReader for ParquetFileReader {
@@ -129,6 +130,18 @@ impl AsyncFileReader for ParquetFileReader {
129130
}
130131
}
131132

133+
impl Drop for ParquetFileReader {
134+
fn drop(&mut self) {
135+
self.file_metrics
136+
.scan_efficiency_ratio
137+
.add_part(self.file_metrics.bytes_scanned.value());
138+
// Multiple ParquetFileReaders may run, so we set_total to avoid adding the total multiple times
139+
self.file_metrics
140+
.scan_efficiency_ratio
141+
.set_total(self.partitioned_file.object_meta.size as usize);
142+
}
143+
}
144+
132145
impl ParquetFileReaderFactory for DefaultParquetFileReaderFactory {
133146
fn create_reader(
134147
&self,
@@ -156,6 +169,7 @@ impl ParquetFileReaderFactory for DefaultParquetFileReaderFactory {
156169
Ok(Box::new(ParquetFileReader {
157170
inner,
158171
file_metrics,
172+
partitioned_file,
159173
}))
160174
}
161175
}
@@ -306,6 +320,18 @@ impl AsyncFileReader for CachedParquetFileReader {
306320
}
307321
}
308322

323+
impl Drop for CachedParquetFileReader {
324+
fn drop(&mut self) {
325+
self.file_metrics
326+
.scan_efficiency_ratio
327+
.add_part(self.file_metrics.bytes_scanned.value());
328+
// Multiple ParquetFileReaders may run, so we set_total to avoid adding the total multiple times
329+
self.file_metrics
330+
.scan_efficiency_ratio
331+
.set_total(self.partitioned_file.object_meta.size as usize);
332+
}
333+
}
334+
309335
/// Wrapper to implement [`FileMetadata`] for [`ParquetMetaData`].
310336
pub struct CachedParquetMetaData(Arc<ParquetMetaData>);
311337

‎datafusion/datasource-parquet/src/row_group_filter.rs‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1533,6 +1533,7 @@ mod tests {
15331533
data: bytes::Bytes,
15341534
pruning_predicate: &PruningPredicate,
15351535
) -> Result<RowGroupAccessPlanFilter> {
1536+
use datafusion_datasource::PartitionedFile;
15361537
use object_store::{ObjectMeta, ObjectStore};
15371538

15381539
let object_meta = ObjectMeta {
@@ -1551,12 +1552,23 @@ mod tests {
15511552
let metrics = ExecutionPlanMetricsSet::new();
15521553
let file_metrics =
15531554
ParquetFileMetrics::new(0, object_meta.location.as_ref(), &metrics);
1554-
let inner = ParquetObjectReader::new(Arc::new(in_memory), object_meta.location)
1555-
.with_file_size(object_meta.size);
1555+
let inner =
1556+
ParquetObjectReader::new(Arc::new(in_memory), object_meta.location.clone())
1557+
.with_file_size(object_meta.size);
1558+
1559+
let partitioned_file = PartitionedFile {
1560+
object_meta,
1561+
partition_values: vec![],
1562+
range: None,
1563+
statistics: None,
1564+
extensions: None,
1565+
metadata_size_hint: None,
1566+
};
15561567

15571568
let reader = ParquetFileReader {
15581569
inner,
15591570
file_metrics: file_metrics.clone(),
1571+
partitioned_file,
15601572
};
15611573
let mut builder = ParquetRecordBatchStreamBuilder::new(reader).await.unwrap();
15621574

‎datafusion/physical-plan/src/metrics/builder.rs‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
use std::{borrow::Cow, sync::Arc};
2121

2222
use crate::metrics::{
23-
value::{PruningMetrics, RatioMetrics},
23+
value::{PruningMetrics, RatioMergeStrategy, RatioMetrics},
2424
MetricType,
2525
};
2626

@@ -283,7 +283,17 @@ impl<'a> MetricBuilder<'a> {
283283
name: impl Into<Cow<'static, str>>,
284284
partition: usize,
285285
) -> RatioMetrics {
286-
let ratio_metrics = RatioMetrics::new();
286+
self.ratio_metrics_with_strategy(name, partition, RatioMergeStrategy::default())
287+
}
288+
289+
/// Consumes self and creates a new [`RatioMetrics`] with a specific merge strategy
290+
pub fn ratio_metrics_with_strategy(
291+
self,
292+
name: impl Into<Cow<'static, str>>,
293+
partition: usize,
294+
merge_strategy: RatioMergeStrategy,
295+
) -> RatioMetrics {
296+
let ratio_metrics = RatioMetrics::new().with_merge_strategy(merge_strategy);
287297
self.with_partition(partition).build(MetricValue::Ratio {
288298
name: name.into(),
289299
ratio_metrics: ratio_metrics.clone(),

‎datafusion/physical-plan/src/metrics/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ pub use baseline::{BaselineMetrics, RecordOutput, SpillMetrics, SplitMetrics};
3636
pub use builder::MetricBuilder;
3737
pub use custom::CustomMetricValue;
3838
pub use value::{
39-
Count, Gauge, MetricValue, PruningMetrics, RatioMetrics, ScopedTimerGuard, Time,
40-
Timestamp,
39+
Count, Gauge, MetricValue, PruningMetrics, RatioMergeStrategy, RatioMetrics,
40+
ScopedTimerGuard, Time, Timestamp,
4141
};
4242

4343
/// Something that tracks a value of interest (metric) of a DataFusion

‎datafusion/physical-plan/src/metrics/value.rs‎

Lines changed: 111 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -437,6 +437,15 @@ impl PruningMetrics {
437437
pub struct RatioMetrics {
438438
part: Arc<AtomicUsize>,
439439
total: Arc<AtomicUsize>,
440+
merge_strategy: RatioMergeStrategy,
441+
}
442+
443+
#[derive(Debug, Clone, Default)]
444+
pub enum RatioMergeStrategy {
445+
#[default]
446+
AddPartAddTotal,
447+
AddPartSetTotal,
448+
SetPartAddTotal,
440449
}
441450

442451
impl RatioMetrics {
@@ -445,9 +454,15 @@ impl RatioMetrics {
445454
Self {
446455
part: Arc::new(AtomicUsize::new(0)),
447456
total: Arc::new(AtomicUsize::new(0)),
457+
merge_strategy: RatioMergeStrategy::AddPartAddTotal,
448458
}
449459
}
450460

461+
pub fn with_merge_strategy(mut self, merge_strategy: RatioMergeStrategy) -> Self {
462+
self.merge_strategy = merge_strategy;
463+
self
464+
}
465+
451466
/// Add `n` to the numerator (`part`) value
452467
pub fn add_part(&self, n: usize) {
453468
self.part.fetch_add(n, Ordering::Relaxed);
@@ -458,10 +473,32 @@ impl RatioMetrics {
458473
self.total.fetch_add(n, Ordering::Relaxed);
459474
}
460475

476+
/// Set the numerator (`part`) value to `n`, overwriting any existing value
477+
pub fn set_part(&self, n: usize) {
478+
self.part.store(n, Ordering::Relaxed);
479+
}
480+
481+
/// Set the denominator (`total`) value to `n`, overwriting any existing value
482+
pub fn set_total(&self, n: usize) {
483+
self.total.store(n, Ordering::Relaxed);
484+
}
485+
461486
/// Merge the value from `other` into `self`
462487
pub fn merge(&self, other: &Self) {
463-
self.add_part(other.part());
464-
self.add_total(other.total());
488+
match self.merge_strategy {
489+
RatioMergeStrategy::AddPartAddTotal => {
490+
self.add_part(other.part());
491+
self.add_total(other.total());
492+
}
493+
RatioMergeStrategy::AddPartSetTotal => {
494+
self.add_part(other.part());
495+
self.set_total(other.total());
496+
}
497+
RatioMergeStrategy::SetPartAddTotal => {
498+
self.set_part(other.part());
499+
self.add_total(other.total());
500+
}
501+
}
465502
}
466503

467504
/// Return the numerator (`part`) value
@@ -784,10 +821,17 @@ impl MetricValue {
784821
name: name.clone(),
785822
pruning_metrics: PruningMetrics::new(),
786823
},
787-
Self::Ratio { name, .. } => Self::Ratio {
788-
name: name.clone(),
789-
ratio_metrics: RatioMetrics::new(),
790-
},
824+
Self::Ratio {
825+
name,
826+
ratio_metrics,
827+
} => {
828+
let merge_strategy = ratio_metrics.merge_strategy.clone();
829+
Self::Ratio {
830+
name: name.clone(),
831+
ratio_metrics: RatioMetrics::new()
832+
.with_merge_strategy(merge_strategy),
833+
}
834+
}
791835
Self::Custom { name, value } => Self::Custom {
792836
name: name.clone(),
793837
value: value.new_empty(),
@@ -1140,6 +1184,67 @@ mod tests {
11401184
assert_eq!("0.033% (1/3000)", tiny_ratio.to_string());
11411185
}
11421186

1187+
#[test]
1188+
fn test_ratio_set_methods() {
1189+
let ratio_metrics = RatioMetrics::new();
1190+
1191+
// Ensure set methods don't increment
1192+
ratio_metrics.set_part(10);
1193+
ratio_metrics.set_part(10);
1194+
ratio_metrics.set_total(40);
1195+
ratio_metrics.set_total(40);
1196+
assert_eq!("25% (10/40)", ratio_metrics.to_string());
1197+
1198+
let ratio_metrics = RatioMetrics::new();
1199+
1200+
// Calling set should change the value
1201+
ratio_metrics.set_part(10);
1202+
ratio_metrics.set_part(30);
1203+
ratio_metrics.set_total(40);
1204+
ratio_metrics.set_total(50);
1205+
assert_eq!("60% (30/50)", ratio_metrics.to_string());
1206+
}
1207+
1208+
#[test]
1209+
fn test_ratio_merge_strategy() {
1210+
// Test AddPartSetTotal strategy
1211+
let ratio_metrics1 =
1212+
RatioMetrics::new().with_merge_strategy(RatioMergeStrategy::AddPartSetTotal);
1213+
1214+
ratio_metrics1.set_part(10);
1215+
ratio_metrics1.set_total(40);
1216+
assert_eq!("25% (10/40)", ratio_metrics1.to_string());
1217+
let ratio_metrics2 =
1218+
RatioMetrics::new().with_merge_strategy(RatioMergeStrategy::AddPartSetTotal);
1219+
ratio_metrics2.set_part(20);
1220+
ratio_metrics2.set_total(40);
1221+
assert_eq!("50% (20/40)", ratio_metrics2.to_string());
1222+
1223+
ratio_metrics1.merge(&ratio_metrics2);
1224+
assert_eq!("75% (30/40)", ratio_metrics1.to_string());
1225+
1226+
// Test SetPartAddTotal strategy
1227+
let ratio_metrics1 =
1228+
RatioMetrics::new().with_merge_strategy(RatioMergeStrategy::SetPartAddTotal);
1229+
ratio_metrics1.set_part(20);
1230+
ratio_metrics1.set_total(50);
1231+
let ratio_metrics2 = RatioMetrics::new();
1232+
ratio_metrics2.set_part(20);
1233+
ratio_metrics2.set_total(50);
1234+
ratio_metrics1.merge(&ratio_metrics2);
1235+
assert_eq!("20% (20/100)", ratio_metrics1.to_string());
1236+
1237+
// Test AddPartAddTotal strategy (default)
1238+
let ratio_metrics1 = RatioMetrics::new();
1239+
ratio_metrics1.set_part(20);
1240+
ratio_metrics1.set_total(50);
1241+
let ratio_metrics2 = RatioMetrics::new();
1242+
ratio_metrics2.set_part(20);
1243+
ratio_metrics2.set_total(50);
1244+
ratio_metrics1.merge(&ratio_metrics2);
1245+
assert_eq!("40% (40/100)", ratio_metrics1.to_string());
1246+
}
1247+
11431248
#[test]
11441249
fn test_display_timestamp() {
11451250
let timestamp = Timestamp::new();

0 commit comments

Comments
 (0)