Skip to content

Commit ccafa6b

Browse files
committed
Fix elapsed_compute metric for Parquet DataSourceExec
The `elapsed_compute` baseline metric for Parquet scans previously reported unrealistically low values (e.g. 14ns for a full table scan) because no timer was wrapping the per-batch compute work. This follows the same pattern used in PR apache#18901 for CSV: instantiate `BaselineMetrics` in `ParquetOpener::open()` and wrap the stream's per-batch processing (projection, schema replacement, metrics copy) with an `elapsed_compute` timer. Closes part of apache#18195.
1 parent 02ce571 commit ccafa6b

1 file changed

Lines changed: 7 additions & 3 deletions

File tree

‎datafusion/datasource-parquet/src/opener.rs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ use datafusion_physical_expr_common::physical_expr::{
4747
PhysicalExpr, is_dynamic_physical_expr,
4848
};
4949
use datafusion_physical_plan::metrics::{
50-
Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, PruningMetrics,
50+
BaselineMetrics, Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, PruningMetrics,
5151
};
5252
use datafusion_pruning::{FilePruner, PruningPredicate, build_pruning_predicate};
5353

@@ -191,6 +191,7 @@ impl FileOpener for ParquetOpener {
191191
let file_name = file_location.to_string();
192192
let file_metrics =
193193
ParquetFileMetrics::new(self.partition_index, &file_name, &self.metrics);
194+
let baseline_metrics = BaselineMetrics::new(&self.metrics, self.partition_index);
194195

195196
let metadata_size_hint = partitioned_file
196197
.metadata_size_hint
@@ -631,7 +632,8 @@ impl FileOpener for ParquetOpener {
631632
let projector = projection.make_projector(&stream_schema)?;
632633

633634
let stream = stream.map_err(DataFusionError::from).map(move |b| {
634-
b.and_then(|mut b| {
635+
let mut timer = baseline_metrics.elapsed_compute().timer();
636+
let result = b.and_then(|mut b| {
635637
copy_arrow_reader_metrics(
636638
&arrow_reader_metrics,
637639
&predicate_cache_inner_records,
@@ -659,7 +661,9 @@ impl FileOpener for ParquetOpener {
659661
} else {
660662
Ok(b)
661663
}
662-
})
664+
});
665+
timer.stop();
666+
result
663667
});
664668

665669
// ----------------------------------------------------------------------

0 commit comments

Comments
 (0)