Skip to content

Commit 13974c2

Browse files
askalt0x501D
authored andcommitted
physical_plan: move metrics to task context
To share physical plans across executions we need to place metrics in some other place. This patch moves them to the task context. When plan is scanned metrics set is registered in the context by the plan address. To display a plan with metrics one should provide the task context, where metrics associated with the plan are stored. Also, fmt errors are fixed and applied several clippy suggestions.
1 parent 8c80357 commit 13974c2

94 files changed

Lines changed: 716 additions & 742 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎benchmarks/src/tpch/run.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,11 +219,12 @@ impl RunOpt {
219219
displayable(physical_plan.as_ref()).indent(true)
220220
);
221221
}
222-
let result = collect(physical_plan.clone(), state.task_ctx()).await?;
222+
let task_ctx = state.task_ctx();
223+
let result = collect(physical_plan.clone(), Arc::clone(&task_ctx)).await?;
223224
if debug {
224225
println!(
225226
"=== Physical plan with metrics ===\n{}\n",
226-
DisplayableExecutionPlan::with_metrics(physical_plan.as_ref())
227+
DisplayableExecutionPlan::with_metrics(physical_plan.as_ref(), task_ctx)
227228
.indent(true)
228229
);
229230
if !result.is_empty() {

‎datafusion-examples/examples/advanced_parquet_index.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ use datafusion::datasource::physical_plan::{
2828
parquet::ParquetFileReaderFactory, FileMeta, FileScanConfig,
2929
};
3030
use datafusion::datasource::TableProvider;
31+
use datafusion::execution::metrics::ExecutionPlanMetricsSet;
3132
use datafusion::execution::object_store::ObjectStoreUrl;
3233
use datafusion::parquet::arrow::arrow_reader::{
3334
ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelection, RowSelector,
@@ -39,7 +40,6 @@ use datafusion::parquet::file::properties::{EnabledStatistics, WriterProperties}
3940
use datafusion::parquet::schema::types::ColumnPath;
4041
use datafusion::physical_expr::PhysicalExpr;
4142
use datafusion::physical_optimizer::pruning::PruningPredicate;
42-
use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
4343
use datafusion::physical_plan::ExecutionPlan;
4444
use datafusion::prelude::*;
4545
use datafusion_common::{

‎datafusion-examples/examples/csv_opener.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ use datafusion::{
2626
physical_plan::{CsvConfig, CsvOpener, FileScanConfig, FileStream},
2727
},
2828
error::Result,
29-
physical_plan::metrics::ExecutionPlanMetricsSet,
29+
execution::metrics::ExecutionPlanMetricsSet,
3030
test_util::aggr_test_schema,
3131
};
3232

‎datafusion-examples/examples/json_opener.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ use datafusion::{
2727
physical_plan::{FileScanConfig, FileStream, JsonOpener},
2828
},
2929
error::Result,
30-
physical_plan::metrics::ExecutionPlanMetricsSet,
30+
execution::metrics::ExecutionPlanMetricsSet,
3131
};
3232

3333
use futures::StreamExt;

‎datafusion-examples/examples/parquet_exec_visitor.rs‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ use datafusion::datasource::file_format::parquet::ParquetFormat;
2121
use datafusion::datasource::listing::{ListingOptions, PartitionedFile};
2222
use datafusion::datasource::physical_plan::ParquetExec;
2323
use datafusion::execution::context::SessionContext;
24-
use datafusion::physical_plan::metrics::MetricValue;
24+
use datafusion::execution::metrics::MetricValue;
25+
use datafusion::execution::TaskContext;
2526
use datafusion::physical_plan::{
2627
execute_stream, visit_execution_plan, ExecutionPlan, ExecutionPlanVisitor,
2728
};
@@ -52,20 +53,22 @@ async fn main() {
5253
let df = ctx.sql("SELECT * FROM my_table").await.unwrap();
5354
let plan = df.create_physical_plan().await.unwrap();
5455

55-
// Create empty visitor
56-
let mut visitor = ParquetExecVisitor {
57-
file_groups: None,
58-
bytes_scanned: None,
59-
};
60-
6156
// Make sure you execute the plan to collect actual execution statistics.
6257
// For example, in this example the `file_scan_config` is known without executing
6358
// but the `bytes_scanned` would be None if we did not execute.
64-
let mut batch_stream = execute_stream(plan.clone(), ctx.task_ctx()).unwrap();
59+
let task_ctx = ctx.task_ctx();
60+
let mut batch_stream = execute_stream(plan.clone(), Arc::clone(&task_ctx)).unwrap();
6561
while let Some(batch) = batch_stream.next().await {
6662
println!("Batch rows: {}", batch.unwrap().num_rows());
6763
}
6864

65+
// Create empty visitor
66+
let mut visitor = ParquetExecVisitor {
67+
file_groups: None,
68+
bytes_scanned: None,
69+
ctx: task_ctx,
70+
};
71+
6972
visit_execution_plan(plan.as_ref(), &mut visitor).unwrap();
7073

7174
println!(
@@ -85,6 +88,7 @@ async fn main() {
8588
struct ParquetExecVisitor {
8689
file_groups: Option<Vec<Vec<PartitionedFile>>>,
8790
bytes_scanned: Option<MetricValue>,
91+
ctx: Arc<TaskContext>,
8892
}
8993

9094
impl ExecutionPlanVisitor for ParquetExecVisitor {
@@ -99,7 +103,7 @@ impl ExecutionPlanVisitor for ParquetExecVisitor {
99103
if let Some(parquet_exec) = maybe_parquet_exec {
100104
self.file_groups = Some(parquet_exec.base_config().file_groups.clone());
101105

102-
let metrics = match parquet_exec.metrics() {
106+
let metrics = match self.ctx.plan_metrics(plan.as_any()) {
103107
None => return Ok(true),
104108
Some(metrics) => metrics,
105109
};

‎datafusion/common/src/column.rs‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -109,21 +109,23 @@ impl Column {
109109
/// where `"foo.BAR"` would be parsed to a reference to column named `foo.BAR`
110110
pub fn from_qualified_name(flat_name: impl Into<String>) -> Self {
111111
let flat_name = flat_name.into();
112-
Self::from_idents(&mut parse_identifiers_normalized(&flat_name, false))
113-
.unwrap_or_else(|| Self {
112+
Self::from_idents(&mut parse_identifiers_normalized(&flat_name, false)).unwrap_or(
113+
Self {
114114
relation: None,
115115
name: flat_name,
116-
})
116+
},
117+
)
117118
}
118119

119120
/// Deserialize a fully qualified name string into a column preserving column text case
120121
pub fn from_qualified_name_ignore_case(flat_name: impl Into<String>) -> Self {
121122
let flat_name = flat_name.into();
122-
Self::from_idents(&mut parse_identifiers_normalized(&flat_name, true))
123-
.unwrap_or_else(|| Self {
123+
Self::from_idents(&mut parse_identifiers_normalized(&flat_name, true)).unwrap_or(
124+
Self {
124125
relation: None,
125126
name: flat_name,
126-
})
127+
},
128+
)
127129
}
128130

129131
/// return the column's name.

‎datafusion/common/src/hash_utils.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ pub trait HashValue {
6363
fn hash_one(&self, state: &RandomState) -> u64;
6464
}
6565

66-
impl<'a, T: HashValue + ?Sized> HashValue for &'a T {
66+
impl<T: HashValue + ?Sized> HashValue for &'_ T {
6767
fn hash_one(&self, state: &RandomState) -> u64 {
6868
T::hash_one(self, state)
6969
}

‎datafusion/common/src/utils/mod.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,7 @@ pub fn longest_consecutive_prefix<T: Borrow<usize>>(
354354
count
355355
}
356356

357-
/// Array Utils
357+
// Array Utils
358358

359359
/// Wrap an array into a single element `ListArray`.
360360
/// For example `[1, 2, 3]` would be converted into `[[1, 2, 3]]`
@@ -363,7 +363,7 @@ pub fn array_into_list_array_nullable(arr: ArrayRef) -> ListArray {
363363
array_into_list_array(arr, true)
364364
}
365365

366-
/// Array Utils
366+
// Array Utils
367367

368368
/// Wrap an array into a single element `ListArray`.
369369
/// For example `[1, 2, 3]` would be converted into `[[1, 2, 3]]`
@@ -563,7 +563,7 @@ pub mod datafusion_strsim {
563563

564564
struct StringWrapper<'a>(&'a str);
565565

566-
impl<'a, 'b> IntoIterator for &'a StringWrapper<'b> {
566+
impl<'b> IntoIterator for &StringWrapper<'b> {
567567
type Item = char;
568568
type IntoIter = Chars<'b>;
569569

‎datafusion/core/benches/sql_planner.rs‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,6 @@ use criterion::Bencher;
2727
use datafusion::datasource::MemTable;
2828
use datafusion::execution::context::SessionContext;
2929
use datafusion_common::ScalarValue;
30-
use itertools::Itertools;
31-
use std::fs::File;
32-
use std::io::{BufRead, BufReader};
33-
use std::path::PathBuf;
3430
use std::sync::Arc;
3531
use test_utils::tpcds::tpcds_schemas;
3632
use test_utils::tpch::tpch_schemas;
@@ -95,28 +91,6 @@ fn register_defs(ctx: SessionContext, defs: Vec<TableDef>) -> SessionContext {
9591
ctx
9692
}
9793

98-
fn register_clickbench_hits_table() -> SessionContext {
99-
let ctx = SessionContext::new();
100-
let rt = Runtime::new().unwrap();
101-
102-
// use an external table for clickbench benchmarks
103-
let path =
104-
if PathBuf::from(format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}")).exists() {
105-
format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}")
106-
} else {
107-
format!("{BENCHMARKS_PATH_2}{CLICKBENCH_DATA_PATH}")
108-
};
109-
110-
let sql = format!("CREATE EXTERNAL TABLE hits STORED AS PARQUET LOCATION '{path}'");
111-
112-
rt.block_on(ctx.sql(&sql)).unwrap();
113-
114-
let count =
115-
rt.block_on(async { ctx.table("hits").await.unwrap().count().await.unwrap() });
116-
assert!(count > 0);
117-
ctx
118-
}
119-
12094
/// Target of this benchmark: control that placeholders replacing does not get slower,
12195
/// if the query does not contain placeholders at all.
12296
fn benchmark_with_param_values_many_columns(ctx: &SessionContext, b: &mut Bencher) {

‎datafusion/core/src/datasource/file_format/arrow.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,10 @@ use datafusion_common::parsers::CompressionTypeVariant;
4646
use datafusion_common::{
4747
not_impl_err, DataFusionError, GetExt, Statistics, DEFAULT_ARROW_EXTENSION,
4848
};
49+
use datafusion_execution::metrics::MetricsSet;
4950
use datafusion_execution::{SendableRecordBatchStream, TaskContext};
5051
use datafusion_physical_expr::PhysicalExpr;
5152
use datafusion_physical_plan::insert::{DataSink, DataSinkExec};
52-
use datafusion_physical_plan::metrics::MetricsSet;
5353

5454
use async_trait::async_trait;
5555
use bytes::Bytes;

0 commit comments

Comments
 (0)