Skip to content

Commit 981a9bb

Browse files
authored
Set query_execution_start_time on snapshot from SessionContext (#4747) (#4750)
* Set query_execution_start_time on snapshot from SessionContext (#4747) * Remove TODO * Clippy
1 parent 5630339 commit 981a9bb

4 files changed

Lines changed: 19 additions & 39 deletions

File tree

‎datafusion/core/src/dataframe.rs‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -87,13 +87,7 @@ impl DataFrame {
8787
}
8888

8989
/// Create a physical plan
90-
pub async fn create_physical_plan(mut self) -> Result<Arc<dyn ExecutionPlan>> {
91-
self.create_physical_plan_impl().await
92-
}
93-
94-
/// Temporary pending #4626
95-
async fn create_physical_plan_impl(&mut self) -> Result<Arc<dyn ExecutionPlan>> {
96-
self.session_state.execution_props.start_execution();
90+
pub async fn create_physical_plan(self) -> Result<Arc<dyn ExecutionPlan>> {
9791
self.session_state.create_physical_plan(&self.plan).await
9892
}
9993

@@ -627,24 +621,24 @@ impl DataFrame {
627621
}
628622

629623
/// Write a `DataFrame` to a CSV file.
630-
pub async fn write_csv(mut self, path: &str) -> Result<()> {
631-
let plan = self.create_physical_plan_impl().await?;
624+
pub async fn write_csv(self, path: &str) -> Result<()> {
625+
let plan = self.session_state.create_physical_plan(&self.plan).await?;
632626
plan_to_csv(&self.session_state, plan, path).await
633627
}
634628

635629
/// Write a `DataFrame` to a Parquet file.
636630
pub async fn write_parquet(
637-
mut self,
631+
self,
638632
path: &str,
639633
writer_properties: Option<WriterProperties>,
640634
) -> Result<()> {
641-
let plan = self.create_physical_plan_impl().await?;
635+
let plan = self.session_state.create_physical_plan(&self.plan).await?;
642636
plan_to_parquet(&self.session_state, plan, path, writer_properties).await
643637
}
644638

645639
/// Executes a query and writes the results to a partitioned JSON file.
646-
pub async fn write_json(mut self, path: impl AsRef<str>) -> Result<()> {
647-
let plan = self.create_physical_plan_impl().await?;
640+
pub async fn write_json(self, path: impl AsRef<str>) -> Result<()> {
641+
let plan = self.session_state.create_physical_plan(&self.plan).await?;
648642
plan_to_json(&self.session_state, plan, path).await
649643
}
650644

‎datafusion/core/src/datasource/view.rs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,6 @@ impl TableProvider for ViewTable {
108108
filters: &[Expr],
109109
limit: Option<usize>,
110110
) -> Result<Arc<dyn ExecutionPlan>> {
111-
// clone state and start_execution so that now() works in views
112-
let mut state_cloned = state.clone();
113-
state_cloned.execution_props.start_execution();
114111
let plan = if let Some(projection) = projection {
115112
// avoiding adding a redundant projection (e.g. SELECT * FROM view)
116113
let current_projection =
@@ -144,7 +141,7 @@ impl TableProvider for ViewTable {
144141
plan = plan.limit(0, Some(limit))?;
145142
}
146143

147-
state_cloned.create_physical_plan(&plan.build()?).await
144+
state.create_physical_plan(&plan.build()?).await
148145
}
149146
}
150147

‎datafusion/core/src/execution/context.rs‎

Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1000,23 +1000,7 @@ impl SessionContext {
10001000
&self,
10011001
logical_plan: &LogicalPlan,
10021002
) -> Result<Arc<dyn ExecutionPlan>> {
1003-
let state_cloned = {
1004-
let mut state = self.state.write();
1005-
state.execution_props.start_execution();
1006-
1007-
// We need to clone `state` to release the lock that is not `Send`. We could
1008-
// make the lock `Send` by using `tokio::sync::Mutex`, but that would require to
1009-
// propagate async even to the `LogicalPlan` building methods.
1010-
// Cloning `state` here is fine as we then pass it as immutable `&state`, which
1011-
// means that we avoid write consistency issues as the cloned version will not
1012-
// be written to. As for eventual modifications that would be applied to the
1013-
// original state after it has been cloned, they will not be picked up by the
1014-
// clone but that is okay, as it is equivalent to postponing the state update
1015-
// by keeping the lock until the end of the function scope.
1016-
state.clone()
1017-
};
1018-
1019-
state_cloned.create_physical_plan(logical_plan).await
1003+
self.state().create_physical_plan(logical_plan).await
10201004
}
10211005

10221006
/// Executes a query and writes the results to a partitioned CSV file.
@@ -1055,9 +1039,12 @@ impl SessionContext {
10551039
Arc::new(TaskContext::from(self))
10561040
}
10571041

1058-
/// Get a copy of the [`SessionState`] of this [`SessionContext`]
1042+
/// Snapshots the [`SessionState`] of this [`SessionContext`] setting the
1043+
/// `query_execution_start_time` to the current time
10591044
pub fn state(&self) -> SessionState {
1060-
self.state.read().clone()
1045+
let mut state = self.state.read().clone();
1046+
state.execution_props.start_execution();
1047+
state
10611048
}
10621049
}
10631050

‎datafusion/physical-expr/src/execution_props.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
// under the License.
1717

1818
use crate::var_provider::{VarProvider, VarType};
19-
use chrono::{DateTime, Utc};
19+
use chrono::{DateTime, TimeZone, Utc};
2020
use std::collections::HashMap;
2121
use std::sync::Arc;
2222

@@ -44,14 +44,16 @@ impl ExecutionProps {
4444
/// Creates a new execution props
4545
pub fn new() -> Self {
4646
ExecutionProps {
47-
query_execution_start_time: chrono::Utc::now(),
47+
// Set this to a fixed sentinel to make it obvious if this is
48+
// not being updated / propagated correctly
49+
query_execution_start_time: Utc.timestamp_nanos(0),
4850
var_providers: None,
4951
}
5052
}
5153

5254
/// Marks the execution of query started timestamp
5355
pub fn start_execution(&mut self) -> &Self {
54-
self.query_execution_start_time = chrono::Utc::now();
56+
self.query_execution_start_time = Utc::now();
5557
&*self
5658
}
5759

0 commit comments

Comments
 (0)