Skip to content

Commit eedc787

Browse files
authored
fix issue with now() returning same value across statements (#3210)
1 parent c72f547 commit eedc787

10 files changed

Lines changed: 133 additions & 51 deletions

File tree

‎datafusion/core/src/dataframe.rs‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,25 @@ impl DataFrame {
8484

8585
/// Create a physical plan
8686
pub async fn create_physical_plan(&self) -> Result<Arc<dyn ExecutionPlan>> {
87-
let state = self.session_state.read().clone();
88-
state.create_physical_plan(&self.plan).await
87+
// this function is copied from SessionContext function of the
88+
// same name
89+
let state_cloned = {
90+
let mut state = self.session_state.write();
91+
state.execution_props.start_execution();
92+
93+
// We need to clone `state` to release the lock that is not `Send`. We could
94+
// make the lock `Send` by using `tokio::sync::Mutex`, but that would require to
95+
// propagate async even to the `LogicalPlan` building methods.
96+
// Cloning `state` here is fine as we then pass it as immutable `&state`, which
97+
// means that we avoid write consistency issues as the cloned version will not
98+
// be written to. As for eventual modifications that would be applied to the
99+
// original state after it has been cloned, they will not be picked up by the
100+
// clone but that is okay, as it is equivalent to postponing the state update
101+
// by keeping the lock until the end of the function scope.
102+
state.clone()
103+
};
104+
105+
state_cloned.create_physical_plan(&self.plan).await
89106
}
90107

91108
/// Filter the DataFrame by column. Returns a new DataFrame only containing the

‎datafusion/core/src/datasource/view.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,12 +80,15 @@ impl TableProvider for ViewTable {
8080

8181
async fn scan(
8282
&self,
83-
ctx: &SessionState,
83+
state: &SessionState,
8484
_projection: &Option<Vec<usize>>,
8585
_filters: &[Expr],
8686
_limit: Option<usize>,
8787
) -> Result<Arc<dyn ExecutionPlan>> {
88-
ctx.create_physical_plan(&self.logical_plan).await
88+
// clone state and start_execution so that now() works in views
89+
let mut state_cloned = state.clone();
90+
state_cloned.execution_props.start_execution();
91+
state_cloned.create_physical_plan(&self.logical_plan).await
8992
}
9093
}
9194

‎datafusion/core/src/execution/context.rs‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -365,20 +365,18 @@ impl SessionContext {
365365
match (or_replace, view) {
366366
(true, Ok(_)) => {
367367
self.deregister_table(name.as_str())?;
368-
let plan = self.optimize(&input)?;
369368
let table =
370-
Arc::new(ViewTable::try_new(plan.clone(), definition)?);
369+
Arc::new(ViewTable::try_new((*input).clone(), definition)?);
371370

372371
self.register_table(name.as_str(), table)?;
373-
Ok(Arc::new(DataFrame::new(self.state.clone(), &plan)))
372+
Ok(Arc::new(DataFrame::new(self.state.clone(), &input)))
374373
}
375374
(_, Err(_)) => {
376-
let plan = self.optimize(&input)?;
377375
let table =
378-
Arc::new(ViewTable::try_new(plan.clone(), definition)?);
376+
Arc::new(ViewTable::try_new((*input).clone(), definition)?);
379377

380378
self.register_table(name.as_str(), table)?;
381-
Ok(Arc::new(DataFrame::new(self.state.clone(), &plan)))
379+
Ok(Arc::new(DataFrame::new(self.state.clone(), &input)))
382380
}
383381
(false, Ok(_)) => Err(DataFusionError::Execution(format!(
384382
"Table '{:?}' already exists",

‎datafusion/core/src/logical_plan/mod.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,10 @@ pub use datafusion_expr::{
5050
StringifiedPlan, Subquery, TableScan, ToStringifiedPlan, Union,
5151
UserDefinedLogicalNode, Values,
5252
},
53-
lower, lpad, ltrim, max, md5, min, not_exists, not_in_subquery, now, now_expr,
54-
nullif, octet_length, or, power, random, regexp_match, regexp_replace, repeat,
55-
replace, reverse, right, round, rpad, rtrim, scalar_subquery, sha224, sha256, sha384,
56-
sha512, signum, sin, split_part, sqrt, starts_with, strpos, substr, sum, tan, to_hex,
53+
lower, lpad, ltrim, max, md5, min, not_exists, not_in_subquery, now, nullif,
54+
octet_length, or, power, random, regexp_match, regexp_replace, repeat, replace,
55+
reverse, right, round, rpad, rtrim, scalar_subquery, sha224, sha256, sha384, sha512,
56+
signum, sin, split_part, sqrt, starts_with, strpos, substr, sum, tan, to_hex,
5757
to_timestamp_micros, to_timestamp_millis, to_timestamp_seconds, translate, trim,
5858
trunc, unalias, upper, when, Expr, ExprSchemable, Literal, Operator,
5959
};

‎datafusion/core/src/physical_plan/mod.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -120,12 +120,12 @@ pub struct ColumnStatistics {
120120

121121
/// `ExecutionPlan` represent nodes in the DataFusion Physical Plan.
122122
///
123-
/// Each `ExecutionPlan` is Partition-aware and is responsible for
123+
/// Each `ExecutionPlan` is partition-aware and is responsible for
124124
/// creating the actual `async` [`SendableRecordBatchStream`]s
125125
/// of [`RecordBatch`] that incrementally compute the operator's
126126
/// output from its input partition.
127127
///
128-
/// [`ExecutionPlan`] can be displayed in an simplified form using the
128+
/// [`ExecutionPlan`] can be displayed in a simplified form using the
129129
/// return value from [`displayable`] in addition to the (normally
130130
/// quite verbose) `Debug` output.
131131
pub trait ExecutionPlan: Debug + Send + Sync {
@@ -168,7 +168,7 @@ pub trait ExecutionPlan: Debug + Send + Sync {
168168
/// The default implementation returns `true`
169169
///
170170
/// WARNING: if you override this default and return `false`, your
171-
/// operator can not rely on datafusion preserving the input order
171+
/// operator can not rely on DataFusion preserving the input order
172172
/// as it will likely not.
173173
fn relies_on_input_order(&self) -> bool {
174174
true
@@ -200,7 +200,7 @@ pub trait ExecutionPlan: Debug + Send + Sync {
200200
/// parallelism may outweigh any benefits
201201
///
202202
/// The default implementation returns `true` unless this operator
203-
/// has signalled it requiers a single child input partition.
203+
/// has signalled it requires a single child input partition.
204204
fn benefits_from_input_partitioning(&self) -> bool {
205205
// By default try to maximize parallelism with more CPUs if
206206
// possible

‎datafusion/core/tests/sql/timestamp.rs‎

Lines changed: 81 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -436,31 +436,93 @@ async fn test_current_timestamp_expressions() -> Result<()> {
436436
}
437437

438438
#[tokio::test]
439-
async fn test_current_timestamp_expressions_non_optimized() -> Result<()> {
440-
let t1 = chrono::Utc::now().timestamp();
439+
async fn test_now_in_same_stmt_using_sql_function() -> Result<()> {
441440
let ctx = SessionContext::new();
442-
let sql = "SELECT NOW(), NOW() as t2";
443441

444-
let msg = format!("Creating logical plan for '{}'", sql);
445-
let plan = ctx.create_logical_plan(sql).expect(&msg);
442+
let df1 = ctx.sql("select now(), now() as now2").await?;
443+
let result = result_vec(&df1.collect().await?);
444+
assert_eq!(result[0][0], result[0][1]);
446445

447-
let msg = format!("Creating physical plan for '{}': {:?}", sql, plan);
448-
let plan = ctx.create_physical_plan(&plan).await.expect(&msg);
446+
Ok(())
447+
}
449448

450-
let msg = format!("Executing physical plan for '{}': {:?}", sql, plan);
451-
let task_ctx = ctx.task_ctx();
452-
let res = collect(plan, task_ctx).await.expect(&msg);
453-
let actual = result_vec(&res);
449+
#[tokio::test]
450+
async fn test_now_across_statements() -> Result<()> {
451+
let ctx = SessionContext::new();
454452

455-
let res1 = actual[0][0].as_str();
456-
let res2 = actual[0][1].as_str();
457-
let t3 = chrono::Utc::now().timestamp();
458-
let t2_naive =
459-
chrono::NaiveDateTime::parse_from_str(res1, "%Y-%m-%d %H:%M:%S%.6f").unwrap();
453+
let actual1 = execute(&ctx, "SELECT NOW()").await;
454+
let res1 = actual1[0][0].as_str();
460455

461-
let t2 = t2_naive.timestamp();
462-
assert!(t1 <= t2 && t2 <= t3);
463-
assert_eq!(res2, res1);
456+
let actual2 = execute(&ctx, "SELECT NOW()").await;
457+
let res2 = actual2[0][0].as_str();
458+
459+
assert!(res1 < res2);
460+
461+
Ok(())
462+
}
463+
464+
#[tokio::test]
465+
async fn test_now_across_statements_using_sql_function() -> Result<()> {
466+
let ctx = SessionContext::new();
467+
468+
let df1 = ctx.sql("select now()").await?;
469+
let rb1 = df1.collect().await?;
470+
let result1 = result_vec(&rb1);
471+
let res1 = result1[0][0].as_str();
472+
473+
let df2 = ctx.sql("select now()").await?;
474+
let rb2 = df2.collect().await?;
475+
let result2 = result_vec(&rb2);
476+
let res2 = result2[0][0].as_str();
477+
478+
assert!(res1 < res2);
479+
480+
Ok(())
481+
}
482+
483+
#[tokio::test]
484+
async fn test_now_dataframe_api() -> Result<()> {
485+
let ctx = SessionContext::new();
486+
let df = ctx.sql("select 1").await?; // use this to get a DataFrame
487+
let df = df.select(vec![now(), now().alias("now2")])?;
488+
let result = result_vec(&df.collect().await?);
489+
assert_eq!(result[0][0], result[0][1]);
490+
491+
Ok(())
492+
}
493+
494+
#[tokio::test]
495+
async fn test_now_dataframe_api_across_statements() -> Result<()> {
496+
let ctx = SessionContext::new();
497+
let df = ctx.sql("select 1").await?; // use this to get a DataFrame
498+
let df = df.select(vec![now()])?;
499+
let result = result_vec(&df.collect().await?);
500+
501+
let df = ctx.sql("select 1").await?;
502+
let df = df.select(vec![now()])?;
503+
let result2 = result_vec(&df.collect().await?);
504+
505+
assert_ne!(result[0][0], result2[0][0]);
506+
507+
Ok(())
508+
}
509+
510+
#[tokio::test]
511+
async fn test_now_in_view() -> Result<()> {
512+
let ctx = SessionContext::new();
513+
let _df = ctx
514+
.sql("create or replace view test_now as select now()")
515+
.await?
516+
.collect()
517+
.await?;
518+
519+
let df = ctx.sql("select * from test_now").await?;
520+
let result = result_vec(&df.collect().await?);
521+
522+
let df1 = ctx.sql("select * from test_now").await?;
523+
let result2 = result_vec(&df1.collect().await?);
524+
525+
assert_ne!(result[0][0], result2[0][0]);
464526

465527
Ok(())
466528
}

‎datafusion/expr/src/expr_fn.rs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -367,10 +367,8 @@ nary_scalar_expr!(Btrim, btrim);
367367
//there is a func concat_ws before, so use concat_ws_expr as name.c
368368
nary_scalar_expr!(ConcatWithSeparator, concat_ws_expr);
369369
nary_scalar_expr!(Concat, concat_expr);
370-
nary_scalar_expr!(Now, now_expr);
371370

372371
// date functions
373-
unary_scalar_expr!(Now, now, "current time"); //TODO this is not a unary expression https://github.com/apache/arrow-datafusion/issues/3069
374372
scalar_expr!(DatePart, date_part, part, date);
375373
scalar_expr!(DateTrunc, date_trunc, part, date);
376374
scalar_expr!(DateBin, date_bin, stride, source, origin);
@@ -398,6 +396,15 @@ pub fn coalesce(args: Vec<Expr>) -> Expr {
398396
}
399397
}
400398

399+
/// Returns current timestamp in nanoseconds, using the same value for all instances of now() in
400+
/// same statement.
401+
pub fn now() -> Expr {
402+
Expr::ScalarFunction {
403+
fun: BuiltinScalarFunction::Now,
404+
args: vec![],
405+
}
406+
}
407+
401408
/// Create a CASE WHEN statement with literal WHEN expressions for comparison to the base expression.
402409
pub fn case(expr: Expr) -> CaseBuilder {
403410
CaseBuilder::new(Some(Box::new(expr)), vec![], vec![], None)
@@ -564,7 +571,6 @@ mod test {
564571
test_unary_scalar_expr!(Atan, atan);
565572
test_unary_scalar_expr!(Floor, floor);
566573
test_unary_scalar_expr!(Ceil, ceil);
567-
test_unary_scalar_expr!(Now, now);
568574
test_unary_scalar_expr!(Round, round);
569575
test_unary_scalar_expr!(Trunc, trunc);
570576
test_unary_scalar_expr!(Abs, abs);

‎datafusion/physical-expr/src/datetime_expressions.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,8 +170,8 @@ pub fn to_timestamp_seconds(args: &[ColumnarValue]) -> Result<ColumnarValue> {
170170
/// specified timestamp.
171171
///
172172
/// The semantics of `now()` require it to return the same value
173-
/// whenever it is called in a query. This this value is chosen during
174-
/// planning time and bound into a closure that
173+
/// wherever it appears within a single statement. This value is
174+
/// chosen during planning time.
175175
pub fn make_now(
176176
now_ts: DateTime<Utc>,
177177
) -> impl Fn(&[ColumnarValue]) -> Result<ColumnarValue> {

‎datafusion/proto/src/from_proto.rs‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ use datafusion_expr::{
3636
character_length, chr, coalesce, concat_expr, concat_ws_expr, cos, date_bin,
3737
date_part, date_trunc, digest, exp, floor, from_unixtime, left, ln, log10, log2,
3838
logical_plan::{PlanType, StringifiedPlan},
39-
lower, lpad, ltrim, md5, now_expr, nullif, octet_length, power, random, regexp_match,
39+
lower, lpad, ltrim, md5, now, nullif, octet_length, power, random, regexp_match,
4040
regexp_replace, repeat, replace, reverse, right, round, rpad, rtrim, sha224, sha256,
4141
sha384, sha512, signum, sin, split_part, sqrt, starts_with, strpos, substr, tan,
4242
to_hex, to_timestamp_micros, to_timestamp_millis, to_timestamp_seconds, translate,
@@ -1117,12 +1117,7 @@ pub fn parse_expr(
11171117
ScalarFunction::ToTimestampSeconds => {
11181118
Ok(to_timestamp_seconds(parse_expr(&args[0], registry)?))
11191119
}
1120-
ScalarFunction::Now => Ok(now_expr(
1121-
args.to_owned()
1122-
.iter()
1123-
.map(|expr| parse_expr(expr, registry))
1124-
.collect::<Result<Vec<_>, _>>()?,
1125-
)),
1120+
ScalarFunction::Now => Ok(now()),
11261121
ScalarFunction::Translate => Ok(translate(
11271122
parse_expr(&args[0], registry)?,
11281123
parse_expr(&args[1], registry)?,

‎docs/source/user-guide/sql/scalar_functions.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,7 @@ Note that `CAST(.. AS Timestamp)` converts to Timestamps with Nanosecond resolut
246246
`extract(field FROM source)`
247247

248248
- The `extract` function retrieves subfields such as year or hour from date/time values.
249-
`source` must be a value expression of type timestamp, Data32, or Data64. `field` is an identifier that selects what field to extract from the source value.
249+
`source` must be a value expression of type timestamp, Date32, or Date64. `field` is an identifier that selects what field to extract from the source value.
250250
The `extract` function returns values of type u32.
251251
- `year` :`extract(year FROM to_timestamp('2020-09-08T12:00:00+00:00')) -> 2020`
252252
- `month`:`extract(month FROM to_timestamp('2020-09-08T12:00:00+00:00')) -> 9`
@@ -273,7 +273,8 @@ Note that `CAST(.. AS Timestamp)` converts to Timestamps with Nanosecond resolut
273273

274274
### `now`
275275

276-
current time
276+
Returns current time as `Timestamp(Nanoseconds, UTC)`. Returns same value for the function
277+
wherever it appears in the statement, using a value chosen at planning time.
277278

278279
## Other Functions
279280

0 commit comments

Comments
 (0)