Skip to content

Commit f5bdc2d

Browse files
authored
Add is_volatile(PhysicalExpr) as a generic public API for volatility (#17351)
1 parent 3f422a1 commit f5bdc2d

4 files changed

Lines changed: 254 additions & 18 deletions

File tree

‎datafusion/physical-expr-common/src/physical_expr.rs‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,22 @@ pub trait PhysicalExpr: Any + Send + Sync + Display + Debug + DynEq + DynHash {
377377
// static expressions will always return 0.
378378
0
379379
}
380+
381+
/// Returns true if the expression node is volatile, i.e. whether it can return
382+
/// different results when evaluated multiple times with the same input.
383+
///
384+
/// Note: unlike [`is_volatile`], this function does not consider inputs:
385+
/// - `random()` returns `true`,
386+
/// - `a + random()` returns `false` (because the operation `+` itself is not volatile.)
387+
///
388+
/// The default to this function was set to `false` when it was created
389+
/// to avoid imposing API churn on implementers, but this is not a safe default in general.
390+
/// It is highly recommended that volatile expressions implement this method and return `true`.
391+
/// This default may be removed in the future if it causes problems or we decide to
392+
/// eat the cost of the breaking change and require all implementers to make a choice.
393+
fn is_volatile_node(&self) -> bool {
394+
false
395+
}
380396
}
381397

382398
#[deprecated(
@@ -560,3 +576,28 @@ pub fn is_dynamic_physical_expr(expr: &Arc<dyn PhysicalExpr>) -> bool {
560576
// If the generation is non-zero, then this `PhysicalExpr` is dynamic.
561577
snapshot_generation(expr) != 0
562578
}
579+
580+
/// Returns true if the expression is volatile, i.e. whether it can return different
581+
/// results when evaluated multiple times with the same input.
582+
///
583+
/// For example the function call `RANDOM()` is volatile as each call will
584+
/// return a different value.
585+
///
586+
/// This method recursively checks if any sub-expression is volatile, for example
587+
/// `1 + RANDOM()` will return `true`.
588+
pub fn is_volatile(expr: &Arc<dyn PhysicalExpr>) -> bool {
589+
if expr.is_volatile_node() {
590+
return true;
591+
}
592+
let mut is_volatile = false;
593+
expr.apply(|e| {
594+
if e.is_volatile_node() {
595+
is_volatile = true;
596+
Ok(TreeNodeRecursion::Stop)
597+
} else {
598+
Ok(TreeNodeRecursion::Continue)
599+
}
600+
})
601+
.expect("infallible closure should not fail");
602+
is_volatile
603+
}

‎datafusion/physical-expr/src/physical_expr.rs‎

Lines changed: 125 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,12 +204,19 @@ pub fn add_offset_to_physical_sort_exprs(
204204
mod tests {
205205
use super::*;
206206

207-
use crate::expressions::{Column, Literal};
207+
use crate::expressions::{BinaryExpr, Column, Literal};
208208
use crate::physical_expr::{
209209
physical_exprs_bag_equal, physical_exprs_contains, physical_exprs_equal,
210210
};
211+
use datafusion_physical_expr_common::physical_expr::is_volatile;
211212

212-
use datafusion_common::ScalarValue;
213+
use arrow::datatypes::{DataType, Schema};
214+
use arrow::record_batch::RecordBatch;
215+
use datafusion_common::{Result, ScalarValue};
216+
use datafusion_expr::ColumnarValue;
217+
use datafusion_expr::Operator;
218+
use std::any::Any;
219+
use std::fmt;
213220

214221
#[test]
215222
fn test_physical_exprs_contains() {
@@ -322,4 +329,120 @@ mod tests {
322329
assert!(physical_exprs_bag_equal(list3.as_slice(), list3.as_slice()));
323330
assert!(physical_exprs_bag_equal(list4.as_slice(), list4.as_slice()));
324331
}
332+
333+
#[test]
334+
fn test_is_volatile_default_behavior() {
335+
// Test that default PhysicalExpr implementations are not volatile
336+
let literal =
337+
Arc::new(Literal::new(ScalarValue::Int32(Some(42)))) as Arc<dyn PhysicalExpr>;
338+
let column = Arc::new(Column::new("test", 0)) as Arc<dyn PhysicalExpr>;
339+
340+
// Test is_volatile_node() - should return false by default
341+
assert!(!literal.is_volatile_node());
342+
assert!(!column.is_volatile_node());
343+
344+
// Test is_volatile() - should return false for non-volatile expressions
345+
assert!(!is_volatile(&literal));
346+
assert!(!is_volatile(&column));
347+
}
348+
349+
/// Mock volatile PhysicalExpr for testing purposes
350+
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
351+
struct MockVolatileExpr {
352+
volatile: bool,
353+
}
354+
355+
impl MockVolatileExpr {
356+
fn new(volatile: bool) -> Self {
357+
Self { volatile }
358+
}
359+
}
360+
361+
impl fmt::Display for MockVolatileExpr {
362+
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
363+
write!(f, "MockVolatile({})", self.volatile)
364+
}
365+
}
366+
367+
impl PhysicalExpr for MockVolatileExpr {
368+
fn as_any(&self) -> &dyn Any {
369+
self
370+
}
371+
372+
fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
373+
Ok(DataType::Boolean)
374+
}
375+
376+
fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
377+
Ok(false)
378+
}
379+
380+
fn evaluate(&self, _batch: &RecordBatch) -> Result<ColumnarValue> {
381+
Ok(ColumnarValue::Scalar(ScalarValue::Boolean(Some(
382+
self.volatile,
383+
))))
384+
}
385+
386+
fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
387+
vec![]
388+
}
389+
390+
fn with_new_children(
391+
self: Arc<Self>,
392+
_children: Vec<Arc<dyn PhysicalExpr>>,
393+
) -> Result<Arc<dyn PhysicalExpr>> {
394+
Ok(self)
395+
}
396+
397+
fn is_volatile_node(&self) -> bool {
398+
self.volatile
399+
}
400+
401+
fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
402+
write!(f, "mock_volatile({})", self.volatile)
403+
}
404+
}
405+
406+
#[test]
407+
fn test_nested_expression_volatility() {
408+
// Test that is_volatile() recursively detects volatility in expression trees
409+
410+
// Create a volatile mock expression
411+
let volatile_expr =
412+
Arc::new(MockVolatileExpr::new(true)) as Arc<dyn PhysicalExpr>;
413+
assert!(volatile_expr.is_volatile_node());
414+
assert!(is_volatile(&volatile_expr));
415+
416+
// Create a non-volatile mock expression
417+
let stable_expr = Arc::new(MockVolatileExpr::new(false)) as Arc<dyn PhysicalExpr>;
418+
assert!(!stable_expr.is_volatile_node());
419+
assert!(!is_volatile(&stable_expr));
420+
421+
// Create a literal (non-volatile)
422+
let literal =
423+
Arc::new(Literal::new(ScalarValue::Int32(Some(42)))) as Arc<dyn PhysicalExpr>;
424+
assert!(!literal.is_volatile_node());
425+
assert!(!is_volatile(&literal));
426+
427+
// Test composite expression: volatile_expr AND literal
428+
// The BinaryExpr itself is not volatile, but contains a volatile child
429+
let composite_expr = Arc::new(BinaryExpr::new(
430+
Arc::clone(&volatile_expr),
431+
Operator::And,
432+
Arc::clone(&literal),
433+
)) as Arc<dyn PhysicalExpr>;
434+
435+
assert!(!composite_expr.is_volatile_node()); // BinaryExpr itself is not volatile
436+
assert!(is_volatile(&composite_expr)); // But it contains a volatile child
437+
438+
// Test composite expression with all non-volatile children
439+
let stable_composite = Arc::new(BinaryExpr::new(
440+
Arc::clone(&stable_expr),
441+
Operator::And,
442+
Arc::clone(&literal),
443+
)) as Arc<dyn PhysicalExpr>;
444+
445+
assert!(!stable_composite.is_volatile_node());
446+
assert!(!is_volatile(&stable_composite)); // No volatile children
447+
}
325448
}

‎datafusion/physical-expr/src/scalar_function.rs‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ use datafusion_expr::sort_properties::ExprProperties;
4646
use datafusion_expr::type_coercion::functions::data_types_with_scalar_udf;
4747
use datafusion_expr::{
4848
expr_vec_fmt, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF,
49+
Volatility,
4950
};
5051

5152
/// Physical expression of a scalar function
@@ -358,4 +359,89 @@ impl PhysicalExpr for ScalarFunctionExpr {
358359
}
359360
write!(f, ")")
360361
}
362+
363+
fn is_volatile_node(&self) -> bool {
364+
self.fun.signature().volatility == Volatility::Volatile
365+
}
366+
}
367+
368+
#[cfg(test)]
369+
mod tests {
370+
use super::*;
371+
use crate::expressions::Column;
372+
use arrow::datatypes::{DataType, Field, Schema};
373+
use datafusion_expr::{ScalarUDF, ScalarUDFImpl, Signature};
374+
use datafusion_physical_expr_common::physical_expr::is_volatile;
375+
use std::any::Any;
376+
377+
/// Test helper to create a mock UDF with a specific volatility
378+
#[derive(Debug, PartialEq, Eq, Hash)]
379+
struct MockScalarUDF {
380+
signature: Signature,
381+
}
382+
383+
impl ScalarUDFImpl for MockScalarUDF {
384+
fn as_any(&self) -> &dyn Any {
385+
self
386+
}
387+
388+
fn name(&self) -> &str {
389+
"mock_function"
390+
}
391+
392+
fn signature(&self) -> &Signature {
393+
&self.signature
394+
}
395+
396+
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
397+
Ok(DataType::Int32)
398+
}
399+
400+
fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
401+
Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some(42))))
402+
}
403+
}
404+
405+
#[test]
406+
fn test_scalar_function_volatile_node() {
407+
// Create a volatile UDF
408+
let volatile_udf = Arc::new(ScalarUDF::from(MockScalarUDF {
409+
signature: Signature::uniform(
410+
1,
411+
vec![DataType::Float32],
412+
Volatility::Volatile,
413+
),
414+
}));
415+
416+
// Create a non-volatile UDF
417+
let stable_udf = Arc::new(ScalarUDF::from(MockScalarUDF {
418+
signature: Signature::uniform(1, vec![DataType::Float32], Volatility::Stable),
419+
}));
420+
421+
let schema = Schema::new(vec![Field::new("a", DataType::Float32, false)]);
422+
let args = vec![Arc::new(Column::new("a", 0)) as Arc<dyn PhysicalExpr>];
423+
let config_options = Arc::new(ConfigOptions::new());
424+
425+
// Test volatile function
426+
let volatile_expr = ScalarFunctionExpr::try_new(
427+
volatile_udf,
428+
args.clone(),
429+
&schema,
430+
Arc::clone(&config_options),
431+
)
432+
.unwrap();
433+
434+
assert!(volatile_expr.is_volatile_node());
435+
let volatile_arc: Arc<dyn PhysicalExpr> = Arc::new(volatile_expr);
436+
assert!(is_volatile(&volatile_arc));
437+
438+
// Test non-volatile function
439+
let stable_expr =
440+
ScalarFunctionExpr::try_new(stable_udf, args, &schema, config_options)
441+
.unwrap();
442+
443+
assert!(!stable_expr.is_volatile_node());
444+
let stable_arc: Arc<dyn PhysicalExpr> = Arc::new(stable_expr);
445+
assert!(!is_volatile(&stable_arc));
446+
}
361447
}

‎datafusion/physical-optimizer/src/filter_pushdown.rs‎

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ use crate::PhysicalOptimizerRule;
3737

3838
use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
3939
use datafusion_common::{config::ConfigOptions, Result};
40-
use datafusion_expr_common::signature::Volatility;
4140
use datafusion_physical_expr::PhysicalExpr;
41+
use datafusion_physical_expr_common::physical_expr::is_volatile;
4242
use datafusion_physical_plan::filter_pushdown::{
4343
ChildFilterPushdownResult, ChildPushdownResult, FilterPushdownPhase,
4444
FilterPushdownPropagation, PushedDown,
@@ -711,7 +711,7 @@ impl<T: Clone> FilteredVec<T> {
711711
fn allow_pushdown_for_expr(expr: &Arc<dyn PhysicalExpr>) -> bool {
712712
let mut allow_pushdown = true;
713713
expr.apply(|e| {
714-
allow_pushdown = allow_pushdown && allow_pushdown_for_expr_inner(e);
714+
allow_pushdown = allow_pushdown && !is_volatile(e);
715715
if allow_pushdown {
716716
Ok(TreeNodeRecursion::Continue)
717717
} else {
@@ -722,20 +722,6 @@ fn allow_pushdown_for_expr(expr: &Arc<dyn PhysicalExpr>) -> bool {
722722
allow_pushdown
723723
}
724724

725-
fn allow_pushdown_for_expr_inner(expr: &Arc<dyn PhysicalExpr>) -> bool {
726-
if let Some(scalar_function) =
727-
expr.as_any()
728-
.downcast_ref::<datafusion_physical_expr::ScalarFunctionExpr>()
729-
{
730-
// Check if the function is volatile using the proper volatility API
731-
if scalar_function.fun().signature().volatility == Volatility::Volatile {
732-
// Volatile functions should not be pushed down
733-
return false;
734-
}
735-
}
736-
true
737-
}
738-
739725
#[cfg(test)]
740726
mod tests {
741727
use super::*;

0 commit comments

Comments
 (0)