Skip to content

Commit e040688

Browse files
committed
Merge apache#25476: fix: evaluate a volatile BETWEEN value one time
2 parents 70c2e4c + 1c10c5f commit e040688

12 files changed

Lines changed: 784 additions & 8 deletions

File tree

‎datafusion/core/tests/user_defined/user_defined_scalar_functions.rs‎

Lines changed: 95 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,11 @@
55
use std::collections::HashMap;
66
use std::hash::{Hash, Hasher};
77
use std::sync::Arc;
8+
use std::sync::atomic::{AtomicUsize, Ordering};
89

910
use arrow::array::{
10-
Array, ArrayRef, Float32Array, Float64Array, Int16Array, Int32Array, RecordBatch,
11-
StringArray, builder::BooleanBuilder, cast::AsArray,
11+
Array, ArrayRef, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array,
12+
RecordBatch, StringArray, builder::BooleanBuilder, cast::AsArray,
1213
};
1314
use arrow::array::{Int8Array, UInt64Array, as_string_array, create_array, record_batch};
1415
use arrow::compute::kernels::numeric::add;
@@ -2341,3 +2342,95 @@ async fn test_extension_metadata_preserve_in_uncorrelated_scalar_subquery() -> R
23412342
assert!(!batches.is_empty());
23422343
Ok(())
23432344
}
2345+
2346+
/// Volatile UDF that counts how many values it produces.
2347+
///
2348+
/// Each row gets the next value of the shared counter, so the counter also
2349+
/// gives the number of rows that the function was evaluated for.
2350+
#[derive(Debug)]
2351+
struct CountingVolatileUdf {
2352+
signature: Signature,
2353+
rows_evaluated: Arc<AtomicUsize>,
2354+
}
2355+
2356+
impl PartialEq for CountingVolatileUdf {
2357+
fn eq(&self, other: &Self) -> bool {
2358+
Arc::ptr_eq(&self.rows_evaluated, &other.rows_evaluated)
2359+
}
2360+
}
2361+
2362+
impl Eq for CountingVolatileUdf {}
2363+
2364+
impl Hash for CountingVolatileUdf {
2365+
fn hash<H: Hasher>(&self, state: &mut H) {
2366+
self.signature.hash(state);
2367+
}
2368+
}
2369+
2370+
impl ScalarUDFImpl for CountingVolatileUdf {
2371+
fn name(&self) -> &str {
2372+
"counting_volatile"
2373+
}
2374+
2375+
fn signature(&self) -> &Signature {
2376+
&self.signature
2377+
}
2378+
2379+
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
2380+
Ok(DataType::Int64)
2381+
}
2382+
2383+
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
2384+
let first = self
2385+
.rows_evaluated
2386+
.fetch_add(args.number_rows, Ordering::SeqCst) as i64;
2387+
let values = (0..args.number_rows as i64).map(|offset| first + offset);
2388+
Ok(ColumnarValue::Array(Arc::new(
2389+
Int64Array::from_iter_values(values),
2390+
)))
2391+
}
2392+
}
2393+
2394+
/// `BETWEEN` must evaluate a volatile value one time per row.
2395+
///
2396+
/// See <https://github.com/apache/datafusion/issues/25457>.
2397+
#[tokio::test]
2398+
async fn between_evaluates_a_volatile_value_one_time_per_row() -> Result<()> {
2399+
let row_count = 100;
2400+
2401+
for predicate in [
2402+
"counting_volatile() BETWEEN 0 AND 1000000",
2403+
"counting_volatile() NOT BETWEEN 0 AND 1000000",
2404+
] {
2405+
let rows_evaluated = Arc::new(AtomicUsize::new(0));
2406+
let udf = ScalarUDF::from(CountingVolatileUdf {
2407+
signature: Signature::exact(vec![], Volatility::Volatile),
2408+
rows_evaluated: Arc::clone(&rows_evaluated),
2409+
});
2410+
2411+
let ctx = SessionContext::new_with_config(
2412+
SessionConfig::new().with_target_partitions(1),
2413+
);
2414+
ctx.register_udf(udf);
2415+
ctx.register_batch(
2416+
"t",
2417+
RecordBatch::try_from_iter(vec![(
2418+
"a",
2419+
Arc::new(Int64Array::from_iter_values(0..row_count as i64)) as ArrayRef,
2420+
)])?,
2421+
)?;
2422+
2423+
ctx.sql(&format!("SELECT a FROM t WHERE {predicate}"))
2424+
.await?
2425+
.collect()
2426+
.await?;
2427+
2428+
let evaluated = rows_evaluated.load(Ordering::SeqCst);
2429+
assert_eq!(
2430+
evaluated, row_count,
2431+
"{predicate} evaluated the volatile value {evaluated} times for {row_count} rows"
2432+
);
2433+
}
2434+
2435+
Ok(())
2436+
}

‎datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1649,6 +1649,15 @@ impl TreeNodeRewriter for Simplifier<'_> {
16491649

16501650
// a between 3 and 5 --> a >= 3 AND a <=5
16511651
// a not between 3 and 5 --> a < 3 OR a > 5
1652+
//
1653+
// The rewrite names `a` two times, so it also evaluates `a` two
1654+
// times. A volatile `a` gives a different value each time, so keep
1655+
// `BETWEEN` together and let the physical planner evaluate the
1656+
// value one time. See
1657+
// https://github.com/apache/datafusion/issues/25457
1658+
Expr::Between(between) if between.expr.is_volatile() => {
1659+
Transformed::no(Expr::Between(between))
1660+
}
16521661
Expr::Between(between) => Transformed::yes(if between.negated {
16531662
let l = *between.expr.clone();
16541663
let r = *between.expr;
@@ -5293,6 +5302,19 @@ mod tests {
52935302
);
52945303
}
52955304

5305+
#[test]
5306+
fn simplify_expr_between_keeps_a_volatile_value_together() {
5307+
// Expanding would evaluate the volatile value two times
5308+
let fun = Arc::new(ScalarUDF::new_from_impl(VolatileUdf::new()));
5309+
let volatile = Expr::ScalarFunction(ScalarFunction::new_udf(fun, vec![]));
5310+
for expr in [
5311+
volatile.clone().between(lit(1i16), lit(2i16)),
5312+
volatile.not_between(lit(1i16), lit(2i16)),
5313+
] {
5314+
assert_eq!(simplify(expr.clone()), expr);
5315+
}
5316+
}
5317+
52965318
#[test]
52975319
fn test_like_and_ilike() {
52985320
let null = lit(ScalarValue::Utf8(None));

0 commit comments

Comments
 (0)