Skip to content

Commit 67f1371

Browse files
committed
fix: preserve logarithm overflow results
Signed-off-by: 1fanwang <[email protected]>
1 parent 0661d60 commit 67f1371

3 files changed

Lines changed: 28 additions & 168 deletions

File tree

‎datafusion/core/tests/expr_api/simplification.rs‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -606,24 +606,18 @@ fn test_simplify_with_cycle_count(
606606

607607
#[test]
608608
fn test_simplify_log() {
609-
// These identities hold only for a base in (0, 1) union (1, inf), so they
610-
// apply to a literal base whose value proves it, not to a column.
611-
// Log(2, 1) ===> 0
612609
{
613-
let expr = log(lit(2i64), lit(1));
614-
test_simplify(expr, lit(0i64));
610+
let expr = log(lit(2.0), lit(1.0));
611+
test_simplify(expr, lit(0.0));
615612
}
616-
// Log(2, 2) ===> 1
617613
{
618-
let expr = log(lit(2i64), lit(2i64));
619-
let expected = lit(1i64);
614+
let expr = log(lit(2.0), lit(2.0));
615+
let expected = lit(1.0);
620616
test_simplify(expr, expected);
621617
}
622-
// Log(2, Power(2, c4)) ===> c4
623618
{
624619
let expr = log(lit(2i64), power(lit(2i64), col("c4_non_null")));
625-
let expected = col("c4_non_null");
626-
test_simplify(expr, expected);
620+
test_simplify(expr.clone(), expr);
627621
}
628622
// Log(c3, 1) ===> Log(c3, 1), since c3 may be 1, 0 or negative
629623
{

‎datafusion/functions/src/math/log.rs‎

Lines changed: 16 additions & 157 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@
1717

1818
//! Math function: `log()`.
1919
20-
use super::power::PowerFunc;
21-
2220
use crate::utils::calculate_binary_math;
2321
use arrow::array::{Array, ArrayRef};
2422
use arrow::datatypes::{
@@ -28,15 +26,12 @@ use arrow::datatypes::{
2826
use arrow::error::ArrowError;
2927
use arrow_buffer::i256;
3028
use datafusion_common::types::NativeType;
31-
use datafusion_common::{
32-
Result, ScalarValue, exec_err, internal_err, plan_datafusion_err, plan_err,
33-
};
34-
use datafusion_expr::expr::ScalarFunction;
29+
use datafusion_common::{Result, ScalarValue, exec_err, plan_datafusion_err, plan_err};
3530
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
3631
use datafusion_expr::sort_properties::{ExprProperties, SortProperties};
3732
use datafusion_expr::{
38-
Coercion, ColumnarValue, Documentation, Expr, ScalarFunctionArgs, ScalarUDF,
39-
TypeSignature, TypeSignatureClass, lit,
33+
Coercion, ColumnarValue, Documentation, Expr, ScalarFunctionArgs, TypeSignature,
34+
TypeSignatureClass, lit,
4035
};
4136
use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};
4237
use datafusion_macros::user_doc;
@@ -312,16 +307,12 @@ impl ScalarUDFImpl for LogFunc {
312307
self.doc()
313308
}
314309

315-
/// Simplify the `log` function by the relevant rules:
316-
/// 1. Log(a, 1) ===> 0
317-
/// 2. Log(a, Power(a, b)) ===> b
318-
/// 3. Log(a, a) ===> 1
319310
fn simplify(
320311
&self,
321-
mut args: Vec<Expr>,
312+
args: Vec<Expr>,
322313
info: &SimplifyContext,
323314
) -> Result<ExprSimplifyResult> {
324-
let mut arg_types = args
315+
let arg_types = args
325316
.iter()
326317
.map(|arg| info.get_data_type(arg))
327318
.collect::<Result<Vec<_>>>()?;
@@ -354,78 +345,10 @@ impl ScalarUDFImpl for LogFunc {
354345
_ => (),
355346
}
356347

357-
let number = args.pop().unwrap();
358-
let number_datatype = arg_types.pop().unwrap();
359-
// default to base 10
360-
let base = if let Some(base) = args.pop() {
361-
base
362-
} else {
363-
lit(ScalarValue::new_ten(&number_datatype)?)
364-
};
365-
366-
match number {
367-
Expr::Literal(value, _)
368-
if value == ScalarValue::new_one(&number_datatype)?
369-
&& is_valid_log_base(&base) =>
370-
{
371-
Ok(ExprSimplifyResult::Simplified(lit(ScalarValue::new_zero(
372-
&info.get_data_type(&base)?,
373-
)?)))
374-
}
375-
Expr::ScalarFunction(ScalarFunction { func, mut args })
376-
if is_pow(&func)
377-
&& args.len() == 2
378-
&& base == args[0]
379-
&& is_valid_log_base(&base) =>
380-
{
381-
let b = args.pop().unwrap(); // length checked above
382-
Ok(ExprSimplifyResult::Simplified(b))
383-
}
384-
number => {
385-
if number == base && is_valid_log_base(&base) {
386-
Ok(ExprSimplifyResult::Simplified(lit(ScalarValue::new_one(
387-
&number_datatype,
388-
)?)))
389-
} else {
390-
let args = match num_args {
391-
1 => vec![number],
392-
2 => vec![base, number],
393-
_ => {
394-
return internal_err!(
395-
"Unexpected number of arguments in log::simplify"
396-
);
397-
}
398-
};
399-
Ok(ExprSimplifyResult::Original(args))
400-
}
401-
}
402-
}
348+
Ok(ExprSimplifyResult::Original(args))
403349
}
404350
}
405351

406-
/// Returns true if the function is `PowerFunc`
407-
fn is_pow(func: &ScalarUDF) -> bool {
408-
func.inner().is::<PowerFunc>()
409-
}
410-
411-
/// Returns true when `base` is a literal that is a valid logarithm base.
412-
///
413-
/// The `log`/`power` identities hold only for a base in `(0, 1) ∪ (1, ∞)`.
414-
/// A base of 1 makes `log(base, x)` infinite, a base of 0 or a negative number
415-
/// makes it undefined, and an infinite base collapses the result to zero, so the
416-
/// rewrites are unsound for all of them. Only a literal proves the value, so a
417-
/// non-literal base keeps the original expression.
418-
pub(super) fn is_valid_log_base(base: &Expr) -> bool {
419-
let Expr::Literal(value, _) = base else {
420-
return false;
421-
};
422-
423-
matches!(
424-
value.cast_to(&DataType::Float64),
425-
Ok(ScalarValue::Float64(Some(value))) if value.is_finite() && value > 0.0 && value != 1.0
426-
)
427-
}
428-
429352
#[cfg(test)]
430353
mod tests {
431354
use std::sync::Arc;
@@ -784,86 +707,22 @@ mod tests {
784707
}
785708

786709
#[test]
787-
// Test that non-simplifiable log() expressions are unchanged after simplification
788710
fn test_log_simplify_original() {
789711
let context = SimplifyContext::default();
790-
// One argument with no simplifications
791-
let result = LogFunc::new().simplify(vec![lit(2)], &context).unwrap();
792-
let ExprSimplifyResult::Original(args) = result else {
793-
panic!("Expected ExprSimplifyResult::Original")
794-
};
795-
assert_eq!(args.len(), 1);
796-
assert_eq!(args[0], lit(2));
797-
// Two arguments with no simplifications
798-
let result = LogFunc::new()
799-
.simplify(vec![lit(2), lit(3)], &context)
800-
.unwrap();
801-
let ExprSimplifyResult::Original(args) = result else {
802-
panic!("Expected ExprSimplifyResult::Original")
803-
};
804-
assert_eq!(args.len(), 2);
805-
assert_eq!(args[0], lit(2));
806-
assert_eq!(args[1], lit(3));
807-
}
808-
809-
#[test]
810-
fn test_log_simplify_rejects_degenerate_base() {
811-
let context = SimplifyContext::default();
812-
813-
// log(base, 1) => 0, log(base, base) => 1 and log(base, power(base, b)) => b
814-
// hold only for a base in (0, 1) union (1, inf).
815-
for base in [
816-
lit(1.0),
817-
lit(0.0),
818-
lit(-2.0),
819-
lit(f64::NAN),
820-
lit(f64::INFINITY),
712+
for args in [
713+
vec![lit(2)],
714+
vec![lit(2), lit(3)],
715+
vec![lit(2.0), lit(1.0)],
716+
vec![lit(2.0), lit(2.0)],
821717
] {
822-
let result = LogFunc::new()
823-
.simplify(vec![base.clone(), lit(1.0)], &context)
824-
.unwrap();
825-
assert!(
826-
matches!(result, ExprSimplifyResult::Original(_)),
827-
"log({base:?}, 1) must not simplify"
828-
);
829-
830-
let result = LogFunc::new()
831-
.simplify(vec![base.clone(), base.clone()], &context)
832-
.unwrap();
833-
assert!(
834-
matches!(result, ExprSimplifyResult::Original(_)),
835-
"log({base:?}, {base:?}) must not simplify"
836-
);
837-
838-
let power = Expr::ScalarFunction(ScalarFunction::new_udf(
839-
Arc::new(ScalarUDF::from(PowerFunc::new())),
840-
vec![base.clone(), lit(2.0)],
841-
));
842-
let result = LogFunc::new()
843-
.simplify(vec![base.clone(), power], &context)
844-
.unwrap();
845-
assert!(
846-
matches!(result, ExprSimplifyResult::Original(_)),
847-
"log({base:?}, power({base:?}, 2)) must not simplify"
848-
);
718+
let result = LogFunc::new().simplify(args.clone(), &context).unwrap();
719+
let ExprSimplifyResult::Original(actual) = result else {
720+
panic!("Expected ExprSimplifyResult::Original")
721+
};
722+
assert_eq!(actual, args);
849723
}
850724
}
851725

852-
#[test]
853-
fn test_log_simplify_accepts_valid_base() {
854-
let context = SimplifyContext::default();
855-
856-
let result = LogFunc::new()
857-
.simplify(vec![lit(2.0), lit(1.0)], &context)
858-
.unwrap();
859-
assert!(matches!(result, ExprSimplifyResult::Simplified(_)));
860-
861-
let result = LogFunc::new()
862-
.simplify(vec![lit(2.0), lit(2.0)], &context)
863-
.unwrap();
864-
assert!(matches!(result, ExprSimplifyResult::Simplified(_)));
865-
}
866-
867726
#[test]
868727
fn test_log_output_ordering() {
869728
// [Unordered, Ascending, Descending, Literal]

‎datafusion/sqllogictest/test_files/math.slt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1252,6 +1252,13 @@ FROM (VALUES (4.0::double), (NULL::double)) t(a);
12521252
4
12531253
NULL
12541254

1255+
query RR
1256+
SELECT log(2.0, power(2.0, a)), ln(power(2.0, a)) / ln(2.0)
1257+
FROM (VALUES (2000.0::double), (-2000.0::double)) t(a);
1258+
----
1259+
Infinity Infinity
1260+
-Infinity -Infinity
1261+
12551262
# Float 16/32/64 for log
12561263
query RT
12571264
SELECT log(2.5, arrow_cast(10.9, 'Float16')), arrow_typeof(log(2.5, arrow_cast(10.9, 'Float16')));

0 commit comments

Comments
 (0)