Skip to content

Commit a8a3ad4

Browse files
1fanwangOmega359
authored andcommitted
fix: preserve nested log and power results (apache#25138)
## Which issue does this PR close? Closes apache#25137. ## Rationale for this change For column exponents of 2000 and -2000, a log-of-power query returned the exponents instead of positive and negative infinity. The query now preserves the evaluated results. ## What changes are included in this PR? Remove the logarithm identities and inverse power/log cancellation. Literal calls still fold through constant evaluation. Null propagation, negative-scale decimal handling and zero/one-exponent power rules remain unchanged. ## What is the testing strategy for this PR? The SQL regression compares nested log/power with explicit evaluation. These CLI checks ran on macOS arm64 with Rust 1.98.1: ```sh cargo run --locked --profile ci -j6 --target-dir target -p datafusion-cli -- --quiet --rc -c "SELECT log(2.0, power(2.0, a)), ln(power(2.0, a)) / ln(2.0) FROM (VALUES (2000.0::double), (-2000.0::double)) t(a);" DATAFUSION_EXPLAIN_FORMAT=indent cargo run --locked --profile ci -j6 --target-dir target -p datafusion-cli -- --quiet --rc -c "EXPLAIN SELECT log(2.0, 1.0) AS zero, log(2.0, 2.0) AS one;" ``` <details><summary>Raw logs</summary> Before, on apache@0661d60: ```text | 2000.0 | inf | | -2000.0 | -inf | ``` After, the same query through the CLI: ```text | inf | inf | | -inf | -inf | cli_exit_code=0 ``` Literal calls still fold: ```text | logical_plan | Projection: Float64(0) AS zero, Float64(1) AS one | | physical_plan | ProjectionExec: expr=[0 as zero, 1 as one] | ``` </details> ## Are there any user-facing changes? These are result corrections. There is no public API change. --------- Signed-off-by: 1fanwang <[email protected]>
1 parent 503ceef commit a8a3ad4

4 files changed

Lines changed: 77 additions & 136 deletions

File tree

‎datafusion/core/tests/expr_api/simplification.rs‎

Lines changed: 26 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -606,24 +606,29 @@ fn test_simplify_with_cycle_count(
606606

607607
#[test]
608608
fn test_simplify_log() {
609-
// Log(c3, 1) ===> 0
610609
{
611-
let expr = log(col("c3_non_null"), lit(1));
612-
test_simplify(expr, lit(0i64));
610+
let expr = log(lit(2.0), lit(1.0));
611+
test_simplify(expr, lit(0.0));
613612
}
614-
// Log(c3, c3) ===> 1
615613
{
616-
let expr = log(col("c3_non_null"), col("c3_non_null"));
617-
let expected = lit(1i64);
614+
let expr = log(lit(2.0), lit(2.0));
615+
let expected = lit(1.0);
618616
test_simplify(expr, expected);
619617
}
620-
// Log(c3, Power(c3, c4)) ===> c4
621618
{
622-
let expr = log(
623-
col("c3_non_null"),
624-
power(col("c3_non_null"), col("c4_non_null")),
625-
);
626-
let expected = col("c4_non_null");
619+
let expr = log(lit(2i64), power(lit(2i64), col("c4_non_null")));
620+
test_simplify(expr.clone(), expr);
621+
}
622+
// Log(c3, 1) ===> Log(c3, 1), since c3 may be 1, 0 or negative
623+
{
624+
let expr = log(col("c3_non_null"), lit(1));
625+
let expected = log(col("c3_non_null"), lit(1));
626+
test_simplify(expr, expected);
627+
}
628+
// Log(c3, c3) ===> Log(c3, c3)
629+
{
630+
let expr = log(col("c3_non_null"), col("c3_non_null"));
631+
let expected = log(col("c3_non_null"), col("c3_non_null"));
627632
test_simplify(expr, expected);
628633
}
629634
// Log(c3, c4) ===> Log(c3, c4)
@@ -649,21 +654,22 @@ fn test_simplify_power() {
649654
Expr::Cast(Cast::new(Box::new(col("c3_non_null")), DataType::Float64));
650655
test_simplify(expr, expected)
651656
}
652-
// Power(c3, Log(c3, c4)) ===> cast(c4 AS Float64)
653-
// The simplifier rewrites `power(b, log(b, x))` to `x`, but the
654-
// rewritten expression must keep the same type as the original
655-
// `power` call. `power` returns Float64, so the UInt32 c4 has to be cast
656-
// to Float64 to preserve the output schema the optimizer already
657-
// committed to.
657+
// Power(c3, Log(c3, c4)) is left alone, since c3 may be 1, 0 or negative.
658658
{
659659
let expr = power(
660660
col("c3_non_null"),
661661
log(col("c3_non_null"), col("c4_non_null")),
662662
);
663-
let expected =
664-
Expr::Cast(Cast::new(Box::new(col("c4_non_null")), DataType::Float64));
663+
let expected = power(
664+
col("c3_non_null"),
665+
log(col("c3_non_null"), col("c4_non_null")),
666+
);
665667
test_simplify(expr, expected)
666668
}
669+
{
670+
let expr = power(lit(2i64), log(lit(2i64), col("c4_non_null")));
671+
test_simplify(expr.clone(), expr)
672+
}
667673
// Power(c3, c4) ===> Power(c3, c4)
668674
{
669675
let expr = power(col("c3_non_null"), col("c4_non_null"));

‎datafusion/functions/src/math/log.rs‎

Lines changed: 18 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@
1717

1818
//! Math function: `log()`.
1919
20-
use super::power::PowerFunc;
21-
2220
use crate::utils::calculate_binary_math;
2321
use arrow::array::{Array, ArrayRef};
2422
use arrow::datatypes::{
@@ -28,15 +26,12 @@ use arrow::datatypes::{
2826
use arrow::error::ArrowError;
2927
use arrow_buffer::i256;
3028
use datafusion_common::types::NativeType;
31-
use datafusion_common::{
32-
Result, ScalarValue, exec_err, internal_err, plan_datafusion_err, plan_err,
33-
};
34-
use datafusion_expr::expr::ScalarFunction;
29+
use datafusion_common::{Result, ScalarValue, exec_err, plan_datafusion_err, plan_err};
3530
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
3631
use datafusion_expr::sort_properties::{ExprProperties, SortProperties};
3732
use datafusion_expr::{
38-
Coercion, ColumnarValue, Documentation, Expr, ScalarFunctionArgs, ScalarUDF,
39-
TypeSignature, TypeSignatureClass, lit,
33+
Coercion, ColumnarValue, Documentation, Expr, ScalarFunctionArgs, TypeSignature,
34+
TypeSignatureClass, lit,
4035
};
4136
use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};
4237
use datafusion_macros::user_doc;
@@ -312,16 +307,12 @@ impl ScalarUDFImpl for LogFunc {
312307
self.doc()
313308
}
314309

315-
/// Simplify the `log` function by the relevant rules:
316-
/// 1. Log(a, 1) ===> 0
317-
/// 2. Log(a, Power(a, b)) ===> b
318-
/// 3. Log(a, a) ===> 1
319310
fn simplify(
320311
&self,
321-
mut args: Vec<Expr>,
312+
args: Vec<Expr>,
322313
info: &SimplifyContext,
323314
) -> Result<ExprSimplifyResult> {
324-
let mut arg_types = args
315+
let arg_types = args
325316
.iter()
326317
.map(|arg| info.get_data_type(arg))
327318
.collect::<Result<Vec<_>>>()?;
@@ -354,60 +345,10 @@ impl ScalarUDFImpl for LogFunc {
354345
_ => (),
355346
}
356347

357-
let number = args.pop().unwrap();
358-
let number_datatype = arg_types.pop().unwrap();
359-
// default to base 10
360-
let base = if let Some(base) = args.pop() {
361-
base
362-
} else {
363-
lit(ScalarValue::new_ten(&number_datatype)?)
364-
};
365-
let base_nullable = info.nullable(&base)?;
366-
367-
match number {
368-
Expr::Literal(value, _)
369-
if value == ScalarValue::new_one(&number_datatype)? && !base_nullable =>
370-
{
371-
Ok(ExprSimplifyResult::Simplified(lit(ScalarValue::new_zero(
372-
&info.get_data_type(&base)?,
373-
)?)))
374-
}
375-
Expr::ScalarFunction(ScalarFunction { func, mut args })
376-
if is_pow(&func)
377-
&& args.len() == 2
378-
&& base == args[0]
379-
&& !base_nullable =>
380-
{
381-
let b = args.pop().unwrap(); // length checked above
382-
Ok(ExprSimplifyResult::Simplified(b))
383-
}
384-
number => {
385-
if number == base && !base_nullable {
386-
Ok(ExprSimplifyResult::Simplified(lit(ScalarValue::new_one(
387-
&number_datatype,
388-
)?)))
389-
} else {
390-
let args = match num_args {
391-
1 => vec![number],
392-
2 => vec![base, number],
393-
_ => {
394-
return internal_err!(
395-
"Unexpected number of arguments in log::simplify"
396-
);
397-
}
398-
};
399-
Ok(ExprSimplifyResult::Original(args))
400-
}
401-
}
402-
}
348+
Ok(ExprSimplifyResult::Original(args))
403349
}
404350
}
405351

406-
/// Returns true if the function is `PowerFunc`
407-
fn is_pow(func: &ScalarUDF) -> bool {
408-
func.inner().is::<PowerFunc>()
409-
}
410-
411352
#[cfg(test)]
412353
mod tests {
413354
use std::sync::Arc;
@@ -766,26 +707,20 @@ mod tests {
766707
}
767708

768709
#[test]
769-
// Test that non-simplifiable log() expressions are unchanged after simplification
770710
fn test_log_simplify_original() {
771711
let context = SimplifyContext::default();
772-
// One argument with no simplifications
773-
let result = LogFunc::new().simplify(vec![lit(2)], &context).unwrap();
774-
let ExprSimplifyResult::Original(args) = result else {
775-
panic!("Expected ExprSimplifyResult::Original")
776-
};
777-
assert_eq!(args.len(), 1);
778-
assert_eq!(args[0], lit(2));
779-
// Two arguments with no simplifications
780-
let result = LogFunc::new()
781-
.simplify(vec![lit(2), lit(3)], &context)
782-
.unwrap();
783-
let ExprSimplifyResult::Original(args) = result else {
784-
panic!("Expected ExprSimplifyResult::Original")
785-
};
786-
assert_eq!(args.len(), 2);
787-
assert_eq!(args[0], lit(2));
788-
assert_eq!(args[1], lit(3));
712+
for args in [
713+
vec![lit(2)],
714+
vec![lit(2), lit(3)],
715+
vec![lit(2.0), lit(1.0)],
716+
vec![lit(2.0), lit(2.0)],
717+
] {
718+
let result = LogFunc::new().simplify(args.clone(), &context).unwrap();
719+
let ExprSimplifyResult::Original(actual) = result else {
720+
panic!("Expected ExprSimplifyResult::Original")
721+
};
722+
assert_eq!(actual, args);
723+
}
789724
}
790725

791726
#[test]

‎datafusion/functions/src/math/power.rs‎

Lines changed: 1 addition & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,16 @@
1616
// under the License.
1717

1818
//! Math function: `power()`.
19-
use super::log::LogFunc;
20-
2119
use crate::utils::calculate_binary_math;
2220
use arrow::array::{Array, ArrayRef};
2321
use arrow::datatypes::{DataType, Float64Type};
2422
use arrow::error::ArrowError;
2523
use datafusion_common::types::{NativeType, logical_float64};
2624
use datafusion_common::utils::take_function_args;
2725
use datafusion_common::{Result, ScalarValue, internal_err};
28-
use datafusion_expr::expr::ScalarFunction;
2926
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
3027
use datafusion_expr::{
31-
Cast, Coercion, ColumnarValue, Documentation, Expr, ScalarFunctionArgs, ScalarUDF,
28+
Cast, Coercion, ColumnarValue, Documentation, Expr, ScalarFunctionArgs,
3229
ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, lit,
3330
};
3431
use datafusion_macros::user_doc;
@@ -131,7 +128,6 @@ impl ScalarUDFImpl for PowerFunc {
131128
/// Simplify the `power` function by the relevant rules:
132129
/// 1. Power(a, 0) ===> 1
133130
/// 2. Power(a, 1) ===> a
134-
/// 3. Power(a, Log(a, b)) ===> b
135131
fn simplify(
136132
&self,
137133
args: Vec<Expr>,
@@ -151,13 +147,6 @@ impl ScalarUDFImpl for PowerFunc {
151147
)));
152148
}
153149

154-
// `simplify` runs on the logical expression *before* type coercion,
155-
// so a simplified sub-expression may still carry its original type
156-
// rather than the Float64 that `power` is declared to return. Cast it
157-
// back when needed to preserve the schema the optimizer already
158-
// committed to — e.g. `power(int_col, 1)` simplifies to `int_col`,
159-
// and the `b` in `power(b, log(b, uint_col))` simplifies to `uint_col`,
160-
// both of which must become Float64.
161150
let cast_to_return_type = |expr: Expr, expr_type: &DataType| {
162151
if expr_type == &return_type {
163152
expr
@@ -179,18 +168,6 @@ impl ScalarUDFImpl for PowerFunc {
179168
base, &base_type,
180169
)))
181170
}
182-
Expr::ScalarFunction(ScalarFunction { func, mut args })
183-
if is_log(&func)
184-
&& args.len() == 2
185-
&& base == args[0]
186-
&& !base_nullable =>
187-
{
188-
let b = args.pop().unwrap(); // length checked above
189-
let b_type = info.get_data_type(&b)?;
190-
Ok(ExprSimplifyResult::Simplified(cast_to_return_type(
191-
b, &b_type,
192-
)))
193-
}
194171
_ => Ok(ExprSimplifyResult::Original(vec![base, exponent])),
195172
}
196173
}
@@ -200,11 +177,6 @@ impl ScalarUDFImpl for PowerFunc {
200177
}
201178
}
202179

203-
/// Return true if this function call is a call to `Log`
204-
fn is_log(func: &ScalarUDF) -> bool {
205-
func.inner().is::<LogFunc>()
206-
}
207-
208180
#[cfg(test)]
209181
mod tests {
210182
use super::*;

‎datafusion/sqllogictest/test_files/math.slt‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1213,7 +1213,9 @@ physical_plan
12131213
01)ProjectionExec: expr=[log(column1@0, 1) as l1, log(column1@0, column1@0) as la, power(column1@0, 0) as p0, power(column1@0, log(column1@0, column2@1)) as pl]
12141214
02)--DataSourceExec: partitions=1, partition_sizes=[1]
12151215

1216-
# Non-nullable bases still use the existing simplifications
1216+
# A column base is left alone even when it is not nullable, because its value is
1217+
# not known to be outside {0, 1, negative}. power(a, 0) does not depend on the
1218+
# base value, so it still folds.
12171219
query TT
12181220
EXPLAIN SELECT
12191221
log(a, 1) AS l1,
@@ -1223,14 +1225,40 @@ EXPLAIN SELECT
12231225
FROM (VALUES (2.0::double, 3.0::double)) AS t(a, b);
12241226
----
12251227
logical_plan
1226-
01)Projection: Float64(0) AS l1, Float64(1) AS la, Float64(1) AS p0, t.b AS pl
1228+
01)Projection: log(t.a, Float64(1)) AS l1, log(t.a, t.a) AS la, Float64(1) AS p0, power(t.a, log(t.a, t.b)) AS pl
12271229
02)--SubqueryAlias: t
1228-
03)----Projection: column2 AS b
1230+
03)----Projection: column1 AS a, column2 AS b
12291231
04)------Values: (Float64(2), Float64(3))
12301232
physical_plan
1231-
01)ProjectionExec: expr=[0 as l1, 1 as la, 1 as p0, column2@1 as pl]
1233+
01)ProjectionExec: expr=[log(column1@0, 1) as l1, log(column1@0, column1@0) as la, 1 as p0, power(column1@0, log(column1@0, column2@1)) as pl]
12321234
02)--DataSourceExec: partitions=1, partition_sizes=[1]
12331235

1236+
query RR
1237+
SELECT power(2.0, log(2.0, a)), power(b, log(b, a))
1238+
FROM (VALUES (-3.0::double, 2.0::double)) t(a, b);
1239+
----
1240+
NaN NaN
1241+
1242+
query I
1243+
SELECT count(*) FROM (VALUES (-3.0::double)) t(a)
1244+
WHERE power(2.0, log(2.0, a)) = -3.0;
1245+
----
1246+
0
1247+
1248+
query R rowsort
1249+
SELECT power(2.0, log(2.0, a))
1250+
FROM (VALUES (4.0::double), (NULL::double)) t(a);
1251+
----
1252+
4
1253+
NULL
1254+
1255+
query RR
1256+
SELECT log(2.0, power(2.0, a)), ln(power(2.0, a)) / ln(2.0)
1257+
FROM (VALUES (2000.0::double), (-2000.0::double)) t(a);
1258+
----
1259+
Infinity Infinity
1260+
-Infinity -Infinity
1261+
12341262
# Float 16/32/64 for log
12351263
query RT
12361264
SELECT log(2.5, arrow_cast(10.9, 'Float16')), arrow_typeof(log(2.5, arrow_cast(10.9, 'Float16')));

0 commit comments

Comments
 (0)