Skip to content

Commit 31012b5

Browse files
lyne7-sclukekim
authored andcommitted
fix: preserve NULL semantics in log and power simplification (apache#24247)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Part of apache#24246. ## Rationale for this change <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. Please explain the problem you are trying to solve in terms of the user-visible behavior, rather than the implementation. For example, "The code in `foo.rs` doesn't handle nulls" is a symptom of the implementation. "COUNT(DISTINCT) returns wrong results when the column contains nulls" is the user-visible problem. --> `log` and `power` simplifications removed a nullable base expression, which could incorrectly produce a non-NULL result when the base was NULL. For example: ```sql SELECT log(a, 1.0), power(a, 0.0) FROM (VALUES (NULL::DOUBLE)) AS t(a); ``` These expressions should both return NULL, but simplification could replace them with 0.0 and 1.0. ## What changes are included in this PR? only apply these simplifications when the removed base expression is non-nullable. <!-- There is no need to duplicate the description in the issue here, but it is sometimes worth providing a summary of the individual changes in this PR. --> ## Are these changes tested? <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> Yes. Added sqllogictests covering results and plans. ## Are there any user-facing changes? Yes. log and power expressions with nullable bases now correctly preserve NULLs. <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. If there are any breaking changes to public APIs, please add the `api change` label. --> (cherry picked from commit c08832d) Conflict resolved when applying to spiceai-54 (DataFusion 54.1): - datafusion/functions/src/math/power.rs: context only. Upstream computes return_type ahead of the null-propagation branch, which 54.1 does not; added this commit's base_nullable binding in the 54.1 position. Both `&& !base_nullable` guards apply to the 54.1 arms unchanged.
1 parent f6222e3 commit 31012b5

3 files changed

Lines changed: 62 additions & 5 deletions

File tree

‎datafusion/functions/src/math/log.rs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -358,23 +358,27 @@ impl ScalarUDFImpl for LogFunc {
358358
} else {
359359
lit(ScalarValue::new_ten(&number_datatype)?)
360360
};
361+
let base_nullable = info.nullable(&base)?;
361362

362363
match number {
363364
Expr::Literal(value, _)
364-
if value == ScalarValue::new_one(&number_datatype)? =>
365+
if value == ScalarValue::new_one(&number_datatype)? && !base_nullable =>
365366
{
366367
Ok(ExprSimplifyResult::Simplified(lit(ScalarValue::new_zero(
367368
&info.get_data_type(&base)?,
368369
)?)))
369370
}
370371
Expr::ScalarFunction(ScalarFunction { func, mut args })
371-
if is_pow(&func) && args.len() == 2 && base == args[0] =>
372+
if is_pow(&func)
373+
&& args.len() == 2
374+
&& base == args[0]
375+
&& !base_nullable =>
372376
{
373377
let b = args.pop().unwrap(); // length checked above
374378
Ok(ExprSimplifyResult::Simplified(b))
375379
}
376380
number => {
377-
if number == base {
381+
if number == base && !base_nullable {
378382
Ok(ExprSimplifyResult::Simplified(lit(ScalarValue::new_one(
379383
&number_datatype,
380384
)?)))

‎datafusion/functions/src/math/power.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -573,6 +573,7 @@ impl ScalarUDFImpl for PowerFunc {
573573
let [base, exponent] = take_function_args("power", args)?;
574574
let base_type = info.get_data_type(&base)?;
575575
let exponent_type = info.get_data_type(&exponent)?;
576+
let base_nullable = info.nullable(&base)?;
576577

577578
// Null propagation
578579
if base_type.is_null() || exponent_type.is_null() {
@@ -584,7 +585,7 @@ impl ScalarUDFImpl for PowerFunc {
584585

585586
match exponent {
586587
Expr::Literal(value, _)
587-
if value == ScalarValue::new_zero(&exponent_type)? =>
588+
if value == ScalarValue::new_zero(&exponent_type)? && !base_nullable =>
588589
{
589590
Ok(ExprSimplifyResult::Simplified(lit(ScalarValue::new_one(
590591
&base_type,
@@ -594,7 +595,10 @@ impl ScalarUDFImpl for PowerFunc {
594595
Ok(ExprSimplifyResult::Simplified(base))
595596
}
596597
Expr::ScalarFunction(ScalarFunction { func, mut args })
597-
if is_log(&func) && args.len() == 2 && base == args[0] =>
598+
if is_log(&func)
599+
&& args.len() == 2
600+
&& base == args[0]
601+
&& !base_nullable =>
598602
{
599603
let b = args.pop().unwrap(); // length checked above
600604
Ok(ExprSimplifyResult::Simplified(b))

‎datafusion/sqllogictest/test_files/math.slt‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -864,6 +864,55 @@ logical_plan
864864
02)--TableScan: aggregate_simple projection=[]
865865
physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/aggregate_simple.csv]]}, projection=[NULL as log(NULL,aggregate_simple.c2)], file_type=csv, has_header=true
866866

867+
# Simplification must preserve NULLs from nullable columns
868+
query RRRRR rowsort
869+
SELECT
870+
log(a, 1),
871+
log(a, a),
872+
log(a, power(a, b)),
873+
power(a, 0),
874+
power(a, log(a, b))
875+
FROM (VALUES (NULL::double, 2.0::double), (2.0, 3.0)) AS t(a, b);
876+
----
877+
0 1 3 1 3
878+
NULL NULL NULL NULL NULL
879+
880+
# Nullable bases must remain in the optimized plan so they can propagate NULL
881+
query TT
882+
EXPLAIN SELECT
883+
log(a, 1) AS l1,
884+
log(a, a) AS la,
885+
power(a, 0) AS p0,
886+
power(a, log(a, b)) AS pl
887+
FROM (VALUES (NULL::double, 2.0::double), (2.0, 3.0)) AS t(a, b);
888+
----
889+
logical_plan
890+
01)Projection: log(t.a, Float64(1)) AS l1, log(t.a, t.a) AS la, power(t.a, Float64(0)) AS p0, power(t.a, log(t.a, t.b)) AS pl
891+
02)--SubqueryAlias: t
892+
03)----Projection: column1 AS a, column2 AS b
893+
04)------Values: (Float64(NULL) AS NULL, Float64(2)), (Float64(2), Float64(3))
894+
physical_plan
895+
01)ProjectionExec: expr=[log(column1@0, 1) as l1, log(column1@0, column1@0) as la, power(column1@0, 0) as p0, power(column1@0, log(column1@0, column2@1)) as pl]
896+
02)--DataSourceExec: partitions=1, partition_sizes=[1]
897+
898+
# Non-nullable bases still use the existing simplifications
899+
query TT
900+
EXPLAIN SELECT
901+
log(a, 1) AS l1,
902+
log(a, a) AS la,
903+
power(a, 0) AS p0,
904+
power(a, log(a, b)) AS pl
905+
FROM (VALUES (2.0::double, 3.0::double)) AS t(a, b);
906+
----
907+
logical_plan
908+
01)Projection: Float64(0) AS l1, Float64(1) AS la, Float64(1) AS p0, t.b AS pl
909+
02)--SubqueryAlias: t
910+
03)----Projection: column2 AS b
911+
04)------Values: (Float64(2), Float64(3))
912+
physical_plan
913+
01)ProjectionExec: expr=[0 as l1, 1 as la, 1 as p0, column2@1 as pl]
914+
02)--DataSourceExec: partitions=1, partition_sizes=[1]
915+
867916
# Float 16/32/64 for log
868917
query RT
869918
SELECT log(2.5, arrow_cast(10.9, 'Float16')), arrow_typeof(log(2.5, arrow_cast(10.9, 'Float16')));

0 commit comments

Comments
 (0)