Skip to content

Commit ea14846

Browse files
lyne7-sckosiew
authored andcommitted
fix: preserve NULL semantics in log and power simplification (apache#24247)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes apache#123` indicates that this PR will close issue apache#123. --> - Part of apache#24246. ## Rationale for this change <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. Please explain the problem you are trying to solve in terms of the user-visible behavior, rather than the implementation. For example, "The code in `foo.rs` doesn't handle nulls" is a symptom of the implementation. "COUNT(DISTINCT) returns wrong results when the column contains nulls" is the user-visible problem. --> `log` and `power` simplifications removed a nullable base expression, which could incorrectly produce a non-NULL result when the base was NULL. For example: ```sql SELECT log(a, 1.0), power(a, 0.0) FROM (VALUES (NULL::DOUBLE)) AS t(a); ``` These expressions should both return NULL, but simplification could replace them with 0.0 and 1.0. ## What changes are included in this PR? only apply these simplifications when the removed base expression is non-nullable. <!-- There is no need to duplicate the description in the issue here, but it is sometimes worth providing a summary of the individual changes in this PR. --> ## Are these changes tested? <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> Yes. Added sqllogictests covering results and plans. ## Are there any user-facing changes? Yes. log and power expressions with nullable bases now correctly preserve NULLs. <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. If there are any breaking changes to public APIs, please add the `api change` label. -->
1 parent b47bcf8 commit ea14846

3 files changed

Lines changed: 62 additions & 5 deletions

File tree

‎datafusion/functions/src/math/log.rs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -362,23 +362,27 @@ impl ScalarUDFImpl for LogFunc {
362362
} else {
363363
lit(ScalarValue::new_ten(&number_datatype)?)
364364
};
365+
let base_nullable = info.nullable(&base)?;
365366

366367
match number {
367368
Expr::Literal(value, _)
368-
if value == ScalarValue::new_one(&number_datatype)? =>
369+
if value == ScalarValue::new_one(&number_datatype)? && !base_nullable =>
369370
{
370371
Ok(ExprSimplifyResult::Simplified(lit(ScalarValue::new_zero(
371372
&info.get_data_type(&base)?,
372373
)?)))
373374
}
374375
Expr::ScalarFunction(ScalarFunction { func, mut args })
375-
if is_pow(&func) && args.len() == 2 && base == args[0] =>
376+
if is_pow(&func)
377+
&& args.len() == 2
378+
&& base == args[0]
379+
&& !base_nullable =>
376380
{
377381
let b = args.pop().unwrap(); // length checked above
378382
Ok(ExprSimplifyResult::Simplified(b))
379383
}
380384
number => {
381-
if number == base {
385+
if number == base && !base_nullable {
382386
Ok(ExprSimplifyResult::Simplified(lit(ScalarValue::new_one(
383387
&number_datatype,
384388
)?)))

‎datafusion/functions/src/math/power.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ impl ScalarUDFImpl for PowerFunc {
140140
let [base, exponent] = take_function_args("power", args)?;
141141
let base_type = info.get_data_type(&base)?;
142142
let exponent_type = info.get_data_type(&exponent)?;
143+
let base_nullable = info.nullable(&base)?;
143144
let return_type =
144145
self.return_type(&[base_type.clone(), exponent_type.clone()])?;
145146

@@ -167,7 +168,7 @@ impl ScalarUDFImpl for PowerFunc {
167168

168169
match exponent {
169170
Expr::Literal(value, _)
170-
if value == ScalarValue::new_zero(&exponent_type)? =>
171+
if value == ScalarValue::new_zero(&exponent_type)? && !base_nullable =>
171172
{
172173
Ok(ExprSimplifyResult::Simplified(lit(ScalarValue::new_one(
173174
&return_type,
@@ -179,7 +180,10 @@ impl ScalarUDFImpl for PowerFunc {
179180
)))
180181
}
181182
Expr::ScalarFunction(ScalarFunction { func, mut args })
182-
if is_log(&func) && args.len() == 2 && base == args[0] =>
183+
if is_log(&func)
184+
&& args.len() == 2
185+
&& base == args[0]
186+
&& !base_nullable =>
183187
{
184188
let b = args.pop().unwrap(); // length checked above
185189
let b_type = info.get_data_type(&b)?;

‎datafusion/sqllogictest/test_files/math.slt‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1182,6 +1182,55 @@ logical_plan
11821182
02)--TableScan: aggregate_simple projection=[]
11831183
physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/aggregate_simple.csv]]}, projection=[NULL as log(NULL,aggregate_simple.c2)], file_type=csv, has_header=true
11841184

1185+
# Simplification must preserve NULLs from nullable columns
1186+
query RRRRR rowsort
1187+
SELECT
1188+
log(a, 1),
1189+
log(a, a),
1190+
log(a, power(a, b)),
1191+
power(a, 0),
1192+
power(a, log(a, b))
1193+
FROM (VALUES (NULL::double, 2.0::double), (2.0, 3.0)) AS t(a, b);
1194+
----
1195+
0 1 3 1 3
1196+
NULL NULL NULL NULL NULL
1197+
1198+
# Nullable bases must remain in the optimized plan so they can propagate NULL
1199+
query TT
1200+
EXPLAIN SELECT
1201+
log(a, 1) AS l1,
1202+
log(a, a) AS la,
1203+
power(a, 0) AS p0,
1204+
power(a, log(a, b)) AS pl
1205+
FROM (VALUES (NULL::double, 2.0::double), (2.0, 3.0)) AS t(a, b);
1206+
----
1207+
logical_plan
1208+
01)Projection: log(t.a, Float64(1)) AS l1, log(t.a, t.a) AS la, power(t.a, Float64(0)) AS p0, power(t.a, log(t.a, t.b)) AS pl
1209+
02)--SubqueryAlias: t
1210+
03)----Projection: column1 AS a, column2 AS b
1211+
04)------Values: (Float64(NULL) AS NULL, Float64(2)), (Float64(2), Float64(3))
1212+
physical_plan
1213+
01)ProjectionExec: expr=[log(column1@0, 1) as l1, log(column1@0, column1@0) as la, power(column1@0, 0) as p0, power(column1@0, log(column1@0, column2@1)) as pl]
1214+
02)--DataSourceExec: partitions=1, partition_sizes=[1]
1215+
1216+
# Non-nullable bases still use the existing simplifications
1217+
query TT
1218+
EXPLAIN SELECT
1219+
log(a, 1) AS l1,
1220+
log(a, a) AS la,
1221+
power(a, 0) AS p0,
1222+
power(a, log(a, b)) AS pl
1223+
FROM (VALUES (2.0::double, 3.0::double)) AS t(a, b);
1224+
----
1225+
logical_plan
1226+
01)Projection: Float64(0) AS l1, Float64(1) AS la, Float64(1) AS p0, t.b AS pl
1227+
02)--SubqueryAlias: t
1228+
03)----Projection: column2 AS b
1229+
04)------Values: (Float64(2), Float64(3))
1230+
physical_plan
1231+
01)ProjectionExec: expr=[0 as l1, 1 as la, 1 as p0, column2@1 as pl]
1232+
02)--DataSourceExec: partitions=1, partition_sizes=[1]
1233+
11851234
# Float 16/32/64 for log
11861235
query RT
11871236
SELECT log(2.5, arrow_cast(10.9, 'Float16')), arrow_typeof(log(2.5, arrow_cast(10.9, 'Float16')));

0 commit comments

Comments
 (0)