Skip to content

Commit 7d9f743

Browse files
lyne7-sclukekim
authored andcommitted
fix: preserve NULL semantics in bitwise xor simplification (apache#24248)
## Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123. --> - Part of apache#24246 . ## Rationale for this change Bitwise XOR simplifications cancelled repeated nullable operands, which could incorrectly produce a non-NULL result when the operand was NULL. For example: ```sql SELECT i XOR i, (i XOR 7) XOR i, i XOR (7 XOR i) FROM (VALUES (NULL::INT)) AS t(i); ``` These expressions should all return NULL, but simplification could replace them with 0, 7, and 7. <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. Please explain the problem you are trying to solve in terms of the user-visible behavior, rather than the implementation. For example, "The code in `foo.rs` doesn't handle nulls" is a symptom of the implementation. "COUNT(DISTINCT) returns wrong results when the column contains nulls" is the user-visible problem. --> ## What changes are included in this PR? Only cancel repeated XOR operands when the removed operand is non-nullable. <!-- There is no need to duplicate the description in the issue here, but it is sometimes worth providing a summary of the individual changes in this PR. --> ## Are these changes tested? <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> Yes. Added sqllogictests covering results. ## Are there any user-facing changes? Yes. Bitwise XOR expressions with repeated nullable operands now correctly preserve NULLs. <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. If there are any breaking changes to public APIs, please add the `api change` label. --> (cherry picked from commit 50cdbde)
1 parent 79055fe commit 7d9f743

2 files changed

Lines changed: 73 additions & 26 deletions

File tree

‎datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs‎

Lines changed: 62 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1367,7 +1367,7 @@ impl TreeNodeRewriter for Simplifier<'_> {
13671367
left,
13681368
op: BitwiseXor,
13691369
right,
1370-
}) if expr_contains(&left, &right, BitwiseXor) => {
1370+
}) if !info.nullable(&right)? && expr_contains(&left, &right, BitwiseXor) => {
13711371
let expr = delete_xor_in_complex_expr(&left, &right, false);
13721372
Transformed::yes(if expr == *right {
13731373
Expr::Literal(
@@ -1384,7 +1384,7 @@ impl TreeNodeRewriter for Simplifier<'_> {
13841384
left,
13851385
op: BitwiseXor,
13861386
right,
1387-
}) if expr_contains(&right, &left, BitwiseXor) => {
1387+
}) if !info.nullable(&left)? && expr_contains(&right, &left, BitwiseXor) => {
13881388
let expr = delete_xor_in_complex_expr(&right, &left, true);
13891389
Transformed::yes(if expr == *left {
13901390
Expr::Literal(
@@ -2970,16 +2970,19 @@ mod tests {
29702970
// c2 ^ ((c2 ^ (c2 | c1)) ^ (c1 & c2)) --> (c2 | c1) ^ (c1 & c2)
29712971

29722972
let expr = bitwise_xor(
2973-
col("c2"),
2973+
col("c2_non_null"),
29742974
bitwise_xor(
2975-
bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))),
2976-
bitwise_and(col("c1"), col("c2")),
2975+
bitwise_xor(
2976+
col("c2_non_null"),
2977+
bitwise_or(col("c2_non_null"), col("c1")),
2978+
),
2979+
bitwise_and(col("c1"), col("c2_non_null")),
29772980
),
29782981
);
29792982

29802983
let expected = bitwise_xor(
2981-
bitwise_or(col("c2"), col("c1")),
2982-
bitwise_and(col("c1"), col("c2")),
2984+
bitwise_or(col("c2_non_null"), col("c1")),
2985+
bitwise_and(col("c1"), col("c2_non_null")),
29832986
);
29842987

29852988
assert_eq!(simplify(expr), expected);
@@ -2988,18 +2991,24 @@ mod tests {
29882991
// c2 ^ (c2 ^ (c2 | c1)) ^ ((c1 & c2) ^ c2) --> c2 ^ ((c2 | c1) ^ (c1 & c2))
29892992

29902993
let expr = bitwise_xor(
2991-
col("c2"),
2994+
col("c2_non_null"),
29922995
bitwise_xor(
2993-
bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))),
2994-
bitwise_xor(bitwise_and(col("c1"), col("c2")), col("c2")),
2996+
bitwise_xor(
2997+
col("c2_non_null"),
2998+
bitwise_or(col("c2_non_null"), col("c1")),
2999+
),
3000+
bitwise_xor(
3001+
bitwise_and(col("c1"), col("c2_non_null")),
3002+
col("c2_non_null"),
3003+
),
29953004
),
29963005
);
29973006

29983007
let expected = bitwise_xor(
2999-
col("c2"),
3008+
col("c2_non_null"),
30003009
bitwise_xor(
3001-
bitwise_or(col("c2"), col("c1")),
3002-
bitwise_and(col("c1"), col("c2")),
3010+
bitwise_or(col("c2_non_null"), col("c1")),
3011+
bitwise_and(col("c1"), col("c2_non_null")),
30033012
),
30043013
);
30053014

@@ -3010,15 +3019,18 @@ mod tests {
30103019

30113020
let expr = bitwise_xor(
30123021
bitwise_xor(
3013-
bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))),
3014-
bitwise_and(col("c1"), col("c2")),
3022+
bitwise_xor(
3023+
col("c2_non_null"),
3024+
bitwise_or(col("c2_non_null"), col("c1")),
3025+
),
3026+
bitwise_and(col("c1"), col("c2_non_null")),
30153027
),
3016-
col("c2"),
3028+
col("c2_non_null"),
30173029
);
30183030

30193031
let expected = bitwise_xor(
3020-
bitwise_or(col("c2"), col("c1")),
3021-
bitwise_and(col("c1"), col("c2")),
3032+
bitwise_or(col("c2_non_null"), col("c1")),
3033+
bitwise_and(col("c1"), col("c2_non_null")),
30223034
);
30233035

30243036
assert_eq!(simplify(expr), expected);
@@ -3028,23 +3040,47 @@ mod tests {
30283040

30293041
let expr = bitwise_xor(
30303042
bitwise_xor(
3031-
bitwise_xor(col("c2"), bitwise_or(col("c2"), col("c1"))),
3032-
bitwise_xor(bitwise_and(col("c1"), col("c2")), col("c2")),
3043+
bitwise_xor(
3044+
col("c2_non_null"),
3045+
bitwise_or(col("c2_non_null"), col("c1")),
3046+
),
3047+
bitwise_xor(
3048+
bitwise_and(col("c1"), col("c2_non_null")),
3049+
col("c2_non_null"),
3050+
),
30333051
),
3034-
col("c2"),
3052+
col("c2_non_null"),
30353053
);
30363054

30373055
let expected = bitwise_xor(
30383056
bitwise_xor(
3039-
bitwise_or(col("c2"), col("c1")),
3040-
bitwise_and(col("c1"), col("c2")),
3057+
bitwise_or(col("c2_non_null"), col("c1")),
3058+
bitwise_and(col("c1"), col("c2_non_null")),
30413059
),
3042-
col("c2"),
3060+
col("c2_non_null"),
30433061
);
30443062

30453063
assert_eq!(simplify(expr), expected);
30463064
}
30473065

3066+
#[test]
3067+
fn test_does_not_cancel_nullable_bitwise_xor() {
3068+
let nullable = col("c3");
3069+
let other = col("c3_non_null");
3070+
3071+
let expr = bitwise_xor(nullable.clone(), nullable.clone());
3072+
assert_eq!(simplify(expr.clone()), expr);
3073+
3074+
let expr = bitwise_xor(
3075+
bitwise_xor(nullable.clone(), other.clone()),
3076+
nullable.clone(),
3077+
);
3078+
assert_eq!(simplify(expr.clone()), expr);
3079+
3080+
let expr = bitwise_xor(nullable.clone(), bitwise_xor(other, nullable));
3081+
assert_eq!(simplify(expr.clone()), expr);
3082+
}
3083+
30483084
#[test]
30493085
fn test_simplify_negated_bitwise_and() {
30503086
// !c4 & c4 --> 0
@@ -3169,13 +3205,13 @@ mod tests {
31693205
#[test]
31703206
fn test_simplify_simple_bitwise_xor() {
31713207
// c4 ^ c4 -> 0
3172-
let expr = (col("c4")).bitxor(col("c4"));
3208+
let expr = (col("c4_non_null")).bitxor(col("c4_non_null"));
31733209
let expected = lit(0u32);
31743210

31753211
assert_eq!(simplify(expr), expected);
31763212

31773213
// c3 ^ c3 -> 0
3178-
let expr = col("c3").bitxor(col("c3"));
3214+
let expr = col("c3_non_null").bitxor(col("c3_non_null"));
31793215
let expected = lit(0i64);
31803216

31813217
assert_eq!(simplify(expr), expected);

‎datafusion/sqllogictest/test_files/scalar.slt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1288,6 +1288,17 @@ select a ^ b, c ^ d, e ^ f from signed_integers;
12881288
-998 -133 -16
12891289
NULL NULL NULL
12901290

1291+
# repeated nullable operands must not be cancelled
1292+
query III rowsort
1293+
select
1294+
(a XOR b) XOR a AS left_associative,
1295+
a XOR (b XOR a) AS right_associative,
1296+
a XOR a AS self_xor
1297+
from (values (NULL::INT, 7), (3, 7)) as t(a, b);
1298+
----
1299+
7 7 0
1300+
NULL NULL NULL
1301+
12911302
# bitwise xor with other operators
12921303
query II rowsort
12931304
select 2 * c - 1 ^ 856 + d + 3, d ^ 7 >> 4 from signed_integers;

0 commit comments

Comments
 (0)