Skip to content

Commit 46bbf0d

Browse files
authored
fix: preserve null semantics in expression simplification (apache#24258)
## Which issue does this PR close? - Closes apache#24246. ## Rationale for this change Set algebra over `IN` / `NOT IN` expressions can erase SQL three-valued-logic semantics when either the tested expression or a list item is nullable. The `array_has` to `IN` rewrite has a similar mismatch: an array containing NULL does not behave like an `IN` list containing NULL. Preserving those NULL branches is required in general expression contexts. In a filter, however, both FALSE and NULL reject the row, so the optimizer can still produce the strongest equivalent plan there. ## What changes are included in this PR? - Keep `IN` / `NOT IN` set operations unchanged when list items may be NULL, and preserve the NULL branch when an otherwise safe set operation becomes a constant for a nullable tested expression. - Avoid rewriting `array_has` to `IN` when a nullable array element would change the result, while retaining the safe non-volatile same-expression case. - Extend `EliminateFilter` to simplify positive `AND` / `OR` trees according to filter semantics. This restores plans such as `EmptyRelation` for disjoint `IN` predicates while retaining correct NULL behavior in projections. The simplification deliberately does not cross `NOT` or arbitrary expressions. - Add unit, runtime, and SQL logic regression coverage for NULL results and the resulting optimized plans. ## Are these changes tested? Yes. The following checks pass: - `cargo clippy --all-targets --all-features -- -D warnings` - `./dev/rust_lint.sh` - `cargo test --profile=ci --test sqllogictests` - `prlimit --nofile=65536:65536 -- env RUST_BACKTRACE=1 cargo test --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-cli --workspace --lib --tests --bins --features avro,json,backtrace,extended_tests,recursive_protection,parquet_encryption` ## Are there any user-facing changes? Yes. Expressions involving nullable inputs retain their SQL three-valued-logic results, while equivalent filter predicates continue to optimize away rejected rows. There are no public API changes.
1 parent cc29ea1 commit 46bbf0d

8 files changed

Lines changed: 625 additions & 80 deletions

File tree

‎datafusion/functions-nested/src/array_has.rs‎

Lines changed: 106 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ impl ScalarUDFImpl for ArrayHas {
128128
fn simplify(
129129
&self,
130130
mut args: Vec<Expr>,
131-
_info: &datafusion_expr::simplify::SimplifyContext,
131+
info: &datafusion_expr::simplify::SimplifyContext,
132132
) -> Result<ExprSimplifyResult> {
133133
let [haystack, needle] = take_function_args(self.name(), &mut args)?;
134134

@@ -150,11 +150,19 @@ impl ScalarUDFImpl for ArrayHas {
150150
ScalarValue::convert_array_to_scalar_vec(&scalar.to_array()?)
151151
{
152152
assert_eq!(scalar_values.len(), 1);
153-
let list = scalar_values
153+
let values = scalar_values
154154
.into_iter()
155155
.flatten()
156156
.flatten()
157-
.map(|v| Expr::Literal(v, None))
157+
.collect::<Vec<_>>();
158+
159+
if values.iter().any(ScalarValue::is_null) {
160+
return Ok(ExprSimplifyResult::Original(args));
161+
}
162+
163+
let list = values
164+
.into_iter()
165+
.map(|value| Expr::Literal(value, None))
158166
.collect();
159167

160168
return Ok(ExprSimplifyResult::Simplified(in_list(
@@ -167,12 +175,27 @@ impl ScalarUDFImpl for ArrayHas {
167175
Expr::ScalarFunction(ScalarFunction { func, args })
168176
if func == &make_array_udf() =>
169177
{
170-
// make_array has a static set of arguments, so we can pull the arguments out from it
171-
return Ok(ExprSimplifyResult::Simplified(in_list(
172-
std::mem::take(needle),
173-
std::mem::take(args),
174-
false,
175-
)));
178+
let mut has_unsafe_nullable_element = false;
179+
for arg in args.iter() {
180+
// `needle IN (needle)` preserves NULL semantics. A different
181+
// nullable element does not: array_has([NULL], value) is false,
182+
// while value IN (NULL) is NULL. Volatile expressions are
183+
// evaluated separately, so syntactic equality is insufficient.
184+
let safe_same_expr = arg == &*needle && !arg.is_volatile();
185+
if info.nullable(arg)? && !safe_same_expr {
186+
has_unsafe_nullable_element = true;
187+
break;
188+
}
189+
}
190+
191+
if !has_unsafe_nullable_element {
192+
// make_array has a static set of arguments, so we can pull the arguments out from it
193+
return Ok(ExprSimplifyResult::Simplified(in_list(
194+
std::mem::take(needle),
195+
std::mem::take(args),
196+
false,
197+
)));
198+
}
176199
}
177200
_ => {}
178201
}
@@ -1197,10 +1220,10 @@ mod tests {
11971220
create_array,
11981221
},
11991222
buffer::OffsetBuffer,
1200-
datatypes::{DataType, Field},
1223+
datatypes::{DataType, Field, Schema},
12011224
};
12021225
use datafusion_common::{
1203-
DataFusionError, ScalarValue, config::ConfigOptions,
1226+
DataFusionError, ScalarValue, ToDFSchema, config::ConfigOptions,
12041227
utils::SingleRowListArrayBuilder,
12051228
};
12061229
use datafusion_expr::simplify::SimplifyContext;
@@ -1263,6 +1286,78 @@ mod tests {
12631286
);
12641287
}
12651288

1289+
#[test]
1290+
fn test_simplify_array_has_with_nullable_make_array_is_unchanged() {
1291+
let haystack = make_array(vec![col("c"), lit(1)]);
1292+
let needle = lit(2);
1293+
let original = vec![haystack, needle];
1294+
let context = SimplifyContext::builder()
1295+
.with_schema(
1296+
Schema::new(vec![Field::new("c", DataType::Int32, true)])
1297+
.to_dfschema_ref()
1298+
.unwrap(),
1299+
)
1300+
.build();
1301+
1302+
let result = ArrayHas::new()
1303+
.simplify(original.clone(), &context)
1304+
.unwrap();
1305+
1306+
let ExprSimplifyResult::Original(args) = result else {
1307+
panic!("Expected ExprSimplifyResult::Original")
1308+
};
1309+
assert_eq!(args, original);
1310+
}
1311+
1312+
#[test]
1313+
fn test_simplify_array_has_with_same_nullable_element() {
1314+
let needle = col("c");
1315+
let haystack = make_array(vec![needle.clone()]);
1316+
let context = SimplifyContext::builder()
1317+
.with_schema(
1318+
Schema::new(vec![Field::new("c", DataType::Int32, true)])
1319+
.to_dfschema_ref()
1320+
.unwrap(),
1321+
)
1322+
.build();
1323+
1324+
let result = ArrayHas::new()
1325+
.simplify(vec![haystack, needle.clone()], &context)
1326+
.unwrap();
1327+
1328+
let ExprSimplifyResult::Simplified(Expr::InList(in_list)) = result else {
1329+
panic!("Expected simplified expression")
1330+
};
1331+
assert_eq!(
1332+
in_list,
1333+
datafusion_expr::expr::InList {
1334+
expr: Box::new(needle.clone()),
1335+
list: vec![needle],
1336+
negated: false,
1337+
}
1338+
);
1339+
}
1340+
1341+
#[test]
1342+
fn test_simplify_array_has_with_null_list_item_is_unchanged() {
1343+
let haystack = lit(SingleRowListArrayBuilder::new(create_array!(
1344+
Int32,
1345+
[Some(1), None]
1346+
))
1347+
.build_list_scalar());
1348+
let needle = lit(2);
1349+
let original = vec![haystack, needle];
1350+
1351+
let result = ArrayHas::new()
1352+
.simplify(original.clone(), &SimplifyContext::default())
1353+
.unwrap();
1354+
1355+
let ExprSimplifyResult::Original(args) = result else {
1356+
panic!("Expected ExprSimplifyResult::Original")
1357+
};
1358+
assert_eq!(args, original);
1359+
}
1360+
12661361
#[test]
12671362
fn test_simplify_array_has_with_null_to_null() {
12681363
let haystack = Expr::Literal(ScalarValue::Null, None);

0 commit comments

Comments
 (0)