Skip to content

Commit 0a5bdd9

Browse files
committed
[core] Negate predicates like Java so NULL rows stay excluded
Predicate::negate wrapped leaves in Not, which was evaluated as !inner, so NOT(p = 'a') selected rows where p is NULL. Reads returned them and delete_by_predicate deleted them. Flip the leaf operator and push through AND/OR like Java Predicate.negate. Leaves without a negated operator stay as Not, which now evaluates to false for NULL input. Closes #1143
1 parent 9568f39 commit 0a5bdd9

8 files changed

Lines changed: 305 additions & 39 deletions

File tree

‎crates/integrations/datafusion/src/filter_pushdown.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1350,7 +1350,7 @@ mod tests {
13501350
.pushed_predicate
13511351
.expect("supported conjunct should still translate")
13521352
.to_string(),
1353-
"(dt = '2024-01-01' AND NOT (hr = 10))"
1353+
"(dt = '2024-01-01' AND hr != 10)"
13541354
);
13551355
assert!(analysis.requires_residual);
13561356
}
@@ -1368,7 +1368,7 @@ mod tests {
13681368
.pushed_predicate
13691369
.expect("NOT partition predicate should translate inexactly")
13701370
.to_string(),
1371-
"NOT (dt = '2024-01-01')"
1371+
"dt != '2024-01-01'"
13721372
);
13731373
assert!(analysis.requires_residual);
13741374
}
@@ -1530,7 +1530,7 @@ mod tests {
15301530
build_pushed_predicate(&[filter], &fields)
15311531
.expect("NOT should translate as an inexact pushed predicate")
15321532
.to_string(),
1533-
"NOT (dt = '2024-01-01')"
1533+
"dt != '2024-01-01'"
15341534
);
15351535
}
15361536

‎crates/integrations/datafusion/src/physical_plan/scan.rs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -452,11 +452,10 @@ fn predicate_covers_candidate(predicate: &Predicate, candidate: &Predicate) -> b
452452
op: PredicateOperator::NotBetween,
453453
literals,
454454
},
455-
Predicate::Not(inner),
455+
// `Predicate::negate` turns NOT(x >= lo AND x <= hi) into
456+
// x < lo OR x > hi.
457+
Predicate::Or(children),
456458
) if literals.len() == 2 => {
457-
let Predicate::And(children) = inner.as_ref() else {
458-
return false;
459-
};
460459
let matches_bound = |candidate: &Predicate,
461460
expected_op: PredicateOperator,
462461
expected_literal: &Datum| {
@@ -476,12 +475,13 @@ fn predicate_covers_candidate(predicate: &Predicate, candidate: &Predicate) -> b
476475
&& candidate_literals[0] == *expected_literal
477476
)
478477
};
479-
children
480-
.iter()
481-
.any(|child| matches_bound(child, PredicateOperator::GtEq, &literals[0]))
478+
children.len() == 2
479+
&& children
480+
.iter()
481+
.any(|child| matches_bound(child, PredicateOperator::Lt, &literals[0]))
482482
&& children
483483
.iter()
484-
.any(|child| matches_bound(child, PredicateOperator::LtEq, &literals[1]))
484+
.any(|child| matches_bound(child, PredicateOperator::Gt, &literals[1]))
485485
}
486486
_ => false,
487487
}

‎crates/paimon/src/arrow/residual.rs‎

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -157,9 +157,36 @@ fn evaluate_predicate_mask(
157157
else {
158158
return Ok(None);
159159
};
160-
Ok(Some(boolean_mask_from_predicate(mask.len(), |row_index| {
161-
!mask.value(row_index)
162-
})))
160+
let negated =
161+
boolean_mask_from_predicate(mask.len(), |row_index| !mask.value(row_index));
162+
// `Predicate::negate` keeps `Not` only around leaves without a
163+
// negated operator. Like Java `NotStartsWith`, `NotLike`, ..., the
164+
// negation is false for NULL input.
165+
if let Predicate::Leaf {
166+
column,
167+
index,
168+
op,
169+
data_type,
170+
..
171+
} = inner.as_ref()
172+
{
173+
if !matches!(op, PredicateOperator::IsNull | PredicateOperator::IsNotNull) {
174+
let not_null = Predicate::Leaf {
175+
column: column.clone(),
176+
index: *index,
177+
data_type: data_type.clone(),
178+
op: PredicateOperator::IsNotNull,
179+
literals: Vec::new(),
180+
};
181+
let Some(not_null) =
182+
evaluate_predicate_mask(batch, &not_null, file_fields, scan_fields)?
183+
else {
184+
return Ok(None);
185+
};
186+
return Ok(Some(combine_filter_masks(&negated, &not_null, false)));
187+
}
188+
}
189+
Ok(Some(negated))
163190
}
164191
Predicate::Leaf {
165192
column,
@@ -2087,6 +2114,22 @@ mod tests {
20872114
assert_eq!(int_values(&out), vec![10, 20]);
20882115
}
20892116

2117+
#[test]
2118+
fn test_not_excludes_null_row() {
2119+
// NOT(age > 25) on [10, null, 30] -> [10]; NOT(unknown) is unknown.
2120+
let f = int_field(0, "age");
2121+
let b = int_batch("age", vec![Some(10), None, Some(30)]);
2122+
let pred = Predicate::Not(Box::new(leaf(
2123+
0,
2124+
DataType::Int(IntType::new()),
2125+
PredicateOperator::Gt,
2126+
vec![Datum::Int(25)],
2127+
)));
2128+
let fp = file_predicates(vec![pred], vec![f.clone()]);
2129+
let out = filter_record_batch_by_predicates(b, &fp, &[f]).unwrap();
2130+
assert_eq!(int_values(&out), vec![10]);
2131+
}
2132+
20902133
#[test]
20912134
fn test_null_row_excluded() {
20922135
// age = [10, null, 30]; age > 5 -> [10, 30] (null → false).

‎crates/paimon/src/predicate_stats.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1052,13 +1052,14 @@ mod tests {
10521052
let dt = DataType::Int(IntType::new());
10531053
let fields = vec![DataField::new(0, "id".to_string(), dt.clone())];
10541054
let mapping = vec![Some(0)];
1055-
let predicate = Predicate::negate(Predicate::Leaf {
1055+
// Built directly: `Predicate::negate` would flip the leaf to `<=`.
1056+
let predicate = Predicate::Not(Box::new(Predicate::Leaf {
10561057
column: "id".to_string(),
10571058
index: 0,
10581059
data_type: dt,
10591060
op: PredicateOperator::Gt,
10601061
literals: vec![Datum::Int(5)],
1061-
});
1062+
}));
10621063

10631064
let with_nulls = MockStats {
10641065
row_count: 5,

‎crates/paimon/src/spec/predicate.rs‎

Lines changed: 122 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,34 @@ pub enum PredicateOperator {
284284
NotBetween,
285285
}
286286

287+
impl PredicateOperator {
288+
/// The operator selecting the complement of the non-NULL rows, matching
289+
/// Java `LeafFunction.negate`. `None` when there is no such operator.
290+
fn negate(self) -> Option<Self> {
291+
Some(match self {
292+
Self::IsNull => Self::IsNotNull,
293+
Self::IsNotNull => Self::IsNull,
294+
Self::Eq => Self::NotEq,
295+
Self::NotEq => Self::Eq,
296+
Self::Lt => Self::GtEq,
297+
Self::GtEq => Self::Lt,
298+
Self::LtEq => Self::Gt,
299+
Self::Gt => Self::LtEq,
300+
Self::In => Self::NotIn,
301+
Self::NotIn => Self::In,
302+
Self::Between => Self::NotBetween,
303+
Self::NotBetween => Self::Between,
304+
Self::StartsWith
305+
| Self::EndsWith
306+
| Self::Contains
307+
| Self::ArrayContains
308+
| Self::ArraysOverlap
309+
| Self::ArrayContainsAll
310+
| Self::Like => return None,
311+
})
312+
}
313+
}
314+
287315
impl fmt::Display for PredicateOperator {
288316
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
289317
match self {
@@ -424,37 +452,48 @@ impl Predicate {
424452

425453
/// Negate a predicate with simplification.
426454
///
455+
/// Like Java `Predicate.negate`, leaf operators are flipped (`=` / `!=`,
456+
/// `<` / `>=`, `IS NULL` / `IS NOT NULL`, `IN` / `NOT IN`, ...) and AND/OR
457+
/// are pushed down with De Morgan, so a NULL input still evaluates to false.
458+
/// A plain `!p` would select NULL rows for `NOT(p = 'a')`.
459+
///
427460
/// - `NOT(NOT(p))` → `p` (double negation elimination)
428461
/// - `NOT(AlwaysTrue)` → `AlwaysFalse`
429462
/// - `NOT(AlwaysFalse)` → `AlwaysTrue`
463+
/// - Leaves without a negated operator stay wrapped in `Not`.
430464
pub fn negate(predicate: Predicate) -> Predicate {
431465
match predicate {
432466
Predicate::Not(inner) => *inner,
433467
Predicate::AlwaysTrue => Predicate::AlwaysFalse,
434468
Predicate::AlwaysFalse => Predicate::AlwaysTrue,
469+
Predicate::And(children) => {
470+
Predicate::or(children.into_iter().map(Predicate::negate).collect())
471+
}
472+
Predicate::Or(children) => {
473+
Predicate::and(children.into_iter().map(Predicate::negate).collect())
474+
}
435475
Predicate::Leaf {
436476
column,
437477
index,
438478
data_type,
439479
op,
440480
literals,
441-
} if literals.is_empty()
442-
&& matches!(op, PredicateOperator::In | PredicateOperator::NotIn) =>
443-
{
444-
// Java negates the leaf function, retaining its NULL exclusion.
445-
Predicate::Leaf {
481+
} => match op.negate() {
482+
Some(op) => Predicate::Leaf {
446483
column,
447484
index,
448485
data_type,
449-
op: if op == PredicateOperator::In {
450-
PredicateOperator::NotIn
451-
} else {
452-
PredicateOperator::In
453-
},
486+
op,
454487
literals,
455-
}
456-
}
457-
other => Predicate::Not(Box::new(other)),
488+
},
489+
None => Predicate::Not(Box::new(Predicate::Leaf {
490+
column,
491+
index,
492+
data_type,
493+
op,
494+
literals,
495+
})),
496+
},
458497
}
459498
}
460499

@@ -2153,7 +2192,22 @@ pub(crate) fn eval_row(predicate: &Predicate, row: &BinaryRow) -> Result<bool> {
21532192
}
21542193
Ok(false)
21552194
}
2156-
Predicate::Not(inner) => Ok(!eval_row(inner, row)?),
2195+
Predicate::Not(inner) => match inner.as_ref() {
2196+
// `negate` keeps `Not` only around leaves without a negated
2197+
// operator. Those leaves are false for NULL input, and so is
2198+
// their negation (Java `NotStartsWith`, `NotLike`, ...).
2199+
Predicate::Leaf {
2200+
index,
2201+
data_type,
2202+
op,
2203+
literals,
2204+
..
2205+
} if !matches!(op, PredicateOperator::IsNull | PredicateOperator::IsNotNull) => {
2206+
let datum = extract_datum(row, *index, data_type)?;
2207+
Ok(datum.is_some() && !eval_leaf(*op, datum.as_ref(), literals))
2208+
}
2209+
_ => Ok(!eval_row(inner, row)?),
2210+
},
21572211
Predicate::Leaf {
21582212
index,
21592213
data_type,
@@ -2872,13 +2926,39 @@ mod tests {
28722926
#[test]
28732927
fn test_not() {
28742928
let pb = PredicateBuilder::new(&test_fields());
2875-
let p = pb.equal("id", Datum::Int(1)).unwrap();
2876-
let negated = Predicate::negate(p.clone());
2929+
let eq = pb.equal("id", Datum::Int(1)).unwrap();
2930+
let gt = pb.greater_than("hr", Datum::Int(2)).unwrap();
28772931

2878-
match &negated {
2879-
Predicate::Not(inner) => assert_eq!(inner.as_ref(), &p),
2880-
other => panic!("expected Not, got {other:?}"),
2881-
}
2932+
// Like Java, leaves flip their operator instead of becoming `Not`.
2933+
assert_eq!(
2934+
Predicate::negate(eq.clone()),
2935+
pb.not_equal("id", Datum::Int(1)).unwrap()
2936+
);
2937+
assert_eq!(
2938+
Predicate::negate(Predicate::and(vec![eq.clone(), gt.clone()])),
2939+
Predicate::or(vec![
2940+
pb.not_equal("id", Datum::Int(1)).unwrap(),
2941+
pb.less_or_equal("hr", Datum::Int(2)).unwrap(),
2942+
])
2943+
);
2944+
assert_eq!(
2945+
Predicate::negate(Predicate::or(vec![eq.clone(), gt.clone()])),
2946+
Predicate::and(vec![
2947+
pb.not_equal("id", Datum::Int(1)).unwrap(),
2948+
pb.less_or_equal("hr", Datum::Int(2)).unwrap(),
2949+
])
2950+
);
2951+
assert_eq!(
2952+
Predicate::negate(pb.is_null("id").unwrap()),
2953+
pb.is_not_null("id").unwrap()
2954+
);
2955+
2956+
// No negated operator: stays `Not`.
2957+
let starts = pb.starts_with("name", Datum::String("a".into())).unwrap();
2958+
assert_eq!(
2959+
Predicate::negate(starts.clone()),
2960+
Predicate::Not(Box::new(starts))
2961+
);
28822962
}
28832963

28842964
// ======================== field_idx_to_partition_idx ========================
@@ -2957,7 +3037,7 @@ mod tests {
29573037
fn test_display_not() {
29583038
let pb = PredicateBuilder::new(&test_fields());
29593039
let p = pb.equal("id", Datum::Int(1)).unwrap();
2960-
let negated = Predicate::negate(p);
3040+
let negated = Predicate::Not(Box::new(p));
29613041
assert_eq!(format!("{negated}"), "NOT (id = 1)");
29623042
}
29633043

@@ -3377,6 +3457,27 @@ mod tests {
33773457
&row
33783458
)
33793459
.unwrap());
3460+
// ... and so is its negation.
3461+
assert!(!eval_row(
3462+
&Predicate::negate(make_leaf(
3463+
"x",
3464+
0,
3465+
PredicateOperator::Eq,
3466+
vec![Datum::Int(42)]
3467+
)),
3468+
&row
3469+
)
3470+
.unwrap());
3471+
assert!(!eval_row(
3472+
&Predicate::Not(Box::new(make_leaf(
3473+
"x",
3474+
0,
3475+
PredicateOperator::Gt,
3476+
vec![Datum::Int(42)]
3477+
))),
3478+
&row
3479+
)
3480+
.unwrap());
33803481
// IsNull / IsNotNull
33813482
assert!(eval_row(&make_leaf("x", 0, PredicateOperator::IsNull, vec![]), &row).unwrap());
33823483
assert!(!eval_row(

‎crates/paimon/src/table/data_file_reader.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3559,7 +3559,9 @@ mod tests {
35593559
.try_collect::<Vec<_>>()
35603560
.await
35613561
.unwrap();
3562-
assert_eq!(collect_ids(&batches), vec![1, 2], "enabled={enabled}");
3562+
// `added` is NULL in the old file, so for id 2 the predicate is
3563+
// NOT(true AND unknown), which is unknown. Java filters it out too.
3564+
assert_eq!(collect_ids(&batches), vec![1], "enabled={enabled}");
35633565
}
35643566
}
35653567

‎crates/paimon/src/table/global_index_scanner/tests.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3949,7 +3949,7 @@ async fn scalar_exactness_keeps_string_candidates_and_dropped_conjuncts_inexact(
39493949
builder
39503950
.equal("other", Datum::String("needle".to_string()))
39513951
.unwrap(),
3952-
Predicate::negate(exact.clone()),
3952+
Predicate::Not(Box::new(exact.clone())),
39533953
] {
39543954
let filter = Predicate::and(vec![exact.clone(), candidate]);
39553955
let (ranges, is_exact) = scanner

0 commit comments

Comments
 (0)