Skip to content

Commit 35fa8f2

Browse files
committed
mask logically NULL streamed keys after the nested apply_cmp
1 parent 49e4f62 commit 35fa8f2

1 file changed

Lines changed: 39 additions & 2 deletions

File tree

‎datafusion/physical-plan/src/joins/piecewise_merge_join/classic_join.rs‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -666,10 +666,16 @@ fn matchable_rows(
666666
)?
667667
.into_array(num_rows)?;
668668
let result = result.as_boolean();
669-
Ok(if result.null_count() > 0 {
669+
let matchable = if result.null_count() > 0 {
670670
prep_null_mask_filter(result)
671671
} else {
672672
result.clone()
673+
};
674+
// The nested `apply_cmp` masks only physical NULLs, so a dictionary key pointing at a
675+
// NULL nested value would pass.
676+
Ok(match stream_values.logical_nulls() {
677+
Some(nulls) => BooleanArray::new(matchable.values() & nulls.inner(), None),
678+
None => matchable,
673679
})
674680
}
675681

@@ -1336,7 +1342,7 @@ mod tests {
13361342
/// run-end encoded, `Null`-typed and dictionary keys, which have no physical NULL there.
13371343
#[test]
13381344
fn buffered_extreme_is_none_for_logically_null_keys() -> Result<()> {
1339-
use arrow::array::{DictionaryArray, Int32Array, NullArray, RunArray};
1345+
use arrow::array::{DictionaryArray, Int32Array, ListArray, NullArray, RunArray};
13401346
use arrow::datatypes::Int32Type;
13411347

13421348
let null_keys: Vec<(&str, ArrayRef)> = vec![
@@ -1355,6 +1361,16 @@ mod tests {
13551361
Arc::new(Int32Array::from(vec![None::<i32>])),
13561362
)?),
13571363
),
1364+
(
1365+
// `matchable_rows` relies on this: the nested `apply_cmp` would not mask it.
1366+
"dictionary_list_null_value",
1367+
Arc::new(DictionaryArray::<Int32Type>::try_new(
1368+
Int32Array::from(vec![0, 0]),
1369+
Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
1370+
None::<Vec<Option<i32>>>,
1371+
])),
1372+
)?),
1373+
),
13581374
];
13591375
// NULLs sort first for `>`/`>=` and last for `<`/`<=`.
13601376
let nulls_first = SortOptions::new(false, true);
@@ -1685,6 +1701,27 @@ mod tests {
16851701
Some(vec![true, true, true, true, true, true, false]),
16861702
),
16871703
),
1704+
(
1705+
// A valid key pointing at a NULL list: logically NULL, physically valid.
1706+
"dictionary_list_null_values",
1707+
Arc::new(DictionaryArray::<Int32Type>::new(
1708+
Int32Array::from(vec![0, 1, 2]),
1709+
Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
1710+
Some(vec![Some(5)]),
1711+
None,
1712+
Some(vec![Some(1)]),
1713+
])),
1714+
)),
1715+
Arc::new(DictionaryArray::<Int32Type>::new(
1716+
Int32Array::from(vec![Some(0), Some(1), Some(2), Some(3), None]),
1717+
Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
1718+
Some(vec![Some(2)]),
1719+
None,
1720+
Some(vec![Some(6)]),
1721+
Some(vec![Some(0)]),
1722+
])),
1723+
)),
1724+
),
16881725
];
16891726

16901727
for (name, buffered, streamed) in cases {

0 commit comments

Comments
 (0)