Skip to content

Commit 0417a8b

Browse files
committed
test(smj): Cover null-aware executor boundaries
Add five executor tests for null-aware LeftAnti sort-merge joins. Cover empty inputs, empty batches before RHS NULLs, startup prefetch across key groups, draining remaining left batches, and dictionary value NULLs. Use row IDs to verify that NULL and duplicate rows are retained or removed according to NOT IN semantics. Exercise batch sizes 1 and 2 at key-group boundaries and check dictionaries with valid indices pointing to NULL values and different index mappings on each side. Reuse the existing join and input helpers, with small fixture helpers modeled on the null-aware HashJoin tests. Execute plans directly so the optimizer cannot simplify away empty-input cases. Refs apache#25546
1 parent 56c1733 commit 0417a8b

1 file changed

Lines changed: 182 additions & 0 deletions

File tree

  • datafusion/physical-plan/src/joins/sort_merge_join

‎datafusion/physical-plan/src/joins/sort_merge_join/tests.rs‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1774,6 +1774,188 @@ async fn join_left_anti() -> Result<()> {
17741774
Ok(())
17751775
}
17761776

1777+
mod null_aware_left_anti_tests {
1778+
use super::*;
1779+
use arrow::array::{Array, ArrayRef, DictionaryArray};
1780+
use arrow::datatypes::Int32Type;
1781+
1782+
fn batch(keys: ArrayRef, row_ids: &[i32]) -> Result<RecordBatch> {
1783+
let schema = Arc::new(Schema::new(vec![
1784+
Field::new("key", keys.data_type().clone(), true),
1785+
Field::new("row_id", DataType::Int32, false),
1786+
]));
1787+
Ok(RecordBatch::try_new(
1788+
schema,
1789+
vec![keys, Arc::new(Int32Array::from(row_ids.to_vec()))],
1790+
)?)
1791+
}
1792+
1793+
fn int_batch(keys: Vec<Option<i32>>, row_ids: &[i32]) -> Result<RecordBatch> {
1794+
batch(Arc::new(Int32Array::from(keys)), row_ids)
1795+
}
1796+
1797+
async fn collect_ids(
1798+
left_batches: Vec<RecordBatch>,
1799+
right_batches: Vec<RecordBatch>,
1800+
batch_size: usize,
1801+
) -> Result<Vec<i32>> {
1802+
let left = build_table_from_batches(left_batches);
1803+
// Unlike build_table_from_batches, this must also accept zero batches.
1804+
let right = TestMemoryExec::try_new_exec(&[right_batches], left.schema(), None)?;
1805+
let on: JoinOn = vec![(
1806+
Arc::new(Column::new("key", 0)),
1807+
Arc::new(Column::new("key", 0)),
1808+
)];
1809+
// Direct execution does not insert sorts. Fixtures are globally sorted
1810+
// ASC NULLS FIRST, including across batch boundaries.
1811+
let join = join_with_options(
1812+
left,
1813+
right,
1814+
on,
1815+
LeftAnti,
1816+
vec![SortOptions::new(false, true)],
1817+
NullEquality::NullEqualsNothing,
1818+
)?
1819+
.with_null_aware(true)?;
1820+
let context = Arc::new(
1821+
TaskContext::default()
1822+
.with_session_config(SessionConfig::new().with_batch_size(batch_size)),
1823+
);
1824+
let batches = common::collect(join.execute(0, context)?).await?;
1825+
let mut row_ids = Vec::new();
1826+
for batch in batches {
1827+
let ids = batch
1828+
.column(1)
1829+
.as_any()
1830+
.downcast_ref::<Int32Array>()
1831+
.expect("Int32 row IDs");
1832+
row_ids.extend(ids.values().iter().copied());
1833+
}
1834+
Ok(row_ids)
1835+
}
1836+
1837+
#[tokio::test]
1838+
async fn empty_rhs_keeps_nulls_and_duplicates() -> Result<()> {
1839+
let left = vec![
1840+
int_batch(vec![None, Some(1), Some(2)], &[0, 1, 2])?,
1841+
int_batch(vec![Some(3), Some(3)], &[3, 4])?,
1842+
];
1843+
// EOF without any batch and EOF after empty batches are both empty RHS.
1844+
for right in [
1845+
vec![],
1846+
vec![int_batch(vec![], &[])?, int_batch(vec![], &[])?],
1847+
] {
1848+
let actual = collect_ids(left.clone(), right, 2).await?;
1849+
assert_eq!(actual, vec![0, 1, 2, 3, 4]);
1850+
}
1851+
Ok(())
1852+
}
1853+
1854+
#[tokio::test]
1855+
async fn rhs_null_after_empty_batch_returns_no_rows() -> Result<()> {
1856+
let left = vec![
1857+
int_batch(vec![None, Some(1)], &[0, 1])?,
1858+
int_batch(vec![Some(2), Some(3)], &[2, 3])?,
1859+
];
1860+
// Cover an all-NULL RHS and a NULL followed by a non-NULL batch.
1861+
for right in [
1862+
vec![int_batch(vec![], &[])?, int_batch(vec![None], &[10])?],
1863+
vec![
1864+
int_batch(vec![], &[])?,
1865+
int_batch(vec![None], &[10])?,
1866+
int_batch(vec![Some(2)], &[11])?,
1867+
],
1868+
] {
1869+
let actual = collect_ids(left.clone(), right, 2).await?;
1870+
assert!(actual.is_empty(), "unexpected rows: {actual:?}");
1871+
}
1872+
Ok(())
1873+
}
1874+
1875+
#[tokio::test]
1876+
async fn startup_preserves_first_rhs_batch_across_key_groups() -> Result<()> {
1877+
let left = vec![
1878+
int_batch(vec![None, Some(1), Some(2)], &[0, 1, 2])?,
1879+
int_batch(vec![], &[])?,
1880+
int_batch(vec![Some(2), Some(3), Some(3)], &[3, 4, 5])?,
1881+
int_batch(vec![Some(4), Some(5)], &[6, 7])?,
1882+
];
1883+
let right = vec![
1884+
int_batch(vec![], &[])?,
1885+
int_batch(vec![Some(2)], &[10])?,
1886+
int_batch(vec![], &[])?,
1887+
int_batch(vec![Some(4)], &[11])?,
1888+
int_batch(vec![Some(4)], &[12])?,
1889+
];
1890+
for batch_size in [1, 2] {
1891+
let actual = collect_ids(left.clone(), right.clone(), batch_size).await?;
1892+
// Keep 1, 3, 3, 5; both occurrences of 2 must match the first RHS batch.
1893+
assert_eq!(actual, vec![1, 4, 5, 7], "batch size {batch_size}");
1894+
}
1895+
Ok(())
1896+
}
1897+
1898+
#[tokio::test]
1899+
async fn exhausted_rhs_drains_remaining_left_batches() -> Result<()> {
1900+
let left = vec![
1901+
int_batch(vec![None, Some(1)], &[0, 1])?,
1902+
int_batch(vec![Some(2)], &[2])?,
1903+
int_batch(vec![Some(3), Some(3)], &[3, 4])?,
1904+
int_batch(vec![Some(4), Some(5)], &[5, 6])?,
1905+
];
1906+
let right = vec![int_batch(vec![Some(2)], &[10])?];
1907+
let actual = collect_ids(left, right, 2).await?;
1908+
// Exclude the initial NULL and retain unmatched rows after RHS EOF,
1909+
// including duplicates and the final partial output batch.
1910+
assert_eq!(actual, vec![1, 3, 4, 5, 6]);
1911+
Ok(())
1912+
}
1913+
1914+
// Like HashJoin's build_table_dict_key, keep all dictionary indices valid
1915+
// so NULL dictionary values exercise logical rather than physical nulls.
1916+
fn dictionary_batch(
1917+
keys: Vec<i32>,
1918+
values: Vec<Option<i32>>,
1919+
row_ids: &[i32],
1920+
) -> Result<RecordBatch> {
1921+
let keys = DictionaryArray::<Int32Type>::try_new(
1922+
Int32Array::from(keys),
1923+
Arc::new(Int32Array::from(values)),
1924+
)?;
1925+
batch(Arc::new(keys), row_ids)
1926+
}
1927+
1928+
#[tokio::test]
1929+
async fn dictionary_value_nulls_follow_not_in_semantics() -> Result<()> {
1930+
let left = dictionary_batch(
1931+
vec![0, 1, 2, 3, 3],
1932+
vec![None, Some(1), Some(2), Some(3)],
1933+
&[0, 1, 2, 3, 4],
1934+
)?;
1935+
assert_eq!(left.column(0).null_count(), 0);
1936+
assert_eq!(left.column(0).logical_null_count(), 1);
1937+
1938+
// The same logical key (2) has a different dictionary index on each side.
1939+
let rhs_with_null = dictionary_batch(vec![0, 1], vec![None, Some(2)], &[10, 11])?;
1940+
assert_eq!(rhs_with_null.column(0).null_count(), 0);
1941+
assert_eq!(rhs_with_null.column(0).logical_null_count(), 1);
1942+
1943+
let cases = [
1944+
(vec![], vec![0, 1, 2, 3, 4]),
1945+
(
1946+
vec![dictionary_batch(vec![0], vec![Some(2)], &[10])?],
1947+
vec![1, 3, 4],
1948+
),
1949+
(vec![rhs_with_null], vec![]),
1950+
];
1951+
for (right, expected) in cases {
1952+
let actual = collect_ids(vec![left.clone()], right, 2).await?;
1953+
assert_eq!(actual, expected);
1954+
}
1955+
Ok(())
1956+
}
1957+
}
1958+
17771959
#[tokio::test]
17781960
async fn join_right_anti_one_one() -> Result<()> {
17791961
let left = build_table(

0 commit comments

Comments
 (0)