@@ -1774,6 +1774,188 @@ async fn join_left_anti() -> Result<()> {
17741774 Ok ( ( ) )
17751775}
17761776
1777+ mod null_aware_left_anti_tests {
1778+ use super :: * ;
1779+ use arrow:: array:: { Array , ArrayRef , DictionaryArray } ;
1780+ use arrow:: datatypes:: Int32Type ;
1781+
1782+ fn batch ( keys : ArrayRef , row_ids : & [ i32 ] ) -> Result < RecordBatch > {
1783+ let schema = Arc :: new ( Schema :: new ( vec ! [
1784+ Field :: new( "key" , keys. data_type( ) . clone( ) , true ) ,
1785+ Field :: new( "row_id" , DataType :: Int32 , false ) ,
1786+ ] ) ) ;
1787+ Ok ( RecordBatch :: try_new (
1788+ schema,
1789+ vec ! [ keys, Arc :: new( Int32Array :: from( row_ids. to_vec( ) ) ) ] ,
1790+ ) ?)
1791+ }
1792+
1793+ fn int_batch ( keys : Vec < Option < i32 > > , row_ids : & [ i32 ] ) -> Result < RecordBatch > {
1794+ batch ( Arc :: new ( Int32Array :: from ( keys) ) , row_ids)
1795+ }
1796+
1797+ async fn collect_ids (
1798+ left_batches : Vec < RecordBatch > ,
1799+ right_batches : Vec < RecordBatch > ,
1800+ batch_size : usize ,
1801+ ) -> Result < Vec < i32 > > {
1802+ let left = build_table_from_batches ( left_batches) ;
1803+ // Unlike build_table_from_batches, this must also accept zero batches.
1804+ let right = TestMemoryExec :: try_new_exec ( & [ right_batches] , left. schema ( ) , None ) ?;
1805+ let on: JoinOn = vec ! [ (
1806+ Arc :: new( Column :: new( "key" , 0 ) ) ,
1807+ Arc :: new( Column :: new( "key" , 0 ) ) ,
1808+ ) ] ;
1809+ // Direct execution does not insert sorts. Fixtures are globally sorted
1810+ // ASC NULLS FIRST, including across batch boundaries.
1811+ let join = join_with_options (
1812+ left,
1813+ right,
1814+ on,
1815+ LeftAnti ,
1816+ vec ! [ SortOptions :: new( false , true ) ] ,
1817+ NullEquality :: NullEqualsNothing ,
1818+ ) ?
1819+ . with_null_aware ( true ) ?;
1820+ let context = Arc :: new (
1821+ TaskContext :: default ( )
1822+ . with_session_config ( SessionConfig :: new ( ) . with_batch_size ( batch_size) ) ,
1823+ ) ;
1824+ let batches = common:: collect ( join. execute ( 0 , context) ?) . await ?;
1825+ let mut row_ids = Vec :: new ( ) ;
1826+ for batch in batches {
1827+ let ids = batch
1828+ . column ( 1 )
1829+ . as_any ( )
1830+ . downcast_ref :: < Int32Array > ( )
1831+ . expect ( "Int32 row IDs" ) ;
1832+ row_ids. extend ( ids. values ( ) . iter ( ) . copied ( ) ) ;
1833+ }
1834+ Ok ( row_ids)
1835+ }
1836+
1837+ #[ tokio:: test]
1838+ async fn empty_rhs_keeps_nulls_and_duplicates ( ) -> Result < ( ) > {
1839+ let left = vec ! [
1840+ int_batch( vec![ None , Some ( 1 ) , Some ( 2 ) ] , & [ 0 , 1 , 2 ] ) ?,
1841+ int_batch( vec![ Some ( 3 ) , Some ( 3 ) ] , & [ 3 , 4 ] ) ?,
1842+ ] ;
1843+ // EOF without any batch and EOF after empty batches are both empty RHS.
1844+ for right in [
1845+ vec ! [ ] ,
1846+ vec ! [ int_batch( vec![ ] , & [ ] ) ?, int_batch( vec![ ] , & [ ] ) ?] ,
1847+ ] {
1848+ let actual = collect_ids ( left. clone ( ) , right, 2 ) . await ?;
1849+ assert_eq ! ( actual, vec![ 0 , 1 , 2 , 3 , 4 ] ) ;
1850+ }
1851+ Ok ( ( ) )
1852+ }
1853+
1854+ #[ tokio:: test]
1855+ async fn rhs_null_after_empty_batch_returns_no_rows ( ) -> Result < ( ) > {
1856+ let left = vec ! [
1857+ int_batch( vec![ None , Some ( 1 ) ] , & [ 0 , 1 ] ) ?,
1858+ int_batch( vec![ Some ( 2 ) , Some ( 3 ) ] , & [ 2 , 3 ] ) ?,
1859+ ] ;
1860+ // Cover an all-NULL RHS and a NULL followed by a non-NULL batch.
1861+ for right in [
1862+ vec ! [ int_batch( vec![ ] , & [ ] ) ?, int_batch( vec![ None ] , & [ 10 ] ) ?] ,
1863+ vec ! [
1864+ int_batch( vec![ ] , & [ ] ) ?,
1865+ int_batch( vec![ None ] , & [ 10 ] ) ?,
1866+ int_batch( vec![ Some ( 2 ) ] , & [ 11 ] ) ?,
1867+ ] ,
1868+ ] {
1869+ let actual = collect_ids ( left. clone ( ) , right, 2 ) . await ?;
1870+ assert ! ( actual. is_empty( ) , "unexpected rows: {actual:?}" ) ;
1871+ }
1872+ Ok ( ( ) )
1873+ }
1874+
1875+ #[ tokio:: test]
1876+ async fn startup_preserves_first_rhs_batch_across_key_groups ( ) -> Result < ( ) > {
1877+ let left = vec ! [
1878+ int_batch( vec![ None , Some ( 1 ) , Some ( 2 ) ] , & [ 0 , 1 , 2 ] ) ?,
1879+ int_batch( vec![ ] , & [ ] ) ?,
1880+ int_batch( vec![ Some ( 2 ) , Some ( 3 ) , Some ( 3 ) ] , & [ 3 , 4 , 5 ] ) ?,
1881+ int_batch( vec![ Some ( 4 ) , Some ( 5 ) ] , & [ 6 , 7 ] ) ?,
1882+ ] ;
1883+ let right = vec ! [
1884+ int_batch( vec![ ] , & [ ] ) ?,
1885+ int_batch( vec![ Some ( 2 ) ] , & [ 10 ] ) ?,
1886+ int_batch( vec![ ] , & [ ] ) ?,
1887+ int_batch( vec![ Some ( 4 ) ] , & [ 11 ] ) ?,
1888+ int_batch( vec![ Some ( 4 ) ] , & [ 12 ] ) ?,
1889+ ] ;
1890+ for batch_size in [ 1 , 2 ] {
1891+ let actual = collect_ids ( left. clone ( ) , right. clone ( ) , batch_size) . await ?;
1892+ // Keep 1, 3, 3, 5; both occurrences of 2 must match the first RHS batch.
1893+ assert_eq ! ( actual, vec![ 1 , 4 , 5 , 7 ] , "batch size {batch_size}" ) ;
1894+ }
1895+ Ok ( ( ) )
1896+ }
1897+
1898+ #[ tokio:: test]
1899+ async fn exhausted_rhs_drains_remaining_left_batches ( ) -> Result < ( ) > {
1900+ let left = vec ! [
1901+ int_batch( vec![ None , Some ( 1 ) ] , & [ 0 , 1 ] ) ?,
1902+ int_batch( vec![ Some ( 2 ) ] , & [ 2 ] ) ?,
1903+ int_batch( vec![ Some ( 3 ) , Some ( 3 ) ] , & [ 3 , 4 ] ) ?,
1904+ int_batch( vec![ Some ( 4 ) , Some ( 5 ) ] , & [ 5 , 6 ] ) ?,
1905+ ] ;
1906+ let right = vec ! [ int_batch( vec![ Some ( 2 ) ] , & [ 10 ] ) ?] ;
1907+ let actual = collect_ids ( left, right, 2 ) . await ?;
1908+ // Exclude the initial NULL and retain unmatched rows after RHS EOF,
1909+ // including duplicates and the final partial output batch.
1910+ assert_eq ! ( actual, vec![ 1 , 3 , 4 , 5 , 6 ] ) ;
1911+ Ok ( ( ) )
1912+ }
1913+
1914+ // Like HashJoin's build_table_dict_key, keep all dictionary indices valid
1915+ // so NULL dictionary values exercise logical rather than physical nulls.
1916+ fn dictionary_batch (
1917+ keys : Vec < i32 > ,
1918+ values : Vec < Option < i32 > > ,
1919+ row_ids : & [ i32 ] ,
1920+ ) -> Result < RecordBatch > {
1921+ let keys = DictionaryArray :: < Int32Type > :: try_new (
1922+ Int32Array :: from ( keys) ,
1923+ Arc :: new ( Int32Array :: from ( values) ) ,
1924+ ) ?;
1925+ batch ( Arc :: new ( keys) , row_ids)
1926+ }
1927+
1928+ #[ tokio:: test]
1929+ async fn dictionary_value_nulls_follow_not_in_semantics ( ) -> Result < ( ) > {
1930+ let left = dictionary_batch (
1931+ vec ! [ 0 , 1 , 2 , 3 , 3 ] ,
1932+ vec ! [ None , Some ( 1 ) , Some ( 2 ) , Some ( 3 ) ] ,
1933+ & [ 0 , 1 , 2 , 3 , 4 ] ,
1934+ ) ?;
1935+ assert_eq ! ( left. column( 0 ) . null_count( ) , 0 ) ;
1936+ assert_eq ! ( left. column( 0 ) . logical_null_count( ) , 1 ) ;
1937+
1938+ // The same logical key (2) has a different dictionary index on each side.
1939+ let rhs_with_null = dictionary_batch ( vec ! [ 0 , 1 ] , vec ! [ None , Some ( 2 ) ] , & [ 10 , 11 ] ) ?;
1940+ assert_eq ! ( rhs_with_null. column( 0 ) . null_count( ) , 0 ) ;
1941+ assert_eq ! ( rhs_with_null. column( 0 ) . logical_null_count( ) , 1 ) ;
1942+
1943+ let cases = [
1944+ ( vec ! [ ] , vec ! [ 0 , 1 , 2 , 3 , 4 ] ) ,
1945+ (
1946+ vec ! [ dictionary_batch( vec![ 0 ] , vec![ Some ( 2 ) ] , & [ 10 ] ) ?] ,
1947+ vec ! [ 1 , 3 , 4 ] ,
1948+ ) ,
1949+ ( vec ! [ rhs_with_null] , vec ! [ ] ) ,
1950+ ] ;
1951+ for ( right, expected) in cases {
1952+ let actual = collect_ids ( vec ! [ left. clone( ) ] , right, 2 ) . await ?;
1953+ assert_eq ! ( actual, expected) ;
1954+ }
1955+ Ok ( ( ) )
1956+ }
1957+ }
1958+
17771959#[ tokio:: test]
17781960async fn join_right_anti_one_one ( ) -> Result < ( ) > {
17791961 let left = build_table (
0 commit comments