@@ -1761,6 +1761,188 @@ async fn join_left_anti() -> Result<()> {
17611761 Ok ( ( ) )
17621762}
17631763
1764+ mod null_aware_left_anti_tests {
1765+ use super :: * ;
1766+ use arrow:: array:: { Array , ArrayRef , DictionaryArray } ;
1767+ use arrow:: datatypes:: Int32Type ;
1768+
1769+ fn batch ( keys : ArrayRef , row_ids : & [ i32 ] ) -> Result < RecordBatch > {
1770+ let schema = Arc :: new ( Schema :: new ( vec ! [
1771+ Field :: new( "key" , keys. data_type( ) . clone( ) , true ) ,
1772+ Field :: new( "row_id" , DataType :: Int32 , false ) ,
1773+ ] ) ) ;
1774+ Ok ( RecordBatch :: try_new (
1775+ schema,
1776+ vec ! [ keys, Arc :: new( Int32Array :: from( row_ids. to_vec( ) ) ) ] ,
1777+ ) ?)
1778+ }
1779+
1780+ fn int_batch ( keys : Vec < Option < i32 > > , row_ids : & [ i32 ] ) -> Result < RecordBatch > {
1781+ batch ( Arc :: new ( Int32Array :: from ( keys) ) , row_ids)
1782+ }
1783+
1784+ async fn collect_ids (
1785+ left_batches : Vec < RecordBatch > ,
1786+ right_batches : Vec < RecordBatch > ,
1787+ batch_size : usize ,
1788+ ) -> Result < Vec < i32 > > {
1789+ let left = build_table_from_batches ( left_batches) ;
1790+ // Unlike build_table_from_batches, this must also accept zero batches.
1791+ let right = TestMemoryExec :: try_new_exec ( & [ right_batches] , left. schema ( ) , None ) ?;
1792+ let on: JoinOn = vec ! [ (
1793+ Arc :: new( Column :: new( "key" , 0 ) ) ,
1794+ Arc :: new( Column :: new( "key" , 0 ) ) ,
1795+ ) ] ;
1796+ // Direct execution does not insert sorts. Fixtures are globally sorted
1797+ // ASC NULLS FIRST, including across batch boundaries.
1798+ let join = join_with_options (
1799+ left,
1800+ right,
1801+ on,
1802+ LeftAnti ,
1803+ vec ! [ SortOptions :: new( false , true ) ] ,
1804+ NullEquality :: NullEqualsNothing ,
1805+ ) ?
1806+ . with_null_aware ( true ) ?;
1807+ let context = Arc :: new (
1808+ TaskContext :: default ( )
1809+ . with_session_config ( SessionConfig :: new ( ) . with_batch_size ( batch_size) ) ,
1810+ ) ;
1811+ let batches = common:: collect ( join. execute ( 0 , context) ?) . await ?;
1812+ let mut row_ids = Vec :: new ( ) ;
1813+ for batch in batches {
1814+ let ids = batch
1815+ . column ( 1 )
1816+ . as_any ( )
1817+ . downcast_ref :: < Int32Array > ( )
1818+ . expect ( "Int32 row IDs" ) ;
1819+ row_ids. extend ( ids. values ( ) . iter ( ) . copied ( ) ) ;
1820+ }
1821+ Ok ( row_ids)
1822+ }
1823+
1824+ #[ tokio:: test]
1825+ async fn empty_rhs_keeps_nulls_and_duplicates ( ) -> Result < ( ) > {
1826+ let left = vec ! [
1827+ int_batch( vec![ None , Some ( 1 ) , Some ( 2 ) ] , & [ 0 , 1 , 2 ] ) ?,
1828+ int_batch( vec![ Some ( 3 ) , Some ( 3 ) ] , & [ 3 , 4 ] ) ?,
1829+ ] ;
1830+ // EOF without any batch and EOF after empty batches are both empty RHS.
1831+ for right in [
1832+ vec ! [ ] ,
1833+ vec ! [ int_batch( vec![ ] , & [ ] ) ?, int_batch( vec![ ] , & [ ] ) ?] ,
1834+ ] {
1835+ let actual = collect_ids ( left. clone ( ) , right, 2 ) . await ?;
1836+ assert_eq ! ( actual, vec![ 0 , 1 , 2 , 3 , 4 ] ) ;
1837+ }
1838+ Ok ( ( ) )
1839+ }
1840+
1841+ #[ tokio:: test]
1842+ async fn rhs_null_after_empty_batch_returns_no_rows ( ) -> Result < ( ) > {
1843+ let left = vec ! [
1844+ int_batch( vec![ None , Some ( 1 ) ] , & [ 0 , 1 ] ) ?,
1845+ int_batch( vec![ Some ( 2 ) , Some ( 3 ) ] , & [ 2 , 3 ] ) ?,
1846+ ] ;
1847+ // Cover an all-NULL RHS and a NULL followed by a non-NULL batch.
1848+ for right in [
1849+ vec ! [ int_batch( vec![ ] , & [ ] ) ?, int_batch( vec![ None ] , & [ 10 ] ) ?] ,
1850+ vec ! [
1851+ int_batch( vec![ ] , & [ ] ) ?,
1852+ int_batch( vec![ None ] , & [ 10 ] ) ?,
1853+ int_batch( vec![ Some ( 2 ) ] , & [ 11 ] ) ?,
1854+ ] ,
1855+ ] {
1856+ let actual = collect_ids ( left. clone ( ) , right, 2 ) . await ?;
1857+ assert ! ( actual. is_empty( ) , "unexpected rows: {actual:?}" ) ;
1858+ }
1859+ Ok ( ( ) )
1860+ }
1861+
1862+ #[ tokio:: test]
1863+ async fn startup_preserves_first_rhs_batch_across_key_groups ( ) -> Result < ( ) > {
1864+ let left = vec ! [
1865+ int_batch( vec![ None , Some ( 1 ) , Some ( 2 ) ] , & [ 0 , 1 , 2 ] ) ?,
1866+ int_batch( vec![ ] , & [ ] ) ?,
1867+ int_batch( vec![ Some ( 2 ) , Some ( 3 ) , Some ( 3 ) ] , & [ 3 , 4 , 5 ] ) ?,
1868+ int_batch( vec![ Some ( 4 ) , Some ( 5 ) ] , & [ 6 , 7 ] ) ?,
1869+ ] ;
1870+ let right = vec ! [
1871+ int_batch( vec![ ] , & [ ] ) ?,
1872+ int_batch( vec![ Some ( 2 ) ] , & [ 10 ] ) ?,
1873+ int_batch( vec![ ] , & [ ] ) ?,
1874+ int_batch( vec![ Some ( 4 ) ] , & [ 11 ] ) ?,
1875+ int_batch( vec![ Some ( 4 ) ] , & [ 12 ] ) ?,
1876+ ] ;
1877+ for batch_size in [ 1 , 2 ] {
1878+ let actual = collect_ids ( left. clone ( ) , right. clone ( ) , batch_size) . await ?;
1879+ // Keep 1, 3, 3, 5; both occurrences of 2 must match the first RHS batch.
1880+ assert_eq ! ( actual, vec![ 1 , 4 , 5 , 7 ] , "batch size {batch_size}" ) ;
1881+ }
1882+ Ok ( ( ) )
1883+ }
1884+
1885+ #[ tokio:: test]
1886+ async fn exhausted_rhs_drains_remaining_left_batches ( ) -> Result < ( ) > {
1887+ let left = vec ! [
1888+ int_batch( vec![ None , Some ( 1 ) ] , & [ 0 , 1 ] ) ?,
1889+ int_batch( vec![ Some ( 2 ) ] , & [ 2 ] ) ?,
1890+ int_batch( vec![ Some ( 3 ) , Some ( 3 ) ] , & [ 3 , 4 ] ) ?,
1891+ int_batch( vec![ Some ( 4 ) , Some ( 5 ) ] , & [ 5 , 6 ] ) ?,
1892+ ] ;
1893+ let right = vec ! [ int_batch( vec![ Some ( 2 ) ] , & [ 10 ] ) ?] ;
1894+ let actual = collect_ids ( left, right, 2 ) . await ?;
1895+ // Exclude the initial NULL and retain unmatched rows after RHS EOF,
1896+ // including duplicates and the final partial output batch.
1897+ assert_eq ! ( actual, vec![ 1 , 3 , 4 , 5 , 6 ] ) ;
1898+ Ok ( ( ) )
1899+ }
1900+
1901+ // Like HashJoin's build_table_dict_key, keep all dictionary indices valid
1902+ // so NULL dictionary values exercise logical rather than physical nulls.
1903+ fn dictionary_batch (
1904+ keys : Vec < i32 > ,
1905+ values : Vec < Option < i32 > > ,
1906+ row_ids : & [ i32 ] ,
1907+ ) -> Result < RecordBatch > {
1908+ let keys = DictionaryArray :: < Int32Type > :: try_new (
1909+ Int32Array :: from ( keys) ,
1910+ Arc :: new ( Int32Array :: from ( values) ) ,
1911+ ) ?;
1912+ batch ( Arc :: new ( keys) , row_ids)
1913+ }
1914+
1915+ #[ tokio:: test]
1916+ async fn dictionary_value_nulls_follow_not_in_semantics ( ) -> Result < ( ) > {
1917+ let left = dictionary_batch (
1918+ vec ! [ 0 , 1 , 2 , 3 , 3 ] ,
1919+ vec ! [ None , Some ( 1 ) , Some ( 2 ) , Some ( 3 ) ] ,
1920+ & [ 0 , 1 , 2 , 3 , 4 ] ,
1921+ ) ?;
1922+ assert_eq ! ( left. column( 0 ) . null_count( ) , 0 ) ;
1923+ assert_eq ! ( left. column( 0 ) . logical_null_count( ) , 1 ) ;
1924+
1925+ // The same logical key (2) has a different dictionary index on each side.
1926+ let rhs_with_null = dictionary_batch ( vec ! [ 0 , 1 ] , vec ! [ None , Some ( 2 ) ] , & [ 10 , 11 ] ) ?;
1927+ assert_eq ! ( rhs_with_null. column( 0 ) . null_count( ) , 0 ) ;
1928+ assert_eq ! ( rhs_with_null. column( 0 ) . logical_null_count( ) , 1 ) ;
1929+
1930+ let cases = [
1931+ ( vec ! [ ] , vec ! [ 0 , 1 , 2 , 3 , 4 ] ) ,
1932+ (
1933+ vec ! [ dictionary_batch( vec![ 0 ] , vec![ Some ( 2 ) ] , & [ 10 ] ) ?] ,
1934+ vec ! [ 1 , 3 , 4 ] ,
1935+ ) ,
1936+ ( vec ! [ rhs_with_null] , vec ! [ ] ) ,
1937+ ] ;
1938+ for ( right, expected) in cases {
1939+ let actual = collect_ids ( vec ! [ left. clone( ) ] , right, 2 ) . await ?;
1940+ assert_eq ! ( actual, expected) ;
1941+ }
1942+ Ok ( ( ) )
1943+ }
1944+ }
1945+
17641946#[ tokio:: test]
17651947async fn join_right_anti_one_one ( ) -> Result < ( ) > {
17661948 let left = build_table (
0 commit comments