@@ -1737,6 +1737,188 @@ async fn join_left_anti() -> Result<()> {
17371737 Ok ( ( ) )
17381738}
17391739
1740+ mod null_aware_left_anti_tests {
1741+ use super :: * ;
1742+ use arrow:: array:: { Array , ArrayRef , DictionaryArray } ;
1743+ use arrow:: datatypes:: Int32Type ;
1744+
1745+ fn batch ( keys : ArrayRef , row_ids : & [ i32 ] ) -> Result < RecordBatch > {
1746+ let schema = Arc :: new ( Schema :: new ( vec ! [
1747+ Field :: new( "key" , keys. data_type( ) . clone( ) , true ) ,
1748+ Field :: new( "row_id" , DataType :: Int32 , false ) ,
1749+ ] ) ) ;
1750+ Ok ( RecordBatch :: try_new (
1751+ schema,
1752+ vec ! [ keys, Arc :: new( Int32Array :: from( row_ids. to_vec( ) ) ) ] ,
1753+ ) ?)
1754+ }
1755+
1756+ fn int_batch ( keys : Vec < Option < i32 > > , row_ids : & [ i32 ] ) -> Result < RecordBatch > {
1757+ batch ( Arc :: new ( Int32Array :: from ( keys) ) , row_ids)
1758+ }
1759+
1760+ async fn collect_ids (
1761+ left_batches : Vec < RecordBatch > ,
1762+ right_batches : Vec < RecordBatch > ,
1763+ batch_size : usize ,
1764+ ) -> Result < Vec < i32 > > {
1765+ let left = build_table_from_batches ( left_batches) ;
1766+ // Unlike build_table_from_batches, this must also accept zero batches.
1767+ let right = TestMemoryExec :: try_new_exec ( & [ right_batches] , left. schema ( ) , None ) ?;
1768+ let on: JoinOn = vec ! [ (
1769+ Arc :: new( Column :: new( "key" , 0 ) ) ,
1770+ Arc :: new( Column :: new( "key" , 0 ) ) ,
1771+ ) ] ;
1772+ // Direct execution does not insert sorts. Fixtures are globally sorted
1773+ // ASC NULLS FIRST, including across batch boundaries.
1774+ let join = join_with_options (
1775+ left,
1776+ right,
1777+ on,
1778+ LeftAnti ,
1779+ vec ! [ SortOptions :: new( false , true ) ] ,
1780+ NullEquality :: NullEqualsNothing ,
1781+ ) ?
1782+ . with_null_aware ( true ) ?;
1783+ let context = Arc :: new (
1784+ TaskContext :: default ( )
1785+ . with_session_config ( SessionConfig :: new ( ) . with_batch_size ( batch_size) ) ,
1786+ ) ;
1787+ let batches = common:: collect ( join. execute ( 0 , context) ?) . await ?;
1788+ let mut row_ids = Vec :: new ( ) ;
1789+ for batch in batches {
1790+ let ids = batch
1791+ . column ( 1 )
1792+ . as_any ( )
1793+ . downcast_ref :: < Int32Array > ( )
1794+ . expect ( "Int32 row IDs" ) ;
1795+ row_ids. extend ( ids. values ( ) . iter ( ) . copied ( ) ) ;
1796+ }
1797+ Ok ( row_ids)
1798+ }
1799+
1800+ #[ tokio:: test]
1801+ async fn empty_rhs_keeps_nulls_and_duplicates ( ) -> Result < ( ) > {
1802+ let left = vec ! [
1803+ int_batch( vec![ None , Some ( 1 ) , Some ( 2 ) ] , & [ 0 , 1 , 2 ] ) ?,
1804+ int_batch( vec![ Some ( 3 ) , Some ( 3 ) ] , & [ 3 , 4 ] ) ?,
1805+ ] ;
1806+ // EOF without any batch and EOF after empty batches are both empty RHS.
1807+ for right in [
1808+ vec ! [ ] ,
1809+ vec ! [ int_batch( vec![ ] , & [ ] ) ?, int_batch( vec![ ] , & [ ] ) ?] ,
1810+ ] {
1811+ let actual = collect_ids ( left. clone ( ) , right, 2 ) . await ?;
1812+ assert_eq ! ( actual, vec![ 0 , 1 , 2 , 3 , 4 ] ) ;
1813+ }
1814+ Ok ( ( ) )
1815+ }
1816+
1817+ #[ tokio:: test]
1818+ async fn rhs_null_after_empty_batch_returns_no_rows ( ) -> Result < ( ) > {
1819+ let left = vec ! [
1820+ int_batch( vec![ None , Some ( 1 ) ] , & [ 0 , 1 ] ) ?,
1821+ int_batch( vec![ Some ( 2 ) , Some ( 3 ) ] , & [ 2 , 3 ] ) ?,
1822+ ] ;
1823+ // Cover an all-NULL RHS and a NULL followed by a non-NULL batch.
1824+ for right in [
1825+ vec ! [ int_batch( vec![ ] , & [ ] ) ?, int_batch( vec![ None ] , & [ 10 ] ) ?] ,
1826+ vec ! [
1827+ int_batch( vec![ ] , & [ ] ) ?,
1828+ int_batch( vec![ None ] , & [ 10 ] ) ?,
1829+ int_batch( vec![ Some ( 2 ) ] , & [ 11 ] ) ?,
1830+ ] ,
1831+ ] {
1832+ let actual = collect_ids ( left. clone ( ) , right, 2 ) . await ?;
1833+ assert ! ( actual. is_empty( ) , "unexpected rows: {actual:?}" ) ;
1834+ }
1835+ Ok ( ( ) )
1836+ }
1837+
1838+ #[ tokio:: test]
1839+ async fn startup_preserves_first_rhs_batch_across_key_groups ( ) -> Result < ( ) > {
1840+ let left = vec ! [
1841+ int_batch( vec![ None , Some ( 1 ) , Some ( 2 ) ] , & [ 0 , 1 , 2 ] ) ?,
1842+ int_batch( vec![ ] , & [ ] ) ?,
1843+ int_batch( vec![ Some ( 2 ) , Some ( 3 ) , Some ( 3 ) ] , & [ 3 , 4 , 5 ] ) ?,
1844+ int_batch( vec![ Some ( 4 ) , Some ( 5 ) ] , & [ 6 , 7 ] ) ?,
1845+ ] ;
1846+ let right = vec ! [
1847+ int_batch( vec![ ] , & [ ] ) ?,
1848+ int_batch( vec![ Some ( 2 ) ] , & [ 10 ] ) ?,
1849+ int_batch( vec![ ] , & [ ] ) ?,
1850+ int_batch( vec![ Some ( 4 ) ] , & [ 11 ] ) ?,
1851+ int_batch( vec![ Some ( 4 ) ] , & [ 12 ] ) ?,
1852+ ] ;
1853+ for batch_size in [ 1 , 2 ] {
1854+ let actual = collect_ids ( left. clone ( ) , right. clone ( ) , batch_size) . await ?;
1855+ // Keep 1, 3, 3, 5; both occurrences of 2 must match the first RHS batch.
1856+ assert_eq ! ( actual, vec![ 1 , 4 , 5 , 7 ] , "batch size {batch_size}" ) ;
1857+ }
1858+ Ok ( ( ) )
1859+ }
1860+
1861+ #[ tokio:: test]
1862+ async fn exhausted_rhs_drains_remaining_left_batches ( ) -> Result < ( ) > {
1863+ let left = vec ! [
1864+ int_batch( vec![ None , Some ( 1 ) ] , & [ 0 , 1 ] ) ?,
1865+ int_batch( vec![ Some ( 2 ) ] , & [ 2 ] ) ?,
1866+ int_batch( vec![ Some ( 3 ) , Some ( 3 ) ] , & [ 3 , 4 ] ) ?,
1867+ int_batch( vec![ Some ( 4 ) , Some ( 5 ) ] , & [ 5 , 6 ] ) ?,
1868+ ] ;
1869+ let right = vec ! [ int_batch( vec![ Some ( 2 ) ] , & [ 10 ] ) ?] ;
1870+ let actual = collect_ids ( left, right, 2 ) . await ?;
1871+ // Exclude the initial NULL and retain unmatched rows after RHS EOF,
1872+ // including duplicates and the final partial output batch.
1873+ assert_eq ! ( actual, vec![ 1 , 3 , 4 , 5 , 6 ] ) ;
1874+ Ok ( ( ) )
1875+ }
1876+
1877+ // Like HashJoin's build_table_dict_key, keep all dictionary indices valid
1878+ // so NULL dictionary values exercise logical rather than physical nulls.
1879+ fn dictionary_batch (
1880+ keys : Vec < i32 > ,
1881+ values : Vec < Option < i32 > > ,
1882+ row_ids : & [ i32 ] ,
1883+ ) -> Result < RecordBatch > {
1884+ let keys = DictionaryArray :: < Int32Type > :: try_new (
1885+ Int32Array :: from ( keys) ,
1886+ Arc :: new ( Int32Array :: from ( values) ) ,
1887+ ) ?;
1888+ batch ( Arc :: new ( keys) , row_ids)
1889+ }
1890+
1891+ #[ tokio:: test]
1892+ async fn dictionary_value_nulls_follow_not_in_semantics ( ) -> Result < ( ) > {
1893+ let left = dictionary_batch (
1894+ vec ! [ 0 , 1 , 2 , 3 , 3 ] ,
1895+ vec ! [ None , Some ( 1 ) , Some ( 2 ) , Some ( 3 ) ] ,
1896+ & [ 0 , 1 , 2 , 3 , 4 ] ,
1897+ ) ?;
1898+ assert_eq ! ( left. column( 0 ) . null_count( ) , 0 ) ;
1899+ assert_eq ! ( left. column( 0 ) . logical_null_count( ) , 1 ) ;
1900+
1901+ // The same logical key (2) has a different dictionary index on each side.
1902+ let rhs_with_null = dictionary_batch ( vec ! [ 0 , 1 ] , vec ! [ None , Some ( 2 ) ] , & [ 10 , 11 ] ) ?;
1903+ assert_eq ! ( rhs_with_null. column( 0 ) . null_count( ) , 0 ) ;
1904+ assert_eq ! ( rhs_with_null. column( 0 ) . logical_null_count( ) , 1 ) ;
1905+
1906+ let cases = [
1907+ ( vec ! [ ] , vec ! [ 0 , 1 , 2 , 3 , 4 ] ) ,
1908+ (
1909+ vec ! [ dictionary_batch( vec![ 0 ] , vec![ Some ( 2 ) ] , & [ 10 ] ) ?] ,
1910+ vec ! [ 1 , 3 , 4 ] ,
1911+ ) ,
1912+ ( vec ! [ rhs_with_null] , vec ! [ ] ) ,
1913+ ] ;
1914+ for ( right, expected) in cases {
1915+ let actual = collect_ids ( vec ! [ left. clone( ) ] , right, 2 ) . await ?;
1916+ assert_eq ! ( actual, expected) ;
1917+ }
1918+ Ok ( ( ) )
1919+ }
1920+ }
1921+
17401922#[ tokio:: test]
17411923async fn join_right_anti_one_one ( ) -> Result < ( ) > {
17421924 let left = build_table (
0 commit comments