@@ -70,6 +70,7 @@ use crate::{
7070} ;
7171
7272use arrow:: array:: { Array , ArrayRef , BooleanBufferBuilder , UInt64Array } ;
73+ use arrow:: buffer:: NullBuffer ;
7374use arrow:: compute:: concat_batches;
7475use arrow:: datatypes:: SchemaRef ;
7576use arrow:: record_batch:: RecordBatch ;
@@ -304,6 +305,11 @@ pub(super) struct JoinLeftData {
304305 batch : RecordBatch ,
305306 /// The build side on expressions values
306307 values : Vec < ArrayRef > ,
308+ /// Logical nulls of the scalar key for uncorrelated null-aware `LeftMark`.
309+ ///
310+ /// Computed once while collecting the build side because dictionary
311+ /// logical-null discovery can scan nested dictionary values.
312+ left_mark_build_key_nulls : Option < NullBuffer > ,
307313 /// Shared bitmap builder for visited left indices
308314 visited_indices_bitmap : SharedBitmapBuilder ,
309315 /// Shared bitmap builder for null marks
@@ -368,6 +374,10 @@ impl JoinLeftData {
368374 & self . values
369375 }
370376
377+ pub ( super ) fn left_mark_build_key_nulls ( & self ) -> Option < & NullBuffer > {
378+ self . left_mark_build_key_nulls . as_ref ( )
379+ }
380+
371381 /// returns a reference to the visited indices bitmap
372382 pub ( super ) fn visited_indices_bitmap ( & self ) -> & SharedBitmapBuilder {
373383 & self . visited_indices_bitmap
@@ -3138,12 +3148,37 @@ async fn collect_left_input(
31383148 && !left_values. is_empty ( )
31393149 && left_values[ 0 ] . logical_null_count ( ) > 0 ;
31403150
3151+ let left_mark_build_key_nulls = if matches ! (
3152+ null_aware,
3153+ Some ( NullAwareMode :: LeftMark { correlated: false } )
3154+ ) {
3155+ left_values[ 0 ]
3156+ . logical_nulls ( )
3157+ . filter ( |nulls| nulls. null_count ( ) > 0 )
3158+ } else {
3159+ None
3160+ } ;
3161+ if let Some ( nulls) = & left_mark_build_key_nulls {
3162+ // `logical_nulls` reuses an array's physical null buffer when it can.
3163+ // Only charge newly materialized buffers, such as dictionary logical
3164+ // nulls, because the build batch's buffers were reserved above.
3165+ let reuses_input_buffer = left_values[ 0 ]
3166+ . nulls ( )
3167+ . is_some_and ( |input| input. inner ( ) . ptr_eq ( nulls. inner ( ) ) ) ;
3168+ if !reuses_input_buffer {
3169+ let retained_size = nulls. buffer ( ) . capacity ( ) ;
3170+ reservation. try_grow ( retained_size) ?;
3171+ metrics. build_mem_used . add ( retained_size) ;
3172+ }
3173+ }
3174+
31413175 let data = JoinLeftData {
31423176 map,
31433177 null_aware_scope_map,
31443178 null_value_build_rows,
31453179 batch,
31463180 values : left_values,
3181+ left_mark_build_key_nulls,
31473182 visited_indices_bitmap : Mutex :: new ( visited_indices_bitmap) ,
31483183 null_indices_bitmap : Mutex :: new ( null_indices_bitmap) ,
31493184 probe_completion : ProbeCompletion :: new ( probe_threads_count) ,
@@ -9186,6 +9221,79 @@ mod tests {
91869221 Ok ( ( ) )
91879222 }
91889223
9224+ fn build_single_column_table ( name : & str , array : ArrayRef ) -> Arc < dyn ExecutionPlan > {
9225+ let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new(
9226+ name,
9227+ array. data_type( ) . clone( ) ,
9228+ true ,
9229+ ) ] ) ) ;
9230+ let batch = RecordBatch :: try_new ( Arc :: clone ( & schema) , vec ! [ array] ) . unwrap ( ) ;
9231+ TestMemoryExec :: try_new_exec ( & [ vec ! [ batch] ] , schema, None ) . unwrap ( )
9232+ }
9233+
9234+ fn nested_dictionary_array (
9235+ leaf_values : Vec < Option < i32 > > ,
9236+ inner_keys : Vec < i32 > ,
9237+ outer_keys : Vec < i32 > ,
9238+ ) -> ArrayRef {
9239+ let leaf_values: ArrayRef = Arc :: new ( Int32Array :: from ( leaf_values) ) ;
9240+ let inner_keys = Int32Array :: from ( inner_keys) ;
9241+ let inner_values: ArrayRef =
9242+ Arc :: new ( DictionaryArray :: < Int32Type > :: new ( inner_keys, leaf_values) ) ;
9243+ Arc :: new ( DictionaryArray :: < Int32Type > :: new (
9244+ Int32Array :: from ( outer_keys) ,
9245+ inner_values,
9246+ ) )
9247+ }
9248+
9249+ /// A nested dictionary's logical null bitmap must be computed once for the
9250+ /// complete build key and remain indexed by the original build row.
9251+ #[ apply( hash_join_exec_configs) ]
9252+ #[ tokio:: test]
9253+ async fn test_null_aware_left_mark_nested_dictionary_logical_null (
9254+ batch_size : usize ,
9255+ ) -> Result < ( ) > {
9256+ let task_ctx = prepare_task_ctx ( batch_size, false ) ;
9257+ let left_key = nested_dictionary_array (
9258+ vec ! [ Some ( 1 ) , None , Some ( 4 ) ] ,
9259+ vec ! [ 0 , 1 , 2 ] ,
9260+ vec ! [ 0 , 1 , 2 ] ,
9261+ ) ;
9262+ assert_eq ! ( left_key. logical_null_count( ) , 1 ) ;
9263+ let left = build_single_column_table ( "c1" , left_key) ;
9264+ let right = build_single_column_table (
9265+ "c2" ,
9266+ nested_dictionary_array (
9267+ vec ! [ Some ( 1 ) , Some ( 2 ) , Some ( 3 ) ] ,
9268+ vec ! [ 0 , 1 , 2 ] ,
9269+ vec ! [ 0 , 1 , 2 ] ,
9270+ ) ,
9271+ ) ;
9272+ let on = vec ! [ (
9273+ Arc :: new( Column :: new_with_schema( "c1" , & left. schema( ) ) ?) as _,
9274+ Arc :: new( Column :: new_with_schema( "c2" , & right. schema( ) ) ?) as _,
9275+ ) ] ;
9276+ let join = HashJoinExec :: try_new (
9277+ left,
9278+ right,
9279+ on,
9280+ None ,
9281+ & JoinType :: LeftMark ,
9282+ None ,
9283+ PartitionMode :: CollectLeft ,
9284+ NullEquality :: NullEqualsNothing ,
9285+ true ,
9286+ ) ?;
9287+
9288+ let batches = common:: collect ( join. execute ( 0 , task_ctx) ?) . await ?;
9289+ let marks = batches
9290+ . iter ( )
9291+ . flat_map ( |batch| batch. column ( batch. num_columns ( ) - 1 ) . as_boolean ( ) . iter ( ) )
9292+ . collect :: < Vec < _ > > ( ) ;
9293+ assert_eq ! ( marks, vec![ Some ( true ) , None , Some ( false ) ] ) ;
9294+ Ok ( ( ) )
9295+ }
9296+
91899297 /// Scalar correlated null-aware left mark join where only the build side's
91909298 /// value key contains NULL.
91919299 ///
0 commit comments