@@ -65,8 +65,7 @@ use crate::{
6565 metrics:: { ExecutionPlanMetricsSet , MetricsSet } ,
6666} ;
6767
68- use arrow:: array:: { ArrayRef , BooleanBufferBuilder } ;
69- use arrow:: compute:: concat_batches;
68+ use arrow:: array:: { Array , ArrayRef , BooleanBufferBuilder , new_null_array} ;
7069use arrow:: datatypes:: SchemaRef ;
7170use arrow:: record_batch:: RecordBatch ;
7271use arrow:: util:: bit_util;
@@ -106,14 +105,14 @@ const ARRAY_MAP_CREATED_COUNT_METRIC_NAME: &str = "array_map_created_count";
106105#[ expect( clippy:: too_many_arguments) ]
107106fn try_create_array_map (
108107 bounds : & Option < PartitionBounds > ,
109- schema : & SchemaRef ,
108+ _schema : & SchemaRef ,
110109 batches : & [ RecordBatch ] ,
111110 on_left : & [ PhysicalExprRef ] ,
112111 reservation : & mut MemoryReservation ,
113112 perfect_hash_join_small_build_threshold : usize ,
114113 perfect_hash_join_min_key_density : f64 ,
115114 null_equality : NullEquality ,
116- ) -> Result < Option < ( ArrayMap , RecordBatch , Vec < ArrayRef > ) > > {
115+ ) -> Result < Option < ( ArrayMap , Vec < ArrayRef > ) > > {
117116 if on_left. len ( ) != 1 {
118117 return Ok ( None ) ;
119118 }
@@ -178,22 +177,58 @@ fn try_create_array_map(
178177 let mem_size = ArrayMap :: estimate_memory_size ( min_val, max_val, num_row) ;
179178 reservation. try_grow ( mem_size) ?;
180179
181- let batch = concat_batches ( schema , batches ) ? ;
182- let left_values = evaluate_expressions_to_arrays ( on_left, & batch ) ?;
180+ // Evaluate key expressions per- batch and concatenate only key columns
181+ let left_values = concat_key_value_arrays ( on_left, batches . iter ( ) ) ?;
183182
184183 let array_map = ArrayMap :: try_new ( & left_values[ 0 ] , min_val, max_val) ?;
185184
186- Ok ( Some ( ( array_map, batch, left_values) ) )
185+ Ok ( Some ( ( array_map, left_values) ) )
186+ }
187+
188+ /// Evaluate key expressions per-batch and concatenate only the resulting key
189+ /// column arrays. This avoids `concat_batches` on the full data (which would
190+ /// duplicate all columns in memory) while still providing contiguous key arrays
191+ /// for `equal_rows_arr`.
192+ fn concat_key_value_arrays < ' a > (
193+ on : & [ PhysicalExprRef ] ,
194+ batches : impl Iterator < Item = & ' a RecordBatch > ,
195+ ) -> Result < Vec < ArrayRef > > {
196+ let batches: Vec < & RecordBatch > = batches. collect ( ) ;
197+ if batches. is_empty ( ) || on. is_empty ( ) {
198+ return Ok ( on. iter ( ) . map ( |_| new_null_array ( & DataType :: Null , 0 ) ) . collect ( ) ) ;
199+ }
200+ // Evaluate key expressions for each batch
201+ let per_batch_keys: Vec < Vec < ArrayRef > > = batches
202+ . iter ( )
203+ . map ( |batch| evaluate_expressions_to_arrays ( on, batch) )
204+ . collect :: < Result < Vec < _ > > > ( ) ?;
205+ // Transpose and concatenate: for each key column, concat across all batches
206+ let num_keys = on. len ( ) ;
207+ ( 0 ..num_keys)
208+ . map ( |key_idx| {
209+ let arrays: Vec < & dyn Array > = per_batch_keys
210+ . iter ( )
211+ . map ( |keys| keys[ key_idx] . as_ref ( ) )
212+ . collect ( ) ;
213+ Ok ( arrow:: compute:: concat ( & arrays) ?)
214+ } )
215+ . collect ( )
187216}
188217
189218/// HashTable and input data for the left (build side) of a join
190219pub ( super ) struct JoinLeftData {
191220 /// The hash table with indices into `batch`
192221 /// Arc is used to allow sharing with SharedBuildAccumulator for hash map pushdown
193222 pub ( super ) map : Arc < Map > ,
194- /// The input rows for the build side
195- batch : RecordBatch ,
196- /// The build side on expressions values
223+ /// The schema of the build side input
224+ schema : SchemaRef ,
225+ /// The input batches for the build side (kept separate to avoid concat_batches)
226+ batches : Vec < RecordBatch > ,
227+ /// Prefix-sum of batch sizes for converting flat indices to (batch_idx, row_idx).
228+ /// batch_offsets[i] is the starting flat index for batches[i].
229+ /// Has length batches.len() + 1 (last element is total row count).
230+ batch_offsets : Vec < usize > ,
231+ /// The build side on expressions values (concatenated key columns only)
197232 values : Vec < ArrayRef > ,
198233 /// Shared bitmap builder for visited left indices
199234 visited_indices_bitmap : SharedBitmapBuilder ,
@@ -225,9 +260,29 @@ impl JoinLeftData {
225260 & self . map
226261 }
227262
228- /// returns a reference to the build side batch
229- pub ( super ) fn batch ( & self ) -> & RecordBatch {
230- & self . batch
263+ /// Returns a reference to a build-side batch for schema/data-type lookups.
264+ /// If batches exist, returns the first one; otherwise creates an empty batch
265+ /// with the correct build-side schema.
266+ pub ( super ) fn batch ( & self ) -> RecordBatch {
267+ self . batches
268+ . first ( )
269+ . cloned ( )
270+ . unwrap_or_else ( || RecordBatch :: new_empty ( Arc :: clone ( & self . schema ) ) )
271+ }
272+
273+ /// returns a reference to the build side batches
274+ pub ( super ) fn batches ( & self ) -> & [ RecordBatch ] {
275+ & self . batches
276+ }
277+
278+ /// returns the batch offsets for flat-to-(batch,row) index conversion
279+ pub ( super ) fn batch_offsets ( & self ) -> & [ usize ] {
280+ & self . batch_offsets
281+ }
282+
283+ /// returns total number of rows across all build batches
284+ pub ( super ) fn num_rows ( & self ) -> usize {
285+ * self . batch_offsets . last ( ) . unwrap_or ( & 0 )
231286 }
232287
233288 /// returns a reference to the build side expressions values
@@ -1965,8 +2020,8 @@ async fn collect_left_input(
19652020 _ => None ,
19662021 } ;
19672022
1968- let ( join_hash_map, batch , left_values ) =
1969- if let Some ( ( array_map, batch , left_value) ) = try_create_array_map (
2023+ let ( join_hash_map, left_values , should_reverse_batches ) =
2024+ if let Some ( ( array_map, left_value) ) = try_create_array_map (
19702025 & bounds,
19712026 & schema,
19722027 & batches,
@@ -1979,7 +2034,7 @@ async fn collect_left_input(
19792034 array_map_created_count. add ( 1 ) ;
19802035 metrics. build_mem_used . add ( array_map. size ( ) ) ;
19812036
1982- ( Map :: ArrayMap ( array_map) , batch , left_value )
2037+ ( Map :: ArrayMap ( array_map) , left_value , false )
19832038 } else {
19842039 // Estimation of memory size, required for hashtable, prior to allocation.
19852040 // Final result can be verified using `RawTable.allocation_info()`
@@ -2025,21 +2080,41 @@ async fn collect_left_input(
20252080 offset += batch. num_rows ( ) ;
20262081 }
20272082
2028- // Merge all batches into a single batch, so we can directly index into the arrays
2029- let batch = concat_batches ( & schema, batches_iter. clone ( ) ) ?;
2030-
2031- let left_values = evaluate_expressions_to_arrays ( & on_left, & batch) ?;
2083+ // Evaluate key expressions per-batch and concatenate only the key columns
2084+ // (avoids concat_batches on the full data, saving memory)
2085+ let left_values = concat_key_value_arrays ( & on_left, batches_iter. clone ( ) ) ?;
20322086
2033- ( Map :: HashMap ( hashmap) , batch , left_values )
2087+ ( Map :: HashMap ( hashmap) , left_values , true )
20342088 } ;
20352089
2090+ // For HashMap: reverse the batches to match the hash map's flat index ordering.
2091+ // The hash map is built iterating in reverse (last batch first), so flat index 0
2092+ // corresponds to the first row of the last original batch. By reversing, batches[0]
2093+ // is the last original batch, matching the hash map's indexing scheme.
2094+ // For ArrayMap: batches are already in the correct (forward) order since the
2095+ // ArrayMap is built from forward-ordered concatenated key columns.
2096+ let mut batches = batches;
2097+ if should_reverse_batches {
2098+ batches. reverse ( ) ;
2099+ }
2100+
2101+ // Compute batch_offsets (prefix sum of batch sizes) for flat-to-(batch,row) conversion
2102+ let batch_offsets = {
2103+ let mut offsets = Vec :: with_capacity ( batches. len ( ) + 1 ) ;
2104+ offsets. push ( 0usize ) ;
2105+ for b in & batches {
2106+ offsets. push ( offsets. last ( ) . unwrap ( ) + b. num_rows ( ) ) ;
2107+ }
2108+ offsets
2109+ } ;
2110+
20362111 // Reserve additional memory for visited indices bitmap and create shared builder
20372112 let visited_indices_bitmap = if with_visited_indices_bitmap {
2038- let bitmap_size = bit_util:: ceil ( batch . num_rows ( ) , 8 ) ;
2113+ let bitmap_size = bit_util:: ceil ( num_rows, 8 ) ;
20392114 reservation. try_grow ( bitmap_size) ?;
20402115 metrics. build_mem_used . add ( bitmap_size) ;
20412116
2042- let mut bitmap_buffer = BooleanBufferBuilder :: new ( batch . num_rows ( ) ) ;
2117+ let mut bitmap_buffer = BooleanBufferBuilder :: new ( num_rows) ;
20432118 bitmap_buffer. append_n ( num_rows, false ) ;
20442119 bitmap_buffer
20452120 } else {
@@ -2080,7 +2155,9 @@ async fn collect_left_input(
20802155
20812156 let data = JoinLeftData {
20822157 map,
2083- batch,
2158+ schema : Arc :: clone ( & schema) ,
2159+ batches,
2160+ batch_offsets,
20842161 values : left_values,
20852162 visited_indices_bitmap : Mutex :: new ( visited_indices_bitmap) ,
20862163 probe_threads_counter : AtomicUsize :: new ( probe_threads_count) ,
0 commit comments