1818//! Explicit immutable build reuse for embedding executors.
1919
2020use super :: * ;
21- use crate :: memory:: MemoryStream ;
2221use arrow:: array:: { Array , AsArray } ;
2322use datafusion_common:: exec_datafusion_err;
2423use datafusion_execution:: memory_pool:: MemoryPool ;
@@ -40,7 +39,6 @@ use datafusion_execution::memory_pool::MemoryPool;
4039/// use hash-table membership filters instead of copying range or IN-list values.
4140pub struct PreparedHashJoinBuild {
4241 build : Arc < JoinBuildData > ,
43- schema : SchemaRef ,
4442 keys : Vec < usize > ,
4543 null_equality : NullEquality ,
4644}
@@ -49,7 +47,7 @@ impl fmt::Debug for PreparedHashJoinBuild {
4947 /// Describe immutable metadata without dumping table contents.
5048 fn fmt ( & self , f : & mut fmt:: Formatter < ' _ > ) -> fmt:: Result {
5149 f. debug_struct ( "PreparedHashJoinBuild" )
52- . field ( "schema" , & self . schema )
50+ . field ( "schema" , & self . build . batch . schema ( ) )
5351 . field ( "keys" , & self . keys )
5452 . field ( "rows" , & self . num_rows ( ) )
5553 . field ( "reserved_bytes" , & self . reserved_bytes ( ) )
@@ -84,7 +82,7 @@ impl PreparedHashJoinBuild {
8482 /// consuming input or modifying either plan. Cache identity is caller-owned.
8583 pub ( super ) fn validate ( & self , join : & HashJoinExec ) -> Result < ( ) > {
8684 let keys = prepared_key_indices ( join) ?;
87- if join. left . schema ( ) != self . schema
85+ if join. left . schema ( ) != self . build . batch . schema ( )
8886 || keys != self . keys
8987 || join. null_equality != self . null_equality
9088 {
@@ -130,7 +128,7 @@ impl HashJoinExec {
130128 /// each consuming join publishes them into its own dynamic filter.
131129 pub async fn prepare_build (
132130 & self ,
133- mut input : SendableRecordBatchStream ,
131+ input : SendableRecordBatchStream ,
134132 pool : Arc < dyn MemoryPool > ,
135133 config : Arc < ConfigOptions > ,
136134 ) -> Result < Arc < PreparedHashJoinBuild > > {
@@ -161,28 +159,6 @@ impl HashJoinExec {
161159 let count = MetricBuilder :: new ( & metrics_set)
162160 . counter ( ARRAY_MAP_CREATED_COUNT_METRIC_NAME , 0 ) ;
163161 let reservation = MemoryConsumer :: new ( "PreparedHashJoinBuild" ) . register ( & pool) ;
164- // Retain each shared input buffer only once. The collector admits an
165- // independent copy only when it actually concatenates buffered batches.
166- // Ordinary task-local joins retain their existing accounting policy.
167- let mut input_memory = RecordBatchMemoryCounter :: new ( ) ;
168- let mut batches = Vec :: new ( ) ;
169- // Reject cumulative UTF-8 offset overflow before Arrow allocates a compact build.
170- let mut concat_values = vec ! [ 0usize ; schema. fields( ) . len( ) ] ;
171- while let Some ( batch) = input. try_next ( ) . await ? {
172- check_byte_concat_sizes ( & batch, & mut concat_values) ?;
173- reservation. try_grow ( input_memory. count_batch ( & batch) ) ?;
174- batches. push ( batch) ;
175- }
176- drop ( input) ;
177- if batches. is_empty ( ) {
178- // Empty UTF-8 arrays still retain an offset buffer.
179- let batch = RecordBatch :: new_empty ( Arc :: clone ( & schema) ) ;
180- reservation. try_grow ( input_memory. count_batch ( & batch) ) ?;
181- batches. push ( batch) ;
182- }
183- let retained_input_bytes = input_memory. memory_usage ( ) ;
184- drop ( input_memory) ;
185- let input = Box :: pin ( MemoryStream :: try_new ( batches, Arc :: clone ( & schema) , None ) ?) ;
186162 let data = collect_left_input (
187163 self . random_state . random_state ( ) . clone ( ) ,
188164 input,
@@ -195,12 +171,11 @@ impl HashJoinExec {
195171 config,
196172 self . null_equality ,
197173 count,
198- Some ( retained_input_bytes ) ,
174+ true ,
199175 )
200176 . await ?;
201177 Ok ( Arc :: new ( PreparedHashJoinBuild {
202178 build : data. build ,
203- schema,
204179 keys,
205180 null_equality : self . null_equality ,
206181 } ) )
@@ -245,11 +220,14 @@ fn utf8_value_span(array: &dyn Array) -> usize {
245220/// Add `batch`'s byte-column spans to `totals`, one entry per schema column, before
246221/// an eventual single-batch concat. Reject offset overflow before allocating
247222/// the value buffer; fixed-width columns leave their totals unchanged.
248- fn check_byte_concat_sizes ( batch : & RecordBatch , totals : & mut [ usize ] ) -> Result < ( ) > {
223+ pub ( super ) fn check_byte_concat_sizes (
224+ batch : & RecordBatch ,
225+ totals : & mut [ usize ] ,
226+ ) -> Result < ( ) > {
249227 for ( array, total) in batch. columns ( ) . iter ( ) . zip ( totals) {
250228 if matches ! ( array. data_type( ) , DataType :: Utf8 ) {
251229 * total = total. checked_add ( utf8_value_span ( array. as_ref ( ) ) )
252- . filter ( |& sum| sum <= i32:: MAX as usize )
230+ . filter ( |& sum| i32:: try_from ( sum ) . is_ok ( ) )
253231 . ok_or_else ( || exec_datafusion_err ! (
254232 "Prepared hash-join UTF-8 column exceeds its offset limit; a compact build is required"
255233 ) ) ?;
@@ -261,8 +239,8 @@ fn check_byte_concat_sizes(batch: &RecordBatch, totals: &mut [usize]) -> Result<
261239impl HashJoinExecBuilder {
262240 /// Attach a fully prepared build to a fresh, compatible join execution.
263241 ///
264- /// Validates before mutation, replaces any task-local build future and
265- /// resets execution metrics. A previously attached task-local dynamic filter
242+ /// [`Self::build`] validates compatibility. Attaching resets the build future
243+ /// and execution metrics. A previously attached task-local dynamic filter
266244 /// keeps its expression handle (also referenced by the probe plan), while
267245 /// its build-report accumulator is reset. The caller must provide a fresh
268246 /// filter expression/probe plan for each independent task.
@@ -276,18 +254,14 @@ impl HashJoinExecBuilder {
276254 /// preserve it. Probe-only rewrites must retain the attached join's `left()`.
277255 /// Replacing that child or changing to incompatible join keys fails; other
278256 /// incompatible join-mode/type changes fail validation in `build`.
279- pub fn with_prepared_build (
280- mut self ,
281- prepared : Arc < PreparedHashJoinBuild > ,
282- ) -> Result < Self > {
283- prepared. validate ( & self . exec ) ?;
257+ pub fn with_prepared_build ( mut self , prepared : Arc < PreparedHashJoinBuild > ) -> Self {
284258 // This removes the ignored child's ordering/equivalences and preserves
285259 // its identity through plan resets.
286260 self . exec . left = Arc :: new ( crate :: empty:: EmptyExec :: new ( self . exec . left . schema ( ) ) ) ;
287261 self . reset_prepared_runtime_state ( ) ;
288262 self . exec . prepared_build = Some ( prepared) ;
289263 self . preserve_properties = false ;
290- Ok ( self )
264+ self
291265 }
292266}
293267
0 commit comments