@@ -127,7 +127,6 @@ impl UnnestExec {
127127 . collect ( ) ;
128128
129129 // Manually build projection mapping from non-unnested input columns to their positions in the output
130- let input_schema = input. schema ( ) ;
131130 let projection_mapping: ProjectionMapping = non_unnested_indices
132131 . iter ( )
133132 . map ( |& input_idx| {
@@ -210,10 +209,11 @@ impl UnnestExec {
210209 // UnnestExec can preserve parent-row boundaries across downstream
211210 // repartitioning while remaining memory-bounded. Stacked UNNEST still
212211 // needs a parent-row-aware chunking strategy to avoid order regressions.
212+ let input_schema = self . input . schema ( ) ;
213213 self . list_column_indices . iter ( ) . any ( |unnest| {
214214 unnest. depth > 1
215215 || is_unnest_placeholder_field (
216- self . input . schema ( ) . field ( unnest. index_in_input_schema ) ,
216+ input_schema . field ( unnest. index_in_input_schema ) ,
217217 )
218218 } )
219219 }
@@ -341,6 +341,16 @@ impl PendingBatch {
341341 fn new ( batch : RecordBatch ) -> Self {
342342 Self { batch, next_row : 0 }
343343 }
344+
345+ fn remaining_rows ( & self ) -> usize {
346+ self . batch . num_rows ( ) . saturating_sub ( self . next_row )
347+ }
348+
349+ fn slice ( & mut self , row_count : usize ) -> RecordBatch {
350+ let batch = self . batch . slice ( self . next_row , row_count) ;
351+ self . next_row += row_count;
352+ batch
353+ }
344354}
345355
346356/// A stream that issues [RecordBatch]es with unnested column data.
@@ -428,13 +438,13 @@ impl UnnestStream {
428438 return Ok ( None ) ;
429439 } ;
430440
431- if pending. next_row >= pending . batch . num_rows ( ) {
441+ if pending. remaining_rows ( ) == 0 {
432442 self . pending = None ;
433443 return Ok ( None ) ;
434444 }
435445
436446 let row_count = if self . disable_chunking_for_recursive_or_staged_unnest {
437- pending. batch . num_rows ( ) . saturating_sub ( pending . next_row )
447+ pending. remaining_rows ( )
438448 } else {
439449 next_input_slice_row_count (
440450 & pending. batch ,
@@ -444,31 +454,32 @@ impl UnnestStream {
444454 self . output_batch_size ,
445455 ) ?
446456 } ;
447- let batch_slice = pending. batch . slice ( pending. next_row , row_count) ;
448- pending. next_row += row_count;
449- let exhausted = pending. next_row >= pending. batch . num_rows ( ) ;
450- if exhausted {
457+ let batch_slice = pending. slice ( row_count) ;
458+ if pending. remaining_rows ( ) == 0 {
451459 self . pending = None ;
452460 }
453461
454- let elapsed_compute = self . metrics . baseline_metrics . elapsed_compute ( ) . clone ( ) ;
455- let timer = elapsed_compute. timer ( ) ;
456- let result = build_batch (
457- & batch_slice,
458- & self . schema ,
459- & self . list_type_columns ,
460- & self . struct_column_indices ,
461- & self . options ,
462- ) ?;
463- timer. done ( ) ;
464-
465- let Some ( result_batch) = result else {
462+ let Some ( result_batch) = self . build_batch ( & batch_slice) ? else {
466463 continue ;
467464 } ;
468465
469466 return Ok ( Some ( result_batch) ) ;
470467 }
471468 }
469+
470+ fn build_batch ( & self , batch : & RecordBatch ) -> Result < Option < RecordBatch > > {
471+ let elapsed_compute = self . metrics . baseline_metrics . elapsed_compute ( ) . clone ( ) ;
472+ let timer = elapsed_compute. timer ( ) ;
473+ let result = build_batch (
474+ batch,
475+ & self . schema ,
476+ & self . list_type_columns ,
477+ & self . struct_column_indices ,
478+ & self . options ,
479+ ) ;
480+ timer. done ( ) ;
481+ result
482+ }
472483}
473484
474485fn next_input_slice_row_count (
@@ -693,17 +704,13 @@ fn list_unnest_at_level(
693704 . iter ( )
694705 . enumerate ( )
695706 . map ( |( i, _) | {
696- let mut involved_in_unnesting = false ;
697- for unnesting in list_type_unnests {
698- if unnesting. index_in_input_schema != i {
699- continue ;
700- }
701- involved_in_unnesting = true ;
702- if unnesting. depth < level_to_unnest {
703- return true ;
704- }
705- }
706- !involved_in_unnesting
707+ let mut unnests = list_type_unnests
708+ . iter ( )
709+ . filter ( |unnest| unnest. index_in_input_schema == i) ;
710+ unnests. any ( |unnest| unnest. depth < level_to_unnest)
711+ || !list_type_unnests
712+ . iter ( )
713+ . any ( |unnest| unnest. index_in_input_schema == i)
707714 } )
708715 . collect ( ) ;
709716
@@ -776,67 +783,68 @@ fn build_batch(
776783 struct_column_indices : & HashSet < usize > ,
777784 options : & UnnestOptions ,
778785) -> Result < Option < RecordBatch > > {
779- let transformed = if list_type_columns. is_empty ( ) {
780- flatten_struct_cols ( batch. columns ( ) , schema, struct_column_indices)
781- } else {
782- let mut temp_unnested_result = HashMap :: new ( ) ;
783- let max_recursion = list_type_columns
784- . iter ( )
785- . map ( |ListUnnest { depth, .. } | * depth)
786- . max ( )
787- . unwrap_or_default ( ) ;
788- let mut flatten_arrs = vec ! [ ] ;
789-
790- for depth in ( 1 ..=max_recursion) . rev ( ) {
791- let input = if depth == max_recursion {
792- batch. columns ( )
793- } else {
794- & flatten_arrs
795- } ;
796- let Some ( temp_result) = list_unnest_at_level (
797- input,
798- list_type_columns,
799- & mut temp_unnested_result,
800- depth,
801- options,
802- ) ?
803- else {
804- return Ok ( None ) ;
805- } ;
806- flatten_arrs = temp_result;
807- }
786+ if list_type_columns. is_empty ( ) {
787+ return flatten_struct_cols ( batch. columns ( ) , schema, struct_column_indices)
788+ . map ( Some ) ;
789+ }
808790
809- let output_order: HashMap < ListUnnest , usize > = list_type_columns
810- . iter ( )
811- . enumerate ( )
812- . map ( |( order, unnest_def) | ( * unnest_def, order) )
813- . collect ( ) ;
814- let mut multi_unnested_per_original_index: HashMap < usize , Vec < ArrayRef > > =
815- HashMap :: new ( ) ;
816- let mut unnested_columns = temp_unnested_result. into_iter ( ) . collect :: < Vec < _ > > ( ) ;
817- unnested_columns. sort_by_key ( |( unnest, _) | {
818- output_order. get ( unnest) . copied ( ) . unwrap_or_default ( )
819- } ) ;
820- for ( unnest, arr) in unnested_columns {
821- multi_unnested_per_original_index
822- . entry ( unnest. index_in_input_schema )
823- . or_default ( )
824- . push ( arr) ;
825- }
791+ let mut temp_unnested_result = HashMap :: new ( ) ;
792+ let max_recursion = list_type_columns
793+ . iter ( )
794+ . map ( |ListUnnest { depth, .. } | * depth)
795+ . max ( )
796+ . unwrap_or_default ( ) ;
797+ let mut flatten_arrs = vec ! [ ] ;
798+
799+ for depth in ( 1 ..=max_recursion) . rev ( ) {
800+ let input = if depth == max_recursion {
801+ batch. columns ( )
802+ } else {
803+ & flatten_arrs
804+ } ;
805+ let Some ( temp_result) = list_unnest_at_level (
806+ input,
807+ list_type_columns,
808+ & mut temp_unnested_result,
809+ depth,
810+ options,
811+ ) ?
812+ else {
813+ return Ok ( None ) ;
814+ } ;
815+ flatten_arrs = temp_result;
816+ }
817+
818+ let output_order: HashMap < ListUnnest , usize > = list_type_columns
819+ . iter ( )
820+ . enumerate ( )
821+ . map ( |( order, unnest_def) | ( * unnest_def, order) )
822+ . collect ( ) ;
823+ let mut multi_unnested_per_original_index = HashMap :: < usize , Vec < _ > > :: new ( ) ;
824+ let mut unnested_columns = temp_unnested_result. into_iter ( ) . collect :: < Vec < _ > > ( ) ;
825+ unnested_columns
826+ . sort_by_key ( |( unnest, _) | output_order. get ( unnest) . copied ( ) . unwrap_or_default ( ) ) ;
827+ for ( unnest, arr) in unnested_columns {
828+ multi_unnested_per_original_index
829+ . entry ( unnest. index_in_input_schema )
830+ . or_default ( )
831+ . push ( arr) ;
832+ }
826833
827- let ret = flatten_arrs
834+ flatten_struct_cols (
835+ & flatten_arrs
828836 . into_iter ( )
829837 . enumerate ( )
830838 . flat_map ( |( col_idx, arr) | {
831839 multi_unnested_per_original_index
832840 . remove ( & col_idx)
833841 . unwrap_or_else ( || vec ! [ arr] )
834842 } )
835- . collect :: < Vec < _ > > ( ) ;
836-
837- flatten_struct_cols ( & ret , schema , struct_column_indices)
838- } ? ;
839- Ok ( Some ( transformed ) )
843+ . collect :: < Vec < _ > > ( ) ,
844+ schema ,
845+ struct_column_indices,
846+ )
847+ . map ( Some )
840848}
841849
842850/// Find the longest list length among the given list arrays for each row.
0 commit comments