Skip to content

Commit 640971f

Browse files
committed
Refactor unnest processing and optimize bookkeeping
Remove duplicate input.schema() lookup in compute_properties. Tighten disable_chunking_for_recursive_or_staged_unnest() by reusing one schema handle. Move pending-batch bookkeeping into small PendingBatch helpers and collapse batch building/timing into UnnestStream::build_batch. Simplify build_batch() to early-return for no-list cases and reduce temporary variables, while keeping the recursive/staged unnest fallback and TODO(apache#20788) behavior intact.
1 parent 8212e00 commit 640971f

1 file changed

Lines changed: 91 additions & 83 deletions

File tree

‎datafusion/physical-plan/src/unnest.rs‎

Lines changed: 91 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,6 @@ impl UnnestExec {
127127
.collect();
128128

129129
// Manually build projection mapping from non-unnested input columns to their positions in the output
130-
let input_schema = input.schema();
131130
let projection_mapping: ProjectionMapping = non_unnested_indices
132131
.iter()
133132
.map(|&input_idx| {
@@ -210,10 +209,11 @@ impl UnnestExec {
210209
// UnnestExec can preserve parent-row boundaries across downstream
211210
// repartitioning while remaining memory-bounded. Stacked UNNEST still
212211
// needs a parent-row-aware chunking strategy to avoid order regressions.
212+
let input_schema = self.input.schema();
213213
self.list_column_indices.iter().any(|unnest| {
214214
unnest.depth > 1
215215
|| is_unnest_placeholder_field(
216-
self.input.schema().field(unnest.index_in_input_schema),
216+
input_schema.field(unnest.index_in_input_schema),
217217
)
218218
})
219219
}
@@ -341,6 +341,16 @@ impl PendingBatch {
341341
fn new(batch: RecordBatch) -> Self {
342342
Self { batch, next_row: 0 }
343343
}
344+
345+
fn remaining_rows(&self) -> usize {
346+
self.batch.num_rows().saturating_sub(self.next_row)
347+
}
348+
349+
fn slice(&mut self, row_count: usize) -> RecordBatch {
350+
let batch = self.batch.slice(self.next_row, row_count);
351+
self.next_row += row_count;
352+
batch
353+
}
344354
}
345355

346356
/// A stream that issues [RecordBatch]es with unnested column data.
@@ -428,13 +438,13 @@ impl UnnestStream {
428438
return Ok(None);
429439
};
430440

431-
if pending.next_row >= pending.batch.num_rows() {
441+
if pending.remaining_rows() == 0 {
432442
self.pending = None;
433443
return Ok(None);
434444
}
435445

436446
let row_count = if self.disable_chunking_for_recursive_or_staged_unnest {
437-
pending.batch.num_rows().saturating_sub(pending.next_row)
447+
pending.remaining_rows()
438448
} else {
439449
next_input_slice_row_count(
440450
&pending.batch,
@@ -444,31 +454,32 @@ impl UnnestStream {
444454
self.output_batch_size,
445455
)?
446456
};
447-
let batch_slice = pending.batch.slice(pending.next_row, row_count);
448-
pending.next_row += row_count;
449-
let exhausted = pending.next_row >= pending.batch.num_rows();
450-
if exhausted {
457+
let batch_slice = pending.slice(row_count);
458+
if pending.remaining_rows() == 0 {
451459
self.pending = None;
452460
}
453461

454-
let elapsed_compute = self.metrics.baseline_metrics.elapsed_compute().clone();
455-
let timer = elapsed_compute.timer();
456-
let result = build_batch(
457-
&batch_slice,
458-
&self.schema,
459-
&self.list_type_columns,
460-
&self.struct_column_indices,
461-
&self.options,
462-
)?;
463-
timer.done();
464-
465-
let Some(result_batch) = result else {
462+
let Some(result_batch) = self.build_batch(&batch_slice)? else {
466463
continue;
467464
};
468465

469466
return Ok(Some(result_batch));
470467
}
471468
}
469+
470+
fn build_batch(&self, batch: &RecordBatch) -> Result<Option<RecordBatch>> {
471+
let elapsed_compute = self.metrics.baseline_metrics.elapsed_compute().clone();
472+
let timer = elapsed_compute.timer();
473+
let result = build_batch(
474+
batch,
475+
&self.schema,
476+
&self.list_type_columns,
477+
&self.struct_column_indices,
478+
&self.options,
479+
);
480+
timer.done();
481+
result
482+
}
472483
}
473484

474485
fn next_input_slice_row_count(
@@ -693,17 +704,13 @@ fn list_unnest_at_level(
693704
.iter()
694705
.enumerate()
695706
.map(|(i, _)| {
696-
let mut involved_in_unnesting = false;
697-
for unnesting in list_type_unnests {
698-
if unnesting.index_in_input_schema != i {
699-
continue;
700-
}
701-
involved_in_unnesting = true;
702-
if unnesting.depth < level_to_unnest {
703-
return true;
704-
}
705-
}
706-
!involved_in_unnesting
707+
let mut unnests = list_type_unnests
708+
.iter()
709+
.filter(|unnest| unnest.index_in_input_schema == i);
710+
unnests.any(|unnest| unnest.depth < level_to_unnest)
711+
|| !list_type_unnests
712+
.iter()
713+
.any(|unnest| unnest.index_in_input_schema == i)
707714
})
708715
.collect();
709716

@@ -776,67 +783,68 @@ fn build_batch(
776783
struct_column_indices: &HashSet<usize>,
777784
options: &UnnestOptions,
778785
) -> Result<Option<RecordBatch>> {
779-
let transformed = if list_type_columns.is_empty() {
780-
flatten_struct_cols(batch.columns(), schema, struct_column_indices)
781-
} else {
782-
let mut temp_unnested_result = HashMap::new();
783-
let max_recursion = list_type_columns
784-
.iter()
785-
.map(|ListUnnest { depth, .. }| *depth)
786-
.max()
787-
.unwrap_or_default();
788-
let mut flatten_arrs = vec![];
789-
790-
for depth in (1..=max_recursion).rev() {
791-
let input = if depth == max_recursion {
792-
batch.columns()
793-
} else {
794-
&flatten_arrs
795-
};
796-
let Some(temp_result) = list_unnest_at_level(
797-
input,
798-
list_type_columns,
799-
&mut temp_unnested_result,
800-
depth,
801-
options,
802-
)?
803-
else {
804-
return Ok(None);
805-
};
806-
flatten_arrs = temp_result;
807-
}
786+
if list_type_columns.is_empty() {
787+
return flatten_struct_cols(batch.columns(), schema, struct_column_indices)
788+
.map(Some);
789+
}
808790

809-
let output_order: HashMap<ListUnnest, usize> = list_type_columns
810-
.iter()
811-
.enumerate()
812-
.map(|(order, unnest_def)| (*unnest_def, order))
813-
.collect();
814-
let mut multi_unnested_per_original_index: HashMap<usize, Vec<ArrayRef>> =
815-
HashMap::new();
816-
let mut unnested_columns = temp_unnested_result.into_iter().collect::<Vec<_>>();
817-
unnested_columns.sort_by_key(|(unnest, _)| {
818-
output_order.get(unnest).copied().unwrap_or_default()
819-
});
820-
for (unnest, arr) in unnested_columns {
821-
multi_unnested_per_original_index
822-
.entry(unnest.index_in_input_schema)
823-
.or_default()
824-
.push(arr);
825-
}
791+
let mut temp_unnested_result = HashMap::new();
792+
let max_recursion = list_type_columns
793+
.iter()
794+
.map(|ListUnnest { depth, .. }| *depth)
795+
.max()
796+
.unwrap_or_default();
797+
let mut flatten_arrs = vec![];
798+
799+
for depth in (1..=max_recursion).rev() {
800+
let input = if depth == max_recursion {
801+
batch.columns()
802+
} else {
803+
&flatten_arrs
804+
};
805+
let Some(temp_result) = list_unnest_at_level(
806+
input,
807+
list_type_columns,
808+
&mut temp_unnested_result,
809+
depth,
810+
options,
811+
)?
812+
else {
813+
return Ok(None);
814+
};
815+
flatten_arrs = temp_result;
816+
}
817+
818+
let output_order: HashMap<ListUnnest, usize> = list_type_columns
819+
.iter()
820+
.enumerate()
821+
.map(|(order, unnest_def)| (*unnest_def, order))
822+
.collect();
823+
let mut multi_unnested_per_original_index = HashMap::<usize, Vec<_>>::new();
824+
let mut unnested_columns = temp_unnested_result.into_iter().collect::<Vec<_>>();
825+
unnested_columns
826+
.sort_by_key(|(unnest, _)| output_order.get(unnest).copied().unwrap_or_default());
827+
for (unnest, arr) in unnested_columns {
828+
multi_unnested_per_original_index
829+
.entry(unnest.index_in_input_schema)
830+
.or_default()
831+
.push(arr);
832+
}
826833

827-
let ret = flatten_arrs
834+
flatten_struct_cols(
835+
&flatten_arrs
828836
.into_iter()
829837
.enumerate()
830838
.flat_map(|(col_idx, arr)| {
831839
multi_unnested_per_original_index
832840
.remove(&col_idx)
833841
.unwrap_or_else(|| vec![arr])
834842
})
835-
.collect::<Vec<_>>();
836-
837-
flatten_struct_cols(&ret, schema, struct_column_indices)
838-
}?;
839-
Ok(Some(transformed))
843+
.collect::<Vec<_>>(),
844+
schema,
845+
struct_column_indices,
846+
)
847+
.map(Some)
840848
}
841849

842850
/// Find the longest list length among the given list arrays for each row.

0 commit comments

Comments
 (0)