Skip to content

Commit c701d04

Browse files
committed
fix: impl values_preserving for GroupColumn
1 parent 4c05ca2 commit c701d04

1 file changed

Lines changed: 108 additions & 0 deletions

File tree

  • datafusion/physical-plan/src/aggregates/group_values/multi_group_by

‎datafusion/physical-plan/src/aggregates/group_values/multi_group_by/list.rs‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ use arrow::datatypes::FieldRef;
3939
use datafusion_common::utils::split_vec_min_alloc;
4040
use datafusion_common::{Result, internal_datafusion_err};
4141
use datafusion_execution::memory_pool::proxy::VecAllocExt;
42+
use datafusion_expr::GroupSelection;
4243
use std::sync::Arc;
4344

4445
/// A [`GroupColumn`] for `List<T>` (`O = i32`) and `LargeList<T>` (`O = i64`).
@@ -209,6 +210,55 @@ impl<O: OffsetSizeTrait> GroupColumn for ListGroupValueBuilder<O> {
209210
))
210211
}
211212

213+
fn values_preserving(&self, selection: GroupSelection<'_>) -> Result<ArrayRef> {
214+
selection.validate_num_groups(self.outer_len)?;
215+
216+
// Flatten the selected outer rows into the child positions they cover,
217+
// rebuilding offsets for the output. Null (and empty) rows contribute a
218+
// zero-length range, matching how they are stored.
219+
let mut child_indices = Vec::new();
220+
let mut offsets = Vec::with_capacity(selection.len() + 1);
221+
let mut nulls = NullBufferBuilder::new(selection.len());
222+
let mut end = O::usize_as(0);
223+
offsets.push(end);
224+
225+
for row in selection.iter() {
226+
nulls.append(!self.outer_nulls.is_null(row));
227+
let start = self.offsets[row].as_usize();
228+
let row_end = self.offsets[row + 1].as_usize();
229+
child_indices.extend(start..row_end);
230+
let len = O::from_usize(row_end - start).ok_or_else(|| {
231+
internal_datafusion_err!(
232+
"List sublist length {} overflows the offset type",
233+
row_end - start
234+
)
235+
})?;
236+
end = end.checked_add(&len).ok_or_else(|| {
237+
internal_datafusion_err!(
238+
"List offset overflow: current={} additional={}",
239+
end.as_usize(),
240+
len.as_usize()
241+
)
242+
})?;
243+
offsets.push(end);
244+
}
245+
246+
let child_selection =
247+
GroupSelection::try_from_indices(&child_indices, self.child.len())?;
248+
let child_array = self.child.values_preserving(child_selection)?;
249+
250+
// SAFETY: offsets start at 0 and grow monotonically by each selected
251+
// row's length, and `child_array` has exactly `end` elements.
252+
let offset_buffer =
253+
unsafe { OffsetBuffer::<O>::new_unchecked(ScalarBuffer::<O>::from(offsets)) };
254+
Ok(Arc::new(GenericListArray::<O>::new(
255+
Arc::clone(&self.field),
256+
offset_buffer,
257+
child_array,
258+
nulls.build(),
259+
)))
260+
}
261+
212262
fn take_n(&mut self, n: usize) -> ArrayRef {
213263
// Number of child elements consumed by the first n outer rows.
214264
let cut_offset = self.offsets[n];
@@ -735,6 +785,64 @@ mod tests {
735785
assert!(!bb.get_bit(0), "pre-set false stays false");
736786
}
737787

788+
#[test]
789+
fn values_preserving_selects_reorders_and_repeats() {
790+
let input = list_array(&[
791+
Some(vec![Some(1), Some(2)]),
792+
None,
793+
Some(vec![]),
794+
Some(vec![Some(3), Some(4), Some(5)]),
795+
]);
796+
let mut builder = ListGroupValueBuilder::<i32>::new(child_field(), make_child());
797+
for i in 0..input.len() {
798+
builder.append_val(&input, i).unwrap();
799+
}
800+
801+
let selection = GroupSelection::try_from_indices(&[3, 1, 0, 3, 2], 4).unwrap();
802+
let out = builder.values_preserving(selection).unwrap();
803+
let out = out.as_any().downcast_ref::<ListArray>().unwrap();
804+
assert_eq!(out.len(), 5);
805+
assert!(out.is_null(1), "selected null row stays null");
806+
let v = |i: usize| -> Vec<i32> {
807+
let v = out.value(i);
808+
v.as_any()
809+
.downcast_ref::<Int32Array>()
810+
.unwrap()
811+
.values()
812+
.to_vec()
813+
};
814+
assert_eq!(v(0), vec![3, 4, 5]);
815+
assert_eq!(v(2), vec![1, 2]);
816+
assert_eq!(v(3), vec![3, 4, 5], "repeated index materialized twice");
817+
assert_eq!(out.value_length(4), 0, "empty list preserved");
818+
819+
// The builder is unchanged by a preserving read.
820+
assert_eq!(builder.len(), 4);
821+
let all = builder.values_preserving(GroupSelection::all(4)).unwrap();
822+
assert_eq!(all.len(), 4);
823+
let built = Box::new(builder).build();
824+
assert_eq!(built.as_ref(), all.as_ref());
825+
}
826+
827+
#[test]
828+
fn values_preserving_empty_selection() {
829+
let mut builder = ListGroupValueBuilder::<i64>::new(child_field(), make_child());
830+
let input = large_list_array(&[Some(vec![Some(1)])]);
831+
builder.append_val(&input, 0).unwrap();
832+
let out = builder
833+
.values_preserving(GroupSelection::try_from_indices(&[], 1).unwrap())
834+
.unwrap();
835+
assert_eq!(out.len(), 0);
836+
}
837+
838+
#[test]
839+
fn values_preserving_rejects_mismatched_group_count() {
840+
let input = list_array(&[Some(vec![Some(1)])]);
841+
let mut builder = ListGroupValueBuilder::<i32>::new(child_field(), make_child());
842+
builder.append_val(&input, 0).unwrap();
843+
assert!(builder.values_preserving(GroupSelection::all(2)).is_err());
844+
}
845+
738846
#[test]
739847
fn size_grows_with_appends() {
740848
let input = list_array(&[Some(vec![Some(1), Some(2), Some(3)])]);

0 commit comments

Comments
 (0)