Skip to content

Commit 696233a

Browse files
authored
Feature: Add FixedSizeList fuzzing support (#4541)
Tracking Issue: #4372 Adds fuzzing functionality for fixed-size list. --------- Signed-off-by: Connor Tsui <[email protected]>
1 parent edf6849 commit 696233a

8 files changed

Lines changed: 45 additions & 35 deletions

File tree

‎fuzz/src/array/compare.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ pub fn compare_canonical_array(
107107
operator,
108108
)
109109
}),
110-
DType::Struct(..) | DType::List(..) => {
110+
DType::Struct(..) | DType::List(..) | DType::FixedSizeList(..) => {
111111
let scalar_vals: Vec<Scalar> = (0..array.len()).map(|i| array.scalar_at(i)).collect();
112112
Ok(BoolArray::from_iter(
113113
scalar_vals
@@ -116,7 +116,6 @@ pub fn compare_canonical_array(
116116
)
117117
.into_array())
118118
}
119-
DType::FixedSizeList(..) => unimplemented!("TODO(connor)[FixedSizeList]"),
120119
d @ (DType::Null | DType::Extension(_)) => {
121120
unreachable!("DType {d} not supported for fuzzing")
122121
}

‎fuzz/src/array/filter.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ pub fn filter_canonical_array(array: &dyn Array, filter: &[bool]) -> VortexResul
9999
)
100100
.map(|a| a.into_array())
101101
}
102-
DType::List(..) => {
102+
DType::List(..) | DType::FixedSizeList(..) => {
103103
let mut indices = Vec::new();
104104
for (idx, bool) in filter.iter().enumerate() {
105105
if *bool {
@@ -108,7 +108,6 @@ pub fn filter_canonical_array(array: &dyn Array, filter: &[bool]) -> VortexResul
108108
}
109109
take_canonical_array_non_nullable_indices(array, indices.as_slice())
110110
}
111-
DType::FixedSizeList(..) => unimplemented!("TODO(connor)[FixedSizeList]"),
112111
d @ (DType::Null | DType::Extension(_)) => {
113112
unreachable!("DType {d} not supported for fuzzing")
114113
}

‎fuzz/src/array/mod.rs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -215,17 +215,18 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction {
215215
}
216216

217217
fn actions_for_dtype(dtype: &DType) -> HashSet<usize> {
218+
// We exclude compare on the nested types.
219+
let nested_actions = [0, 1, 2, 3, 4, 6];
220+
218221
match dtype {
219222
DType::Struct(sdt, _) => sdt
220223
.fields()
221224
.map(|child| actions_for_dtype(&child))
222225
// exclude compare
223-
.fold((0..=4).chain(iter::once(6)).collect(), |acc, actions| {
226+
.fold(nested_actions.into(), |acc, actions| {
224227
acc.intersection(&actions).copied().collect()
225228
}),
226-
// Once we support more list operations also recurse here on child dtype
227-
// compress, slice
228-
DType::List(..) => [0, 1].into_iter().collect(),
229+
DType::List(..) | DType::FixedSizeList(..) => nested_actions.into(),
229230
_ => ALL_ACTIONS.collect(),
230231
}
231232
}

‎fuzz/src/array/search_sorted.rs‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -116,15 +116,10 @@ pub fn search_sorted_canonical_array(
116116
};
117117
Ok(SearchNullableSlice(opt_values).search_sorted(&Some(to_find), side))
118118
}
119-
DType::Struct(..) => {
119+
DType::Struct(..) | DType::List(..) | DType::FixedSizeList(..) => {
120120
let scalar_vals = (0..array.len()).map(|i| array.scalar_at(i)).collect_vec();
121121
Ok(scalar_vals.search_sorted(&scalar.cast(array.dtype())?, side))
122122
}
123-
DType::List(..) => {
124-
let scalar_vals = (0..array.len()).map(|i| array.scalar_at(i)).collect_vec();
125-
Ok(scalar_vals.search_sorted(&scalar.cast(array.dtype())?, side))
126-
}
127-
DType::FixedSizeList(..) => unimplemented!("TODO(connor)[FixedSizeList]"),
128123
d @ (DType::Null | DType::Extension(_)) => {
129124
unreachable!("DType {d} not supported for fuzzing")
130125
}

‎fuzz/src/array/slice.rs‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33

44
use vortex_array::accessor::ArrayAccessor;
55
use vortex_array::arrays::{
6-
BoolArray, DecimalArray, ListArray, PrimitiveArray, StructArray, VarBinViewArray,
6+
BoolArray, DecimalArray, FixedSizeListArray, ListArray, PrimitiveArray, StructArray,
7+
VarBinViewArray,
78
};
89
use vortex_array::validity::Validity;
910
use vortex_array::{Array, ArrayRef, IntoArray, ToCanonical};
@@ -86,7 +87,15 @@ pub fn slice_canonical_array(
8687
.into_array();
8788
ListArray::try_new(elements, offsets, validity).map(|a| a.into_array())
8889
}
89-
DType::FixedSizeList(..) => unimplemented!("TODO(connor)[FixedSizeList]"),
90+
DType::FixedSizeList(..) => {
91+
let fsl_array = array.to_fixed_size_list();
92+
let list_size = fsl_array.list_size() as usize;
93+
let elements =
94+
slice_canonical_array(fsl_array.elements(), start * list_size, stop * list_size)?;
95+
96+
FixedSizeListArray::try_new(elements, fsl_array.list_size(), validity, array.len())
97+
.map(|a| a.into_array())
98+
}
9099
DType::Decimal(decimal_dtype, _) => {
91100
let decimal_array = array.to_decimal();
92101
Ok(

‎fuzz/src/array/sort.rs‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ pub fn sort_canonical_array(array: &dyn Array) -> VortexResult<ArrayRef> {
6161
opt_values.sort();
6262
Ok(VarBinViewArray::from_iter(opt_values, array.dtype().clone()).into_array())
6363
}
64-
DType::Struct(..) => {
64+
DType::Struct(..) | DType::List(..) | DType::FixedSizeList(..) => {
6565
let mut sort_indices = (0..array.len()).collect::<Vec<_>>();
6666
sort_indices.sort_by(|a, b| {
6767
array
@@ -71,17 +71,6 @@ pub fn sort_canonical_array(array: &dyn Array) -> VortexResult<ArrayRef> {
7171
});
7272
take_canonical_array_non_nullable_indices(array, &sort_indices)
7373
}
74-
DType::List(..) => {
75-
let mut sort_indices = (0..array.len()).collect::<Vec<_>>();
76-
sort_indices.sort_by(|a, b| {
77-
array
78-
.scalar_at(*a)
79-
.partial_cmp(&array.scalar_at(*b))
80-
.vortex_expect("must be a valid comparison")
81-
});
82-
take_canonical_array_non_nullable_indices(array, &sort_indices)
83-
}
84-
DType::FixedSizeList(..) => unimplemented!("TODO(connor)[FixedSizeList]"),
8574
d @ (DType::Null | DType::Extension(_)) => {
8675
unreachable!("DType {d} not supported for fuzzing")
8776
}

‎fuzz/src/array/take.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ pub fn take_canonical_array(
113113
)
114114
.map(|a| a.into_array())
115115
}
116-
DType::List(..) => {
116+
DType::List(..) | DType::FixedSizeList(..) => {
117117
let mut builder = builder_with_capacity(array.dtype(), indices_slice_non_opt.len());
118118
for idx in indices {
119119
if let Some(idx) = idx {
@@ -124,7 +124,6 @@ pub fn take_canonical_array(
124124
}
125125
Ok(builder.finish())
126126
}
127-
DType::FixedSizeList(..) => unimplemented!("TODO(connor)[FixedSizeList]"),
128127
d @ (DType::Null | DType::Extension(_)) => {
129128
unreachable!("DType {d} not supported for fuzzing")
130129
}

‎vortex-array/src/arrow/convert.rs‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,9 @@ use arrow_array::types::{
1212
TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type,
1313
};
1414
use arrow_array::{
15-
Array as ArrowArray, ArrowPrimitiveType, BooleanArray as ArrowBooleanArray, GenericByteArray,
16-
GenericByteViewArray, GenericListArray, NullArray as ArrowNullArray, OffsetSizeTrait,
15+
Array as ArrowArray, ArrowPrimitiveType, BooleanArray as ArrowBooleanArray,
16+
FixedSizeListArray as ArrowFixedSizeListArray, GenericByteArray, GenericByteViewArray,
17+
GenericListArray, NullArray as ArrowNullArray, OffsetSizeTrait,
1718
PrimitiveArray as ArrowPrimitiveArray, RecordBatch, StructArray as ArrowStructArray,
1819
make_array,
1920
};
@@ -28,8 +29,8 @@ use vortex_error::{VortexExpect as _, vortex_panic};
2829
use vortex_scalar::i256;
2930

3031
use crate::arrays::{
31-
BoolArray, DecimalArray, ListArray, NullArray, PrimitiveArray, StructArray, TemporalArray,
32-
VarBinArray, VarBinViewArray,
32+
BoolArray, DecimalArray, FixedSizeListArray, ListArray, NullArray, PrimitiveArray, StructArray,
33+
TemporalArray, VarBinArray, VarBinViewArray,
3334
};
3435
use crate::arrow::FromArrowArray;
3536
use crate::validity::Validity;
@@ -325,7 +326,24 @@ impl<O: OffsetSizeTrait + NativePType> FromArrowArray<&GenericListArray<O>> for
325326
value.offsets().clone().into_array(),
326327
nulls(value.nulls(), nullable),
327328
)
328-
.vortex_expect("Failed to convert Arrow StructArray to Vortex StructArray")
329+
.vortex_expect("Failed to convert Arrow ListArray to Vortex ListArray")
330+
.into_array()
331+
}
332+
}
333+
334+
impl FromArrowArray<&ArrowFixedSizeListArray> for ArrayRef {
335+
fn from_arrow(array: &ArrowFixedSizeListArray, nullable: bool) -> Self {
336+
let DataType::FixedSizeList(field, list_size) = array.data_type() else {
337+
vortex_panic!("Invalid data type for ListArray: {}", array.data_type());
338+
};
339+
340+
FixedSizeListArray::try_new(
341+
Self::from_arrow(array.values().as_ref(), field.is_nullable()),
342+
*list_size as u32,
343+
nulls(array.nulls(), nullable),
344+
array.len(),
345+
)
346+
.vortex_expect("Failed to convert Arrow FixedSizeListArray to Vortex FixedSizeListArray")
329347
.into_array()
330348
}
331349
}
@@ -378,6 +396,7 @@ impl FromArrowArray<&dyn ArrowArray> for ArrayRef {
378396
DataType::Struct(_) => Self::from_arrow(array.as_struct(), nullable),
379397
DataType::List(_) => Self::from_arrow(array.as_list::<i32>(), nullable),
380398
DataType::LargeList(_) => Self::from_arrow(array.as_list::<i64>(), nullable),
399+
DataType::FixedSizeList(..) => Self::from_arrow(array.as_fixed_size_list(), nullable),
381400
DataType::Null => Self::from_arrow(as_null_array(array), nullable),
382401
DataType::Timestamp(u, _) => match u {
383402
ArrowTimeUnit::Second => {

0 commit comments

Comments
 (0)