Skip to content

Commit 8912578

Browse files
committed
Integrate fixed-size binary support
Signed-off-by: Connor Tsui <[email protected]>
1 parent 49c1cad commit 8912578

37 files changed

Lines changed: 406 additions & 42 deletions

File tree

‎encodings/sparse/src/canonical.rs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ use vortex_array::ArrayRef;
99
use vortex_array::ExecutionCtx;
1010
use vortex_array::IntoArray;
1111
use vortex_array::arrays::BoolArray;
12+
use vortex_array::arrays::FixedSizeBinary;
13+
use vortex_array::arrays::FixedSizeBinaryArray;
1214
use vortex_array::arrays::FixedSizeList;
1315
use vortex_array::arrays::FixedSizeListArray;
1416
use vortex_array::arrays::ListView;
@@ -20,6 +22,7 @@ use vortex_array::arrays::Struct;
2022
use vortex_array::arrays::StructArray;
2123
use vortex_array::arrays::VarBinView;
2224
use vortex_array::arrays::VarBinViewArray;
25+
use vortex_array::arrays::fixed_size_binary::FixedSizeBinaryArrayExt;
2326
use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt;
2427
use vortex_array::arrays::listview::ListViewArrayExt;
2528
use vortex_array::arrays::listview::ListViewArraySlotsExt;
@@ -56,6 +59,7 @@ use vortex_buffer::BitBuffer;
5659
use vortex_buffer::Buffer;
5760
use vortex_buffer::BufferString;
5861
use vortex_buffer::ByteBuffer;
62+
use vortex_buffer::ByteBufferMut;
5963
use vortex_buffer::buffer;
6064
use vortex_buffer::buffer_mut;
6165
use vortex_error::VortexError;
@@ -142,6 +146,14 @@ pub(super) fn execute_sparse(parts: SparseParts, ctx: &mut ExecutionCtx) -> Vort
142146
let fill = fill_value.as_binary().value().cloned();
143147
execute_varbin(&patches, &fill_value, dtype.clone(), fill, len, ctx)?
144148
}
149+
DType::FixedSizeBinary(byte_width, nullability) => execute_sparse_fixed_size_binary(
150+
&patches,
151+
&fill_value,
152+
*byte_width,
153+
*nullability,
154+
len,
155+
ctx,
156+
)?,
145157
DType::List(values_dtype, nullability) => execute_sparse_lists(
146158
&patches,
147159
&fill_value,
@@ -168,6 +180,43 @@ pub(super) fn execute_sparse(parts: SparseParts, ctx: &mut ExecutionCtx) -> Vort
168180
})
169181
}
170182

183+
fn execute_sparse_fixed_size_binary(
184+
resolved: &Patches,
185+
fill_scalar: &Scalar,
186+
byte_width: u32,
187+
nullability: Nullability,
188+
len: usize,
189+
ctx: &mut ExecutionCtx,
190+
) -> VortexResult<ArrayRef> {
191+
let byte_width_usize = byte_width as usize;
192+
let fill = fill_scalar
193+
.as_binary()
194+
.value()
195+
.cloned()
196+
.unwrap_or_else(|| ByteBuffer::zeroed(byte_width_usize));
197+
let mut dense = ByteBufferMut::with_capacity(len.saturating_mul(byte_width_usize));
198+
for _ in 0..len {
199+
dense.extend_from_slice(fill.as_slice());
200+
}
201+
202+
let indices = resolved.indices().as_::<Primitive>().into_owned();
203+
let values = resolved.values().as_::<FixedSizeBinary>().into_owned();
204+
let patch_bytes = values.buffer_handle().to_host_sync();
205+
match_each_integer_ptype!(indices.ptype(), |I| {
206+
for (patch_row, patch_index) in indices.as_slice::<I>().iter().enumerate() {
207+
let patch_index = <usize as NumCast>::from(*patch_index)
208+
.vortex_expect("fixed-size binary patch index must fit in usize");
209+
let source = patch_row * byte_width_usize;
210+
let target = patch_index * byte_width_usize;
211+
dense[target..target + byte_width_usize]
212+
.copy_from_slice(&patch_bytes[source..source + byte_width_usize]);
213+
}
214+
});
215+
216+
let validity = sparse_validity(resolved, fill_scalar, nullability, len, ctx)?;
217+
Ok(FixedSizeBinaryArray::new(dense.freeze(), byte_width, len, validity).into_array())
218+
}
219+
171220
fn execute_sparse_lists(
172221
resolved: &Patches,
173222
fill_value: &Scalar,

‎fuzz/src/array/compare.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,10 @@ pub fn compare_canonical_array(
174174
result_nullability,
175175
)
176176
}
177-
DType::List(..) | DType::FixedSizeList(..) | DType::Struct(..) => {
177+
DType::FixedSizeBinary(..)
178+
| DType::List(..)
179+
| DType::FixedSizeList(..)
180+
| DType::Struct(..) => {
178181
let scalar_vals: Vec<Scalar> = (0..array.len())
179182
.map(|i| array.execute_scalar(i, ctx).vortex_expect("scalar_at"))
180183
.collect();

‎fuzz/src/array/fill_null.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ pub fn fill_null_canonical_array(
4242
fill_primitive_array(array, fill_value, result_nullability, ctx)
4343
}
4444
Canonical::Decimal(array) => fill_decimal_array(array, fill_value, result_nullability, ctx),
45+
Canonical::FixedSizeBinary(array) => array.into_array().fill_null(fill_value.clone())?,
4546
Canonical::VarBinView(array) => {
4647
fill_varbinview_array(array, fill_value, result_nullability, ctx)
4748
}

‎fuzz/src/array/filter.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ pub fn filter_canonical_array(
9898
.collect::<Vec<_>>();
9999
Ok(VarBinViewArray::from_iter(values, array.dtype().clone()).into_array())
100100
}
101-
DType::List(..) | DType::FixedSizeList(..) => {
101+
DType::FixedSizeBinary(..) | DType::List(..) | DType::FixedSizeList(..) => {
102102
let mut indices = Vec::new();
103103
for (idx, bool) in filter.iter().enumerate() {
104104
if *bool {

‎fuzz/src/array/mask.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ use vortex_array::arrays::StructArray;
1717
use vortex_array::arrays::VarBinViewArray;
1818
use vortex_array::arrays::bool::BoolArrayExt;
1919
use vortex_array::arrays::extension::ExtensionArrayExt;
20+
use vortex_array::arrays::fixed_size_binary::FixedSizeBinaryArrayExt;
2021
use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt;
2122
use vortex_array::arrays::fixed_size_list::FixedSizeListArraySlotsExt;
2223
use vortex_array::arrays::listview::ListViewArraySlotsExt;
@@ -93,6 +94,16 @@ pub fn mask_canonical_array(
9394
.into_array()
9495
})
9596
}
97+
Canonical::FixedSizeBinary(array) => {
98+
let new_validity = mask_validity(&array.validity()?, mask, ctx);
99+
vortex_array::arrays::FixedSizeBinaryArray::new(
100+
array.buffer_handle().to_host_sync(),
101+
array.byte_width(),
102+
array.len(),
103+
new_validity,
104+
)
105+
.into_array()
106+
}
96107
Canonical::VarBinView(array) => {
97108
let new_validity = mask_validity(&array.validity()?, mask, ctx);
98109
VarBinViewArray::new_handle(

‎fuzz/src/array/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -486,7 +486,7 @@ fn actions_for_dtype(dtype: &DType) -> HashSet<ActionType> {
486486
// These support all actions
487487
ActionType::iter().collect()
488488
}
489-
DType::Utf8(_) | DType::Binary(_) => {
489+
DType::Utf8(_) | DType::Binary(_) | DType::FixedSizeBinary(..) => {
490490
// Utf8/Binary supports everything except Sum and FillNull
491491
// Actions: Compress, Slice, Take, SearchSorted, Filter, Compare, Cast, MinMax, Mask, ScalarAt
492492
[

‎fuzz/src/array/scalar_at.rs‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ use vortex_array::ExecutionCtx;
88
use vortex_array::IntoArray;
99
use vortex_array::arrays::bool::BoolArrayExt;
1010
use vortex_array::arrays::extension::ExtensionArrayExt;
11+
use vortex_array::arrays::fixed_size_binary::FixedSizeBinaryArrayExt;
1112
use vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt;
1213
use vortex_array::arrays::listview::ListViewArrayExt;
1314
use vortex_array::arrays::struct_::StructArrayExt;
@@ -38,19 +39,18 @@ pub fn scalar_at_canonical_array(
3839
array.to_bit_buffer().value(index),
3940
array.dtype().nullability(),
4041
),
41-
Canonical::Primitive(array) => {
42-
match_each_native_ptype!(array.ptype(), |T| {
43-
Scalar::primitive(array.as_slice::<T>()[index], array.dtype().nullability())
44-
})
45-
}
46-
Canonical::Decimal(array) => {
47-
match_each_decimal_value_type!(array.values_type(), |D| {
48-
Scalar::decimal(
49-
DecimalValue::from(array.buffer::<D>()[index]),
50-
array.decimal_dtype(),
51-
array.dtype().nullability(),
52-
)
53-
})
42+
Canonical::Primitive(array) => match_each_native_ptype!(array.ptype(), |T| {
43+
Scalar::primitive(array.as_slice::<T>()[index], array.dtype().nullability())
44+
}),
45+
Canonical::Decimal(array) => match_each_decimal_value_type!(array.values_type(), |D| {
46+
Scalar::decimal(
47+
DecimalValue::from(array.buffer::<D>()[index]),
48+
array.decimal_dtype(),
49+
array.dtype().nullability(),
50+
)
51+
}),
52+
Canonical::FixedSizeBinary(array) => {
53+
Scalar::fixed_size_binary(array.value(index), array.dtype().nullability())
5454
}
5555
Canonical::VarBinView(array) => varbin_scalar(array.bytes_at(index), array.dtype()),
5656
Canonical::List(array) => {

‎fuzz/src/array/search_sorted.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,10 @@ pub fn search_sorted_canonical_array(
143143
};
144144
SearchNullableSlice(opt_values).search_sorted(&Some(to_find), side)
145145
}
146-
DType::List(..) | DType::FixedSizeList(..) | DType::Struct(..) => {
146+
DType::FixedSizeBinary(..)
147+
| DType::List(..)
148+
| DType::FixedSizeList(..)
149+
| DType::Struct(..) => {
147150
let scalar_vals = (0..array.len())
148151
.map(|i| array.execute_scalar(i, ctx))
149152
.collect::<VortexResult<Vec<_>>>()?;

‎fuzz/src/array/slice.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ pub fn slice_canonical_array(
126126
)
127127
.map(|a| a.into_array())
128128
}
129-
DType::Map(..) => {
129+
DType::Map(..) | DType::FixedSizeBinary(..) => {
130130
let mut builder = builder_with_capacity(array.dtype(), stop - start);
131131
for idx in start..stop {
132132
builder.append_scalar(&array.execute_scalar(idx, ctx)?)?;

‎fuzz/src/array/sort.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,10 @@ pub fn sort_canonical_array(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexR
9292
opt_values.sort();
9393
Ok(VarBinViewArray::from_iter(opt_values, array.dtype().clone()).into_array())
9494
}
95-
DType::List(..) | DType::FixedSizeList(..) | DType::Struct(..) => {
95+
DType::FixedSizeBinary(..)
96+
| DType::List(..)
97+
| DType::FixedSizeList(..)
98+
| DType::Struct(..) => {
9699
let mut sort_indices = (0..array.len()).collect::<Vec<_>>();
97100
sort_indices.sort_by(|a, b| {
98101
let lhs = array.execute_scalar(*a, ctx).vortex_expect("scalar_at");

0 commit comments

Comments
 (0)