Skip to content

Commit b2bd758

Browse files
committed
feat(compress): keep signedness when narrowing internal children
Signed-off-by: Connor Tsui <[email protected]>
1 parent 83066fe commit b2bd758

25 files changed

Lines changed: 305 additions & 378 deletions

File tree

‎encodings/fastlanes/src/rle/array/mod.rs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ mod tests {
125125
use vortex_array::IntoArray;
126126
use vortex_array::VortexSessionExecute;
127127
use vortex_array::arrays::PrimitiveArray;
128-
use vortex_array::arrays::primitive::PrimitiveArrayExt;
128+
use vortex_array::arrays::WidenArray;
129129
use vortex_array::assert_arrays_eq;
130130
use vortex_array::dtype::DType;
131131
use vortex_array::dtype::Nullability;
@@ -435,11 +435,12 @@ mod tests {
435435

436436
// Simulate cascading compression: narrow u16->u8 then re-encode with RLE,
437437
// matching the path taken by the BtrBlocks compressor.
438-
let indices_prim = rle
439-
.indices()
440-
.clone()
441-
.execute::<PrimitiveArray>(&mut ctx)?
442-
.narrow(&mut ctx)?;
438+
let indices_prim = rle.indices().clone().execute::<PrimitiveArray>(&mut ctx)?;
439+
#[expect(
440+
deprecated,
441+
reason = "TODO(connor): decide whether this child narrows its dtype or becomes a Widen"
442+
)]
443+
let indices_prim = WidenArray::encode_values(indices_prim, &mut ctx)?;
443444
let re_encoded = RLEData::encode(indices_prim.as_view(), &mut ctx)?;
444445

445446
// Reconstruct the outer RLE with re-encoded indices.

‎vortex-array/src/arrays/primitive/array/cast.rs‎

Lines changed: 0 additions & 177 deletions
Original file line numberDiff line numberDiff line change
@@ -34,180 +34,3 @@ impl PrimitiveData {
3434
unsafe { std::slice::from_raw_parts(raw_slice.cast(), byte_buffer.len() / size_of::<T>()) }
3535
}
3636
}
37-
38-
#[cfg(test)]
39-
mod tests {
40-
use std::sync::LazyLock;
41-
42-
use rstest::rstest;
43-
use vortex_buffer::Buffer;
44-
use vortex_buffer::buffer;
45-
use vortex_session::VortexSession;
46-
47-
use crate::VortexSessionExecute;
48-
use crate::arrays::PrimitiveArray;
49-
use crate::arrays::primitive::PrimitiveArrayExt;
50-
use crate::assert_arrays_eq;
51-
use crate::dtype::DType;
52-
use crate::dtype::Nullability;
53-
use crate::dtype::PType;
54-
use crate::validity::Validity;
55-
56-
static SESSION: LazyLock<VortexSession> = LazyLock::new(VortexSession::empty);
57-
58-
#[test]
59-
fn test_downcast_all_invalid() {
60-
let array = PrimitiveArray::new(
61-
buffer![0_u32, 0, 0, 0, 0, 0, 0, 0, 0, 0],
62-
Validity::AllInvalid,
63-
);
64-
65-
let result = array.narrow(&mut SESSION.create_execution_ctx()).unwrap();
66-
assert_eq!(
67-
result.dtype(),
68-
&DType::Primitive(PType::U8, Nullability::Nullable)
69-
);
70-
assert!(matches!(result.validity(), Ok(Validity::AllInvalid)));
71-
assert_arrays_eq!(
72-
result,
73-
PrimitiveArray::from_option_iter([None::<u8>; 10]),
74-
&mut SESSION.create_execution_ctx()
75-
);
76-
}
77-
78-
#[rstest]
79-
#[case(vec![0_i64, 127], PType::U8)]
80-
#[case(vec![-128_i64, 127], PType::I8)]
81-
#[case(vec![-129_i64, 127], PType::I16)]
82-
#[case(vec![-128_i64, 128], PType::I16)]
83-
#[case(vec![-32768_i64, 32767], PType::I16)]
84-
#[case(vec![-32769_i64, 32767], PType::I32)]
85-
#[case(vec![-32768_i64, 32768], PType::I32)]
86-
#[case(vec![i32::MIN as i64, i32::MAX as i64], PType::I32)]
87-
fn test_downcast_signed(#[case] values: Vec<i64>, #[case] expected_ptype: PType) {
88-
let array = PrimitiveArray::from_iter(values.iter().copied());
89-
let result = array.narrow(&mut SESSION.create_execution_ctx()).unwrap();
90-
assert_eq!(
91-
result.dtype(),
92-
&DType::Primitive(expected_ptype, Nullability::NonNullable)
93-
);
94-
assert_eq!(result.len(), values.len());
95-
let mut ctx = SESSION.create_execution_ctx();
96-
for (index, expected) in values.into_iter().enumerate() {
97-
assert_eq!(
98-
result
99-
.execute_scalar(index, &mut ctx)
100-
.unwrap()
101-
.as_primitive()
102-
.as_::<i64>(),
103-
Some(expected)
104-
);
105-
}
106-
}
107-
108-
#[rstest]
109-
#[case(vec![0_u64, 255], PType::U8)]
110-
#[case(vec![0_u64, 256], PType::U16)]
111-
#[case(vec![0_u64, 65535], PType::U16)]
112-
#[case(vec![0_u64, 65536], PType::U32)]
113-
#[case(vec![0_u64, u32::MAX as u64], PType::U32)]
114-
fn test_downcast_unsigned(#[case] values: Vec<u64>, #[case] expected_ptype: PType) {
115-
let array = PrimitiveArray::from_iter(values.iter().copied());
116-
let result = array.narrow(&mut SESSION.create_execution_ctx()).unwrap();
117-
assert_eq!(
118-
result.dtype(),
119-
&DType::Primitive(expected_ptype, Nullability::NonNullable)
120-
);
121-
assert_eq!(result.len(), values.len());
122-
let mut ctx = SESSION.create_execution_ctx();
123-
for (index, expected) in values.into_iter().enumerate() {
124-
assert_eq!(
125-
result
126-
.execute_scalar(index, &mut ctx)
127-
.unwrap()
128-
.as_primitive()
129-
.as_::<u64>(),
130-
Some(expected)
131-
);
132-
}
133-
}
134-
135-
#[test]
136-
fn test_downcast_keeps_original_if_too_large() {
137-
let array = PrimitiveArray::from_iter(vec![0_u64, u64::MAX]);
138-
let result = array.narrow(&mut SESSION.create_execution_ctx()).unwrap();
139-
assert_eq!(result.ptype(), PType::U64);
140-
assert_arrays_eq!(
141-
result,
142-
PrimitiveArray::from_iter([0u64, u64::MAX]),
143-
&mut SESSION.create_execution_ctx()
144-
);
145-
}
146-
147-
#[test]
148-
fn test_downcast_preserves_nullability() {
149-
let array = PrimitiveArray::from_option_iter([Some(0_i32), None, Some(127)]);
150-
let result = array.narrow(&mut SESSION.create_execution_ctx()).unwrap();
151-
assert_eq!(
152-
result.dtype(),
153-
&DType::Primitive(PType::U8, Nullability::Nullable)
154-
);
155-
// Check that validity is preserved (the array should still have nullable values)
156-
assert!(matches!(result.validity(), Ok(Validity::Array(_))));
157-
assert_arrays_eq!(
158-
result,
159-
PrimitiveArray::from_option_iter([Some(0u8), None, Some(127)]),
160-
&mut SESSION.create_execution_ctx()
161-
);
162-
}
163-
164-
#[test]
165-
fn test_downcast_preserves_values() {
166-
let values = vec![-100_i16, 0, 100];
167-
let array = PrimitiveArray::from_iter(values);
168-
let result = array.narrow(&mut SESSION.create_execution_ctx()).unwrap();
169-
170-
assert_eq!(result.ptype(), PType::I8);
171-
// Check that the values were properly downscaled
172-
let downscaled_values: Vec<i8> = result.as_slice::<i8>().to_vec();
173-
assert_eq!(downscaled_values, vec![-100_i8, 0, 100]);
174-
}
175-
176-
#[test]
177-
fn test_downcast_with_mixed_signs_chooses_signed() {
178-
let array = PrimitiveArray::from_iter(vec![-1_i32, 200]);
179-
let result = array.narrow(&mut SESSION.create_execution_ctx()).unwrap();
180-
assert_eq!(result.ptype(), PType::I16);
181-
assert_arrays_eq!(
182-
result,
183-
PrimitiveArray::from_iter([-1i16, 200]),
184-
&mut SESSION.create_execution_ctx()
185-
);
186-
}
187-
188-
#[test]
189-
fn test_downcast_floats() {
190-
let array = PrimitiveArray::from_iter(vec![1.0_f32, 2.0, 3.0]);
191-
let result = array.narrow(&mut SESSION.create_execution_ctx()).unwrap();
192-
// Floats should remain unchanged since they can't be downscaled to integers
193-
assert_eq!(result.ptype(), PType::F32);
194-
assert_arrays_eq!(
195-
result,
196-
PrimitiveArray::from_iter([1f32, 2., 3.]),
197-
&mut SESSION.create_execution_ctx()
198-
);
199-
}
200-
201-
#[test]
202-
fn test_downcast_empty_array() {
203-
let array = PrimitiveArray::new(Buffer::<i32>::empty(), Validity::AllInvalid);
204-
let result = array.narrow(&mut SESSION.create_execution_ctx()).unwrap();
205-
let array2 = PrimitiveArray::new(Buffer::<i64>::empty(), Validity::NonNullable);
206-
let result2 = array2.narrow(&mut SESSION.create_execution_ctx()).unwrap();
207-
assert!(result.is_empty());
208-
assert!(result2.is_empty());
209-
// Empty arrays should not have their validity changed
210-
assert!(matches!(result.validity(), Ok(Validity::AllInvalid)));
211-
assert!(matches!(result2.validity(), Ok(Validity::NonNullable)));
212-
}
213-
}

‎vortex-array/src/arrays/primitive/array/mod.rs‎

Lines changed: 0 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,11 @@ pub use patch::chunk_range;
3939
pub use patch::patch_chunk;
4040

4141
use crate::ArrayRef;
42-
use crate::aggregate_fn::NumericalAggregateOpts;
43-
use crate::aggregate_fn::fns::min_max::min_max;
4442
use crate::array::child_to_validity;
4543
use crate::array::validity_to_child;
4644
use crate::array_slots;
4745
use crate::arrays::bool::BoolArrayExt;
4846
use crate::buffer::BufferHandle;
49-
use crate::builtins::ArrayBuiltins;
5047

5148
#[array_slots(Primitive)]
5249
pub struct PrimitiveSlots {
@@ -153,93 +150,6 @@ pub trait PrimitiveArrayExt: TypedArrayRef<Primitive> {
153150
PrimitiveArrayExt::validity(self),
154151
)
155152
}
156-
157-
/// Narrow the array to the smallest possible integer type that can represent all values.
158-
fn narrow(&self, ctx: &mut ExecutionCtx) -> VortexResult<PrimitiveArray> {
159-
if !self.ptype().is_int() {
160-
return Ok(self.to_owned());
161-
}
162-
163-
let Some(min_max) = min_max(self.as_ref(), ctx, NumericalAggregateOpts::default())? else {
164-
return Ok(PrimitiveArray::new(
165-
Buffer::<u8>::zeroed(self.len()),
166-
PrimitiveArrayExt::validity(self),
167-
));
168-
};
169-
170-
// If we can't cast to i64, then leave the array as its original type.
171-
// It's too big to downcast anyway.
172-
let Ok(min) = min_max
173-
.min
174-
.cast(&PType::I64.into())
175-
.and_then(|s| i64::try_from(&s))
176-
else {
177-
return Ok(self.to_owned());
178-
};
179-
let Ok(max) = min_max
180-
.max
181-
.cast(&PType::I64.into())
182-
.and_then(|s| i64::try_from(&s))
183-
else {
184-
return Ok(self.to_owned());
185-
};
186-
187-
let nullability = self.dtype().nullability();
188-
189-
if min < 0 || max < 0 {
190-
// Signed
191-
if min >= i8::MIN as i64 && max <= i8::MAX as i64 {
192-
let result = self
193-
.as_ref()
194-
.cast(DType::Primitive(PType::I8, nullability))?
195-
.execute::<PrimitiveArray>(ctx)?;
196-
return Ok(result);
197-
}
198-
199-
if min >= i16::MIN as i64 && max <= i16::MAX as i64 {
200-
let result = self
201-
.as_ref()
202-
.cast(DType::Primitive(PType::I16, nullability))?
203-
.execute::<PrimitiveArray>(ctx)?;
204-
return Ok(result);
205-
}
206-
207-
if min >= i32::MIN as i64 && max <= i32::MAX as i64 {
208-
let result = self
209-
.as_ref()
210-
.cast(DType::Primitive(PType::I32, nullability))?
211-
.execute::<PrimitiveArray>(ctx)?;
212-
return Ok(result);
213-
}
214-
} else {
215-
// Unsigned
216-
if max <= u8::MAX as i64 {
217-
let result = self
218-
.as_ref()
219-
.cast(DType::Primitive(PType::U8, nullability))?
220-
.execute::<PrimitiveArray>(ctx)?;
221-
return Ok(result);
222-
}
223-
224-
if max <= u16::MAX as i64 {
225-
let result = self
226-
.as_ref()
227-
.cast(DType::Primitive(PType::U16, nullability))?
228-
.execute::<PrimitiveArray>(ctx)?;
229-
return Ok(result);
230-
}
231-
232-
if max <= u32::MAX as i64 {
233-
let result = self
234-
.as_ref()
235-
.cast(DType::Primitive(PType::U32, nullability))?
236-
.execute::<PrimitiveArray>(ctx)?;
237-
return Ok(result);
238-
}
239-
}
240-
241-
Ok(self.to_owned())
242-
}
243153
}
244154
impl<T: TypedArrayRef<Primitive>> PrimitiveArrayExt for T {}
245155

‎vortex-array/src/arrays/runend/compress.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ use crate::arrays::DecimalArray;
2222
use crate::arrays::Primitive;
2323
use crate::arrays::PrimitiveArray;
2424
use crate::arrays::VarBinViewArray;
25+
use crate::arrays::WidenArray;
2526
use crate::arrays::bool::BoolArrayExt;
2627
use crate::arrays::decimal::DecimalArrayExt;
27-
use crate::arrays::primitive::PrimitiveArrayExt;
2828
use crate::arrays::runend::iter::trimmed_ends_iter;
2929
use crate::buffer::BufferHandle;
3030
use crate::dtype::NativePType;
@@ -88,9 +88,11 @@ pub fn runend_encode(
8888
}
8989
};
9090

91-
let ends = ends
92-
.narrow(ctx)
93-
.vortex_expect("Ends must succeed downcasting");
91+
#[expect(
92+
deprecated,
93+
reason = "TODO(connor): decide whether this child narrows its dtype or becomes a Widen"
94+
)]
95+
let ends = WidenArray::encode_values(ends, ctx).vortex_expect("Ends must succeed downcasting");
9496

9597
ends.statistics()
9698
.set(Stat::IsStrictSorted, Precision::Exact(true.into()));

‎vortex-array/src/builders/dict/mod.rs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ use crate::arrays::Primitive;
1616
use crate::arrays::PrimitiveArray;
1717
use crate::arrays::VarBin;
1818
use crate::arrays::VarBinView;
19-
use crate::arrays::primitive::PrimitiveArrayExt;
19+
use crate::arrays::WidenArray;
2020
use crate::dtype::PType;
2121
use crate::match_each_native_ptype;
2222

@@ -84,7 +84,12 @@ pub fn dict_encode_with_constraints(
8484
ctx: &mut ExecutionCtx,
8585
) -> VortexResult<DictArray> {
8686
let mut encoder = dict_encoder_in(array, constraints, ctx.allocator().clone());
87-
let codes = encoder.encode(array, ctx)?.narrow(ctx)?;
87+
let codes = encoder.encode(array, ctx)?;
88+
#[expect(
89+
deprecated,
90+
reason = "TODO(connor): decide whether this child narrows its dtype or becomes a Widen"
91+
)]
92+
let codes = WidenArray::encode_values(codes, ctx)?;
8893
// SAFETY: The encoding process will produce a value set of codes and values
8994
// All values in the dictionary are guaranteed to be referenced by at least one code
9095
// since we build the dictionary from the codes we observe during encoding

‎vortex-btrblocks/src/schemes/binary/varbin.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ use vortex_array::VTable;
1818
use vortex_array::arrays::PrimitiveArray;
1919
use vortex_array::arrays::VarBin;
2020
use vortex_array::arrays::VarBinArray;
21-
use vortex_array::arrays::primitive::PrimitiveArrayExt;
21+
use vortex_array::arrays::WidenArray;
2222
use vortex_array::arrays::varbin::VarBinArraySlotsExt;
2323
use vortex_array::builders::VarBinBuilder;
2424
use vortex_compressor::scheme::CompressionEstimate;
@@ -83,9 +83,12 @@ impl Scheme for VarBinScheme {
8383
let offsets = varbin
8484
.offsets()
8585
.clone()
86-
.execute::<PrimitiveArray>(exec_ctx)?
87-
.narrow(exec_ctx)?
88-
.into_array();
86+
.execute::<PrimitiveArray>(exec_ctx)?;
87+
#[expect(
88+
deprecated,
89+
reason = "TODO(connor): decide whether this child narrows its dtype or becomes a Widen"
90+
)]
91+
let offsets = WidenArray::encode_values(offsets, exec_ctx)?.into_array();
8992
let compressed_offsets =
9093
compressor.compress_child(&offsets, &compress_ctx, self.id(), 0, exec_ctx)?;
9194

0 commit comments

Comments
 (0)