Skip to content

Commit bf82f23

Browse files
feat: add between kernel for DecimalByteParts (#10152)
Co-authored-by: Claude Fable 5.1 <[email protected]> Signed-off-by: Joe Isaacs <[email protected]>
1 parent 23a59e3 commit bf82f23

6 files changed

Lines changed: 315 additions & 6 deletions

File tree

Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
use vortex_array::ArrayRef;
5+
use vortex_array::ArrayView;
6+
use vortex_array::ExecutionCtx;
7+
use vortex_array::IntoArray;
8+
use vortex_array::arrays::BoolArray;
9+
use vortex_array::arrays::ConstantArray;
10+
use vortex_array::builtins::ArrayBuiltins;
11+
use vortex_array::dtype::DType;
12+
use vortex_array::scalar::PValue;
13+
use vortex_array::scalar::Scalar;
14+
use vortex_array::scalar::ScalarValue;
15+
use vortex_array::scalar_fn::fns::between::BetweenKernel;
16+
use vortex_array::scalar_fn::fns::between::BetweenOptions;
17+
use vortex_array::scalar_fn::fns::between::StrictComparison;
18+
use vortex_buffer::BitBuffer;
19+
use vortex_error::VortexExpect;
20+
use vortex_error::VortexResult;
21+
22+
use crate::DecimalByteParts;
23+
use crate::decimal_byte_parts::DecimalBytePartsArraySlotsExt;
24+
use crate::decimal_byte_parts::compute::compare::Sign;
25+
use crate::decimal_byte_parts::compute::compare::decimal_value_wrapper_to_primitive;
26+
27+
impl BetweenKernel for DecimalByteParts {
28+
fn between(
29+
array: ArrayView<'_, Self>,
30+
lower: &ArrayRef,
31+
upper: &ArrayRef,
32+
options: &BetweenOptions,
33+
_ctx: &mut ExecutionCtx,
34+
) -> VortexResult<Option<ArrayRef>> {
35+
let (Some(lower_const), Some(upper_const)) = (lower.as_constant(), upper.as_constant())
36+
else {
37+
return Ok(None);
38+
};
39+
40+
// As in the compare kernel, the MSP alone only determines the ordering when it holds the
41+
// whole value; with lower parts present, fall back to the canonical decimal.
42+
if !array.lower_parts().is_empty() {
43+
return Ok(None);
44+
}
45+
46+
let ptype = array.msp().dtype().as_ptype();
47+
let nullability =
48+
array.dtype().nullability() | lower.dtype().nullability() | upper.dtype().nullability();
49+
50+
let lower_dv = lower_const
51+
.as_decimal()
52+
.decimal_value()
53+
.vortex_expect("the between short circuit strips a null lower bound");
54+
let upper_dv = upper_const
55+
.as_decimal()
56+
.decimal_value()
57+
.vortex_expect("the between short circuit strips a null upper bound");
58+
59+
// A bound that does not fit in the MSP type lies beyond every MSP value. Beyond the near
60+
// end of the range it constrains nothing, so it is replaced with that inclusive end of the
61+
// range; beyond the far end it excludes every value.
62+
let (lower_value, lower_strict) = match decimal_value_wrapper_to_primitive(lower_dv, ptype)
63+
{
64+
Ok(value) => (value, options.lower_strict),
65+
Err(Sign::Negative) => (PValue::min_value(ptype).into(), StrictComparison::NonStrict),
66+
Err(Sign::Positive) => {
67+
let validity = array.validity()?.union_nullability(nullability);
68+
return Ok(Some(
69+
BoolArray::new(BitBuffer::new_unset(array.len()), validity).into_array(),
70+
));
71+
}
72+
};
73+
let (upper_value, upper_strict) = match decimal_value_wrapper_to_primitive(upper_dv, ptype)
74+
{
75+
Ok(value) => (value, options.upper_strict),
76+
Err(Sign::Positive) => (PValue::max_value(ptype).into(), StrictComparison::NonStrict),
77+
Err(Sign::Negative) => {
78+
let validity = array.validity()?.union_nullability(nullability);
79+
return Ok(Some(
80+
BoolArray::new(BitBuffer::new_unset(array.len()), validity).into_array(),
81+
));
82+
}
83+
};
84+
85+
let bound = |value: ScalarValue, bound: &ArrayRef| -> VortexResult<ArrayRef> {
86+
let dtype = DType::Primitive(ptype, bound.dtype().nullability());
87+
Ok(ConstantArray::new(Scalar::try_new(dtype, Some(value))?, bound.len()).into_array())
88+
};
89+
let lower_msp = bound(lower_value, lower)?;
90+
let upper_msp = bound(upper_value, upper)?;
91+
92+
array
93+
.msp()
94+
.clone()
95+
.between(
96+
lower_msp,
97+
upper_msp,
98+
BetweenOptions {
99+
lower_strict,
100+
upper_strict,
101+
},
102+
)
103+
.map(Some)
104+
}
105+
}
106+
107+
#[cfg(test)]
108+
mod tests {
109+
use std::sync::LazyLock;
110+
111+
use rstest::rstest;
112+
use vortex_array::IntoArray;
113+
use vortex_array::VortexSessionExecute;
114+
use vortex_array::arrays::BoolArray;
115+
use vortex_array::arrays::ConstantArray;
116+
use vortex_array::arrays::DecimalArray;
117+
use vortex_array::arrays::PrimitiveArray;
118+
use vortex_array::assert_arrays_eq;
119+
use vortex_array::builtins::ArrayBuiltins;
120+
use vortex_array::dtype::DecimalDType;
121+
use vortex_array::dtype::Nullability;
122+
use vortex_array::scalar::DecimalValue;
123+
use vortex_array::scalar::Scalar;
124+
use vortex_array::scalar_fn::fns::between::BetweenOptions;
125+
use vortex_array::scalar_fn::fns::between::StrictComparison;
126+
use vortex_array::validity::Validity;
127+
use vortex_buffer::Buffer;
128+
use vortex_buffer::buffer;
129+
use vortex_error::VortexResult;
130+
use vortex_session::VortexSession;
131+
132+
use crate::DecimalByteParts;
133+
use crate::decimal_byte_parts::testing::i128_parts;
134+
135+
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
136+
let session = vortex_array::array_session();
137+
crate::initialize(&session);
138+
session
139+
});
140+
141+
fn opts(lower: StrictComparison, upper: StrictComparison) -> BetweenOptions {
142+
BetweenOptions {
143+
lower_strict: lower,
144+
upper_strict: upper,
145+
}
146+
}
147+
148+
fn bound(value: i128, dtype: DecimalDType, len: usize) -> vortex_array::ArrayRef {
149+
ConstantArray::new(
150+
Scalar::decimal(DecimalValue::I128(value), dtype, Nullability::NonNullable),
151+
len,
152+
)
153+
.into_array()
154+
}
155+
156+
#[rstest]
157+
#[case(StrictComparison::NonStrict, StrictComparison::NonStrict)]
158+
#[case(StrictComparison::Strict, StrictComparison::NonStrict)]
159+
#[case(StrictComparison::NonStrict, StrictComparison::Strict)]
160+
#[case(StrictComparison::Strict, StrictComparison::Strict)]
161+
fn msp_only_against_decimal_baseline(
162+
#[case] lower_strict: StrictComparison,
163+
#[case] upper_strict: StrictComparison,
164+
) -> VortexResult<()> {
165+
let mut ctx = SESSION.create_execution_ctx();
166+
let decimal_dtype = DecimalDType::new(10, 2);
167+
let values = [100i32, 200, 300, 400, 500];
168+
let validity = Validity::from_iter([true, false, true, true, true]);
169+
let parts = DecimalByteParts::try_new(
170+
PrimitiveArray::new(buffer![100i32, 200, 300, 400, 500], validity.clone()).into_array(),
171+
decimal_dtype,
172+
)?
173+
.into_array();
174+
let canonical = DecimalArray::new(
175+
Buffer::from_iter(values.iter().map(|v| *v as i128)),
176+
decimal_dtype,
177+
validity,
178+
)
179+
.into_array();
180+
181+
let lower = bound(200, decimal_dtype, parts.len());
182+
let upper = bound(400, decimal_dtype, parts.len());
183+
let options = opts(lower_strict, upper_strict);
184+
185+
let expected = canonical
186+
.between(lower.clone(), upper.clone(), options.clone())?
187+
.execute::<BoolArray>(&mut ctx)?;
188+
let actual = parts
189+
.between(lower, upper, options)?
190+
.execute::<BoolArray>(&mut ctx)?;
191+
assert_arrays_eq!(actual, expected, &mut ctx);
192+
Ok(())
193+
}
194+
195+
#[rstest]
196+
// Bounds beyond the near end of the i32 range constrain nothing.
197+
#[case::wide_both_sides(i32::MIN as i128 - 1, i32::MAX as i128 + 1, [true, true, true])]
198+
#[case::wide_lower(i32::MIN as i128 - 1, 200, [true, true, false])]
199+
#[case::wide_upper(200, i32::MAX as i128 + 1, [false, true, true])]
200+
// Bounds beyond the far end exclude everything.
201+
#[case::lower_above_max(i32::MAX as i128 + 1, i32::MAX as i128 + 2, [false, false, false])]
202+
#[case::upper_below_min(i32::MIN as i128 - 2, i32::MIN as i128 - 1, [false, false, false])]
203+
fn unconvertible_bounds(
204+
#[case] lower: i128,
205+
#[case] upper: i128,
206+
#[case] expected: [bool; 3],
207+
) -> VortexResult<()> {
208+
let mut ctx = SESSION.create_execution_ctx();
209+
let decimal_dtype = DecimalDType::new(38, 2);
210+
let parts =
211+
DecimalByteParts::try_new(buffer![100i32, 200, 400].into_array(), decimal_dtype)?
212+
.into_array();
213+
214+
let actual = parts
215+
.between(
216+
bound(lower, decimal_dtype, 3),
217+
bound(upper, decimal_dtype, 3),
218+
opts(StrictComparison::NonStrict, StrictComparison::NonStrict),
219+
)?
220+
.execute::<BoolArray>(&mut ctx)?;
221+
assert_arrays_eq!(actual, BoolArray::from_iter(expected), &mut ctx);
222+
Ok(())
223+
}
224+
225+
#[test]
226+
fn unconvertible_bound_keeps_nulls() -> VortexResult<()> {
227+
let mut ctx = SESSION.create_execution_ctx();
228+
let decimal_dtype = DecimalDType::new(38, 2);
229+
let parts = DecimalByteParts::try_new(
230+
PrimitiveArray::from_option_iter([Some(100i32), None, Some(400)]).into_array(),
231+
decimal_dtype,
232+
)?
233+
.into_array();
234+
235+
let actual = parts
236+
.between(
237+
bound(i32::MAX as i128 + 1, decimal_dtype, 3),
238+
bound(i32::MAX as i128 + 2, decimal_dtype, 3),
239+
opts(StrictComparison::NonStrict, StrictComparison::NonStrict),
240+
)?
241+
.execute::<BoolArray>(&mut ctx)?;
242+
let expected = BoolArray::from_iter([Some(false), None, Some(false)]);
243+
assert_arrays_eq!(actual, expected, &mut ctx);
244+
Ok(())
245+
}
246+
247+
#[test]
248+
fn with_lower_parts_falls_back_to_canonical() -> VortexResult<()> {
249+
let mut ctx = SESSION.create_execution_ctx();
250+
let values = vec![1i128 << 70, (1i128 << 70) + 1, 5, -(1i128 << 70)];
251+
let parts = i128_parts(values, Validity::NonNullable).into_array();
252+
let decimal_dtype = *parts.dtype().as_decimal_opt().unwrap();
253+
254+
let actual = parts
255+
.between(
256+
bound(5, decimal_dtype, 4),
257+
bound(1i128 << 70, decimal_dtype, 4),
258+
opts(StrictComparison::NonStrict, StrictComparison::Strict),
259+
)?
260+
.execute::<BoolArray>(&mut ctx)?;
261+
let expected = BoolArray::from_iter([false, false, true, false]);
262+
assert_arrays_eq!(actual, expected, &mut ctx);
263+
Ok(())
264+
}
265+
}

‎encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ impl CompareKernel for DecimalByteParts {
9090
// Used to represent the overflow direction when trying to
9191
// convert into the scalar type.
9292
#[derive(Debug)]
93-
enum Sign {
93+
pub(super) enum Sign {
9494
Positive,
9595
Negative,
9696
}
@@ -108,8 +108,9 @@ fn unconvertible_value(sign: Sign, operator: CompareOperator, nullability: Nulla
108108
}
109109
}
110110

111-
// this value return None is the decimal scalar cannot be cast the ptype.
112-
fn decimal_value_wrapper_to_primitive(
111+
/// Converts a decimal value into the MSP's primitive type, or returns the direction in which the
112+
/// value overflows that type.
113+
pub(super) fn decimal_value_wrapper_to_primitive(
113114
decimal_value: DecimalValue,
114115
ptype: PType,
115116
) -> Result<ScalarValue, Sign> {

‎encodings/decimal-byte-parts/src/decimal_byte_parts/compute/kernel.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33

44
use vortex_array::optimizer::kernels::ArrayKernelsExt;
55
use vortex_array::scalar_fn::ScalarFnVTable;
6+
use vortex_array::scalar_fn::fns::between::Between;
7+
use vortex_array::scalar_fn::fns::between::BetweenExecuteAdaptor;
68
use vortex_array::scalar_fn::fns::binary::Binary;
79
use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor;
810
use vortex_session::VortexSession;
@@ -16,4 +18,9 @@ pub(crate) fn initialize(session: &VortexSession) {
1618
DecimalByteParts,
1719
CompareExecuteAdaptor(DecimalByteParts),
1820
);
21+
kernels.register_execute_parent_kernel(
22+
Between.id(),
23+
DecimalByteParts,
24+
BetweenExecuteAdaptor(DecimalByteParts),
25+
);
1926
}

‎encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4+
mod between;
45
mod cast;
56
mod compare;
67
mod filter;

‎vortex-array/src/arrays/decimal/compute/between.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ use crate::ExecutionCtx;
1010
use crate::IntoArray;
1111
use crate::array::ArrayView;
1212
use crate::arrays::BoolArray;
13-
use crate::arrays::ConstantArray;
1413
use crate::arrays::Decimal;
1514
use crate::dtype::NativeDecimalType;
1615
use crate::dtype::Nullability;
@@ -83,8 +82,9 @@ fn between_unpack<T: NativeDecimalType>(
8382
Some(v) => Some(v),
8483
None => {
8584
if lower_dv.as_i256() >= i256::ZERO {
85+
let validity = arr.validity()?.union_nullability(nullability);
8686
return Ok(Some(
87-
ConstantArray::new(Scalar::bool(false, nullability), arr.len()).into_array(),
87+
BoolArray::new(BitBuffer::new_unset(arr.len()), validity).into_array(),
8888
));
8989
}
9090
None
@@ -95,8 +95,9 @@ fn between_unpack<T: NativeDecimalType>(
9595
Some(v) => Some(v),
9696
None => {
9797
if upper_dv.as_i256() < i256::ZERO {
98+
let validity = arr.validity()?.union_nullability(nullability);
9899
return Ok(Some(
99-
ConstantArray::new(Scalar::bool(false, nullability), arr.len()).into_array(),
100+
BoolArray::new(BitBuffer::new_unset(arr.len()), validity).into_array(),
100101
));
101102
}
102103
None

0 commit comments

Comments
 (0)