Skip to content

Commit b991aa6

Browse files
committed
Push decimal comparisons, between and fills through Widen
Compare decimal children at a common precision, narrow constants that fit, and answer out-of-range comparisons without widening the child values. Share the cast and execution paths between integer and decimal kernels. Decimal fills outside the child precision use canonical execution. Assert narrow comparison operand dtypes and concrete out-of-range results through the registered comparison kernel, including validity and null constants. Compare logical results with canonical decimal operations. Signed-off-by: Connor Tsui <[email protected]>
1 parent 7146a1b commit b991aa6

9 files changed

Lines changed: 399 additions & 61 deletions

File tree

‎vortex-array/src/arrays/widen/array.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@ impl WidenArray {
3030
/// **must** be strictly narrower. When `values` is itself a `Widen`, the new array stores its
3131
/// values instead, so that `Widen` arrays do not nest.
3232
///
33-
/// A decimal child **must** be a valid decimal array for its declared dtype.
33+
/// A decimal child **must** be a valid decimal array for its declared dtype. Comparisons,
34+
/// bounds checks and fills rely on this requirement. A violation gives incorrect results,
35+
/// but no undefined behavior.
3436
pub fn try_new(mut values: ArrayRef, dtype: DType) -> VortexResult<Self> {
3537
// NB: Check `values` before the flatten. The values of a `Widen` child are narrower than
3638
// the child, so the check in `try_from_parts` alone would accept a child as wide as `dtype`

‎vortex-array/src/arrays/widen/compute/between.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use crate::arrays::Constant;
99
use crate::arrays::Widen;
1010
use crate::arrays::widen::WidenArraySlotsExt;
1111
use crate::arrays::widen::compute::constant_fits_values;
12-
use crate::arrays::widen::compute::constant_in_values_ptype;
12+
use crate::arrays::widen::compute::constant_in_values_dtype;
1313
use crate::builtins::ArrayBuiltins;
1414
use crate::scalar_fn::fns::between::BetweenOptions;
1515
use crate::scalar_fn::fns::between::BetweenReduce;
@@ -28,8 +28,8 @@ impl BetweenReduce for Widen {
2828
&& constant_fits_values(lower, values.dtype()) == Some(true)
2929
&& constant_fits_values(upper, values.dtype()) == Some(true)
3030
{
31-
let lower = constant_in_values_ptype(lower, values, array.len())?;
32-
let upper = constant_in_values_ptype(upper, values, array.len())?;
31+
let lower = constant_in_values_dtype(lower, values, array.len())?;
32+
let upper = constant_in_values_dtype(upper, values, array.len())?;
3333
return values
3434
.clone()
3535
.between(lower, upper, options.clone())

‎vortex-array/src/arrays/widen/compute/compare.rs‎

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,12 @@ use crate::arrays::BoolArray;
1313
use crate::arrays::ConstantArray;
1414
use crate::arrays::Widen;
1515
use crate::arrays::widen::WidenArraySlotsExt;
16-
use crate::arrays::widen::compute::cast_keeping_nullability;
1716
use crate::arrays::widen::compute::constant_fits_values;
18-
use crate::arrays::widen::compute::constant_in_values_ptype;
17+
use crate::arrays::widen::compute::constant_in_values_dtype;
1918
use crate::arrays::widen::compute::ptype_holding_both_values;
2019
use crate::builtins::ArrayBuiltins;
20+
use crate::dtype::DType;
21+
use crate::dtype::DecimalDType;
2122
use crate::scalar::Scalar;
2223
use crate::scalar_fn::fns::binary::CompareKernel;
2324
use crate::scalar_fn::fns::operators::CompareOperator;
@@ -50,24 +51,47 @@ impl CompareKernel for Widen {
5051
return compare_out_of_range(lhs, constant, operator, ctx).map(Some);
5152
}
5253

53-
let constant = constant_in_values_ptype(constant, values, lhs.len())?;
54+
let constant = constant_in_values_dtype(constant, values, lhs.len())?;
5455

5556
values.binary(constant, Operator::from(operator)).map(Some)
5657
}
5758
}
5859

5960
/// Compares the values of two `Widen` arrays at the wider of the two value types.
61+
///
62+
/// Decimal values with the same scale compare at the larger of the two child precisions, which
63+
/// holds every value of both children.
6064
fn compare_narrow_values(
6165
lhs: ArrayView<'_, Widen>,
6266
rhs: ArrayView<'_, Widen>,
6367
operator: CompareOperator,
6468
) -> VortexResult<Option<ArrayRef>> {
65-
let Some(ptype) = ptype_holding_both_values(lhs, rhs) else {
66-
return Ok(None);
67-
};
69+
let lhs_values = lhs.values();
70+
let rhs_values = rhs.values();
71+
72+
let dtype = match (lhs_values.dtype(), rhs_values.dtype()) {
73+
(DType::Primitive(..), DType::Primitive(..)) => {
74+
let Some(ptype) = ptype_holding_both_values(lhs, rhs) else {
75+
return Ok(None);
76+
};
6877

69-
let lhs_values = cast_keeping_nullability(lhs.values(), ptype)?;
70-
let rhs_values = cast_keeping_nullability(rhs.values(), ptype)?;
78+
DType::from(ptype)
79+
}
80+
(DType::Decimal(lhs_dtype, _), DType::Decimal(rhs_dtype, _))
81+
if lhs_dtype.scale() == rhs_dtype.scale() =>
82+
{
83+
DType::Decimal(
84+
DecimalDType::new(
85+
lhs_dtype.precision().max(rhs_dtype.precision()),
86+
lhs_dtype.scale(),
87+
),
88+
lhs_values.dtype().nullability(),
89+
)
90+
}
91+
_ => return Ok(None),
92+
};
93+
let lhs_values = lhs_values.cast(dtype.with_nullability(lhs_values.dtype().nullability()))?;
94+
let rhs_values = rhs_values.cast(dtype.with_nullability(rhs_values.dtype().nullability()))?;
7195

7296
lhs_values
7397
.binary(rhs_values, Operator::from(operator))

‎vortex-array/src/arrays/widen/compute/fill_null.rs‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use crate::ArrayRef;
77
use crate::array::ArrayView;
88
use crate::arrays::Widen;
99
use crate::arrays::widen::WidenArraySlotsExt;
10-
use crate::arrays::widen::compute::cast_keeping_nullability;
10+
use crate::arrays::widen::compute::constant_fits_values;
1111
use crate::arrays::widen::compute::ptype_holding_values_and_scalar;
1212
use crate::arrays::widen::compute::wrap_values;
1313
use crate::builtins::ArrayBuiltins;
@@ -20,15 +20,30 @@ impl FillNullReduce for Widen {
2020
array: ArrayView<'_, Self>,
2121
fill_value: &Scalar,
2222
) -> VortexResult<Option<ArrayRef>> {
23-
let Some(ptype) = ptype_holding_values_and_scalar(array, fill_value)? else {
23+
if !fill_value.dtype().eq_ignore_nullability(array.dtype()) {
2424
return Ok(None);
25-
};
25+
}
26+
27+
let values = array.values();
28+
let dtype = match values.dtype() {
29+
DType::Primitive(..) => {
30+
let Some(ptype) = ptype_holding_values_and_scalar(array, fill_value)? else {
31+
return Ok(None);
32+
};
2633

27-
// The values widen to a type that holds the fill value. When only the logical type holds
28-
// it, the result is a plain array of the logical type.
29-
let widened = cast_keeping_nullability(array.values(), ptype)?;
34+
DType::Primitive(ptype, values.dtype().nullability())
35+
}
36+
DType::Decimal(..)
37+
if constant_fits_values(fill_value, values.dtype()) == Some(true) =>
38+
{
39+
values.dtype().clone()
40+
}
41+
// Decimal fills outside the child precision use the canonical fill kernel.
42+
_ => return Ok(None),
43+
};
44+
let widened = values.cast(dtype.clone())?;
3045
let fill_value =
31-
fill_value.cast(&DType::Primitive(ptype, fill_value.dtype().nullability()))?;
46+
fill_value.cast(&dtype.with_nullability(fill_value.dtype().nullability()))?;
3247
let filled = widened.fill_null(fill_value)?;
3348
let dtype = array.dtype().with_nullability(filled.dtype().nullability());
3449

‎vortex-array/src/arrays/widen/compute/mod.rs‎

Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -100,34 +100,41 @@ pub(super) fn cast_keeping_nullability(array: &ArrayRef, ptype: PType) -> Vortex
100100
array.cast(DType::Primitive(ptype, array.dtype().nullability()))
101101
}
102102

103-
/// Returns whether the range of the integer dtype `values` holds `constant`.
103+
/// Returns whether the dtype range of `values` holds `constant`.
104104
///
105-
/// Returns `None` when `values` is not an integer dtype, when `constant` is null, and when
106-
/// `constant` is not an integer with the signedness of `values`.
105+
/// Integers must have the same signedness, and decimals must have the same scale. Returns `None`
106+
/// for null constants, unsupported dtypes, or incompatible constants.
107107
pub(super) fn constant_fits_values(constant: &Scalar, values: &DType) -> Option<bool> {
108-
let DType::Primitive(ptype, _) = values else {
109-
return None;
110-
};
111-
let value = constant.as_primitive_opt()?.pvalue()?;
112-
113-
// `PValue` compares an integer with another value through `i64` or `u64`, and it panics when
114-
// the other value does not convert. So only integers of the same signedness compare here.
115-
let value_ptype = value.ptype();
116-
if !ptype.is_int() || !value_ptype.is_int() {
117-
return None;
108+
match (values, constant.dtype()) {
109+
(DType::Decimal(values_dtype, _), DType::Decimal(constant_dtype, _))
110+
if values_dtype.scale() == constant_dtype.scale() =>
111+
{
112+
let value = constant.as_decimal().decimal_value()?;
113+
114+
Some(value.fits_in_precision(*values_dtype))
115+
}
116+
(DType::Primitive(values_ptype, _), DType::Primitive(constant_ptype, _))
117+
if values_ptype.is_int()
118+
&& constant_ptype.is_int()
119+
&& values_ptype.is_signed_int() == constant_ptype.is_signed_int() =>
120+
{
121+
// `PValue` compares through `i64` or `u64`, so the signedness must match.
122+
let value = constant.as_primitive().pvalue()?;
123+
124+
Some(
125+
PValue::min_value(*values_ptype) <= value
126+
&& value <= PValue::max_value(*values_ptype),
127+
)
128+
}
129+
_ => None,
118130
}
119-
if ptype.is_signed_int() != value_ptype.is_signed_int() {
120-
return None;
121-
}
122-
123-
Some(PValue::min_value(*ptype) <= value && value <= PValue::max_value(*ptype))
124131
}
125132

126133
/// Returns the wider of the value types of two [`Widen`] arrays, or `None` when a value type is
127134
/// not primitive.
128135
///
129-
/// The logical dtypes **must** have the same signedness, which a debug build asserts. The value
130-
/// types then have that signedness too, so the wider one holds every value of the narrower one.
136+
/// The logical dtypes **must** have the same signedness. The value types then have that signedness
137+
/// too, so the wider one holds every value of the narrower one.
131138
pub(super) fn ptype_holding_both_values(
132139
lhs: ArrayView<'_, Widen>,
133140
rhs: ArrayView<'_, Widen>,
@@ -147,7 +154,7 @@ pub(super) fn ptype_holding_both_values(
147154

148155
/// Returns `constant`, which **must** fit the type of `values`, as a constant array of that type
149156
/// with the nullability of `constant`.
150-
pub(super) fn constant_in_values_ptype(
157+
pub(super) fn constant_in_values_dtype(
151158
constant: &Scalar,
152159
values: &ArrayRef,
153160
len: usize,

‎vortex-array/src/arrays/widen/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
//! [`WidenArray::encode`] picks the narrowest such type for a [`PrimitiveArray`].
1212
//!
1313
//! A decimal child has the logical scale and a lower precision, and **must** be valid for its dtype.
14+
//! Comparisons, bounds checks and fills use the child dtype to bound its valid values.
1415
//! [`WidenArray::encode_decimal`] picks the smallest precision that holds the valid values and the
1516
//! scale. Decimal execution copies the child values into the logical canonical width.
1617
//!

‎vortex-array/src/arrays/widen/tests/compute.rs‎

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ use vortex_mask::Mask;
99

1010
use super::SESSION;
1111
use super::all_invalid;
12+
use super::assert_compare_matches_canonical;
1213
use super::canonical;
1314
use super::non_nullable;
1415
use super::one_row;
@@ -18,7 +19,6 @@ use super::stored_ptype;
1819
use super::unsigned;
1920
use super::widen;
2021
use crate::ArrayRef;
21-
use crate::ExecutionCtx;
2222
use crate::IntoArray;
2323
use crate::VortexSessionExecute;
2424
use crate::arrays::BoolArray;
@@ -50,26 +50,6 @@ use crate::scalar_fn::fns::operators::CompareOperator;
5050
use crate::scalar_fn::fns::operators::Operator;
5151
use crate::validity::Validity;
5252

53-
/// Asserts that `lhs op rhs` and `rhs op lhs` equal the same compares of the canonical operands.
54-
#[track_caller]
55-
fn assert_compare_matches_canonical(
56-
lhs: &ArrayRef,
57-
rhs: &ArrayRef,
58-
operator: Operator,
59-
ctx: &mut ExecutionCtx,
60-
) -> VortexResult<()> {
61-
let canonical_lhs = canonical(lhs, ctx)?;
62-
let canonical_rhs = canonical(rhs, ctx)?;
63-
64-
let expected = canonical_lhs.binary(canonical_rhs.clone(), operator)?;
65-
assert_arrays_eq!(lhs.binary(rhs.clone(), operator)?, expected, ctx);
66-
67-
let expected = canonical_rhs.binary(canonical_lhs, operator)?;
68-
assert_arrays_eq!(rhs.binary(lhs.clone(), operator)?, expected, ctx);
69-
70-
Ok(())
71-
}
72-
7353
/// Slice, filter and mask run on the values, so each result is a `Widen`. The conformance suites
7454
/// check the values.
7555
#[test]

0 commit comments

Comments
 (0)