Skip to content

Commit 40e0b1c

Browse files
authored
Add a scan for the range of valid decimal values (vortex-data#10939)
## Summary - Tracking Issue: vortex-data#10701 Decimal `Widen`, legacy decimal reads and DecimalByteParts splitting need raw extrema before choosing or checking precision. The existing decimal min/max implementation constructs checked scalars, so callers cannot use it directly for values outside the declared precision. ## Changes Expose the raw range from the existing pairwise scanner in `aggregate_fn/fns/min_max/decimal.rs`. Both the aggregate and new raw-range callers use that implementation. Scalar construction stays in the aggregate layer, and the raw helper ignores null slots without allocating or checking precision. Tests cover all six storage widths, native extrema, null payloads, empty and all-null inputs, and aggregate result dtypes. Benchmarks add wide decimal cases to the existing min/max target. Checks are left to CI. ## API Changes Add public `vortex_array::arrays::decimal::valid_range`, re-exported from the existing decimal min/max implementation. --------- Signed-off-by: Connor Tsui <[email protected]>
1 parent b2faca8 commit 40e0b1c

4 files changed

Lines changed: 259 additions & 33 deletions

File tree

‎vortex-array/benches/aggregate_max.rs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,17 @@ use vortex_array::VortexSessionExecute;
1111
use vortex_array::aggregate_fn::NumericalAggregateOpts;
1212
use vortex_array::aggregate_fn::fns::min_max::min_max;
1313
use vortex_array::aggregate_fn::session::AggregateFnSession;
14+
use vortex_array::arrays::DecimalArray;
1415
use vortex_array::arrays::PrimitiveArray;
1516
use vortex_array::arrays::VarBinViewArray;
1617
use vortex_array::dtype::DType;
18+
use vortex_array::dtype::DecimalDType;
19+
use vortex_array::dtype::NativeDecimalType;
1720
use vortex_array::dtype::Nullability;
21+
use vortex_array::dtype::i256;
1822
use vortex_array::optimizer::kernels::KernelSession;
23+
use vortex_array::validity::Validity;
24+
use vortex_buffer::Buffer;
1925
use vortex_session::VortexSession;
2026

2127
#[global_allocator]
@@ -63,6 +69,62 @@ fn max_i64(bencher: Bencher) {
6369
.bench_refs(|(a, ctx)| a.statistics().compute_max::<i64>(ctx));
6470
}
6571

72+
#[derive(Clone, Copy, Debug)]
73+
enum DecimalValidity {
74+
NonNullable,
75+
ScatteredNulls,
76+
}
77+
78+
const DECIMAL_VALIDITIES: &[DecimalValidity] = &[
79+
DecimalValidity::NonNullable,
80+
DecimalValidity::ScatteredNulls,
81+
];
82+
83+
fn bench_decimal_min_max<T: NativeDecimalType>(
84+
bencher: Bencher,
85+
values: Buffer<T>,
86+
decimal_dtype: DecimalDType,
87+
validity: DecimalValidity,
88+
) {
89+
let validity = match validity {
90+
DecimalValidity::NonNullable => Validity::NonNullable,
91+
DecimalValidity::ScatteredNulls => {
92+
let mut rng = StdRng::seed_from_u64(8);
93+
Validity::from_iter((0..N).map(|_| rng.random_bool(0.5)))
94+
}
95+
};
96+
97+
bencher
98+
.with_inputs(|| {
99+
(
100+
DecimalArray::new(values.clone(), decimal_dtype, validity.clone()).into_array(),
101+
SESSION.create_execution_ctx(),
102+
)
103+
})
104+
.bench_refs(|(array, ctx)| min_max(array, ctx, NumericalAggregateOpts::default()));
105+
}
106+
107+
#[divan::bench(args = DECIMAL_VALIDITIES)]
108+
fn min_max_decimal_i128(bencher: Bencher, validity: DecimalValidity) {
109+
let mut rng = StdRng::seed_from_u64(9);
110+
let bound = 10i128.pow(38) - 1;
111+
let values = (0..N).map(|_| rng.random_range(-bound..=bound)).collect();
112+
113+
bench_decimal_min_max(bencher, values, DecimalDType::new(38, 2), validity);
114+
}
115+
116+
#[divan::bench(args = DECIMAL_VALIDITIES)]
117+
fn min_max_decimal_i256(bencher: Bencher, validity: DecimalValidity) {
118+
let mut rng = StdRng::seed_from_u64(10);
119+
let factor = i256::from_i128(10i128.pow(38));
120+
let bound = 10i128.pow(37) - 1;
121+
let values = (0..N)
122+
.map(|_| i256::from_i128(rng.random_range(-bound..=bound)) * factor)
123+
.collect();
124+
125+
bench_decimal_min_max(bencher, values, DecimalDType::new(76, 2), validity);
126+
}
127+
66128
#[divan::bench]
67129
fn max_f64(bencher: Bencher) {
68130
let mut rng = StdRng::seed_from_u64(3);

‎vortex-array/src/aggregate_fn/fns/min_max/decimal.rs‎

Lines changed: 54 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ use crate::ExecutionCtx;
1111
use crate::aggregate_fn::AggregateArgs;
1212
use crate::aggregate_fn::NumericalAggregateOpts;
1313
use crate::arrays::DecimalArray;
14-
use crate::dtype::DecimalDType;
1514
use crate::dtype::NativeDecimalType;
1615
use crate::dtype::Nullability::NonNullable;
1716
use crate::match_each_decimal_value_type;
@@ -38,45 +37,67 @@ fn compute_min_max_with_validity<D>(
3837
where
3938
D: Into<DecimalValue> + NativeDecimalType,
4039
{
40+
let validity = array
41+
.as_ref()
42+
.validity()?
43+
.execute_mask(array.as_ref().len(), ctx)?;
44+
if matches!(&validity, Mask::AllFalse(_)) {
45+
return Ok(None);
46+
}
47+
48+
let decimal_dtype = array.decimal_dtype();
49+
4150
Ok(
42-
match array
43-
.as_ref()
44-
.validity()?
45-
.execute_mask(array.as_ref().len(), ctx)?
46-
{
47-
Mask::AllTrue(_) => compute_min_max(array.buffer::<D>().iter(), array.decimal_dtype()),
48-
Mask::AllFalse(_) => None,
49-
Mask::Values(v) => compute_min_max(
50-
array
51-
.buffer::<D>()
52-
.iter()
53-
.zip(v.bit_buffer().iter())
54-
.filter_map(|(v, m)| m.then_some(v)),
55-
array.decimal_dtype(),
56-
),
57-
},
51+
valid_range(array.buffer::<D>().as_slice(), &validity).map(|(min, max)| {
52+
let min_scalar = Scalar::decimal(min.into(), decimal_dtype, NonNullable);
53+
let max_scalar = if min == max {
54+
min_scalar.clone()
55+
} else {
56+
Scalar::decimal(max.into(), decimal_dtype, NonNullable)
57+
};
58+
59+
MinMaxResult {
60+
min: min_scalar,
61+
max: max_scalar,
62+
}
63+
}),
5864
)
5965
}
6066

61-
fn compute_min_max<'a, T>(
62-
iter: impl Iterator<Item = &'a T>,
63-
decimal_dtype: DecimalDType,
64-
) -> Option<MinMaxResult>
67+
/// Returns the smallest and the largest valid value, or `None` when no value is valid.
68+
///
69+
/// Null slots can hold any value and do not contribute to the range. The scan allocates nothing
70+
/// and does not check decimal precision.
71+
///
72+
/// # Panics
73+
///
74+
/// Panics if `validity` and `values` have different lengths.
75+
pub fn valid_range<T: NativeDecimalType>(values: &[T], validity: &Mask) -> Option<(T, T)> {
76+
assert_eq!(
77+
validity.len(),
78+
values.len(),
79+
"valid_range requires one validity bit per value"
80+
);
81+
82+
match validity {
83+
Mask::AllTrue(_) => compute_min_max(values.iter()),
84+
Mask::AllFalse(_) => None,
85+
Mask::Values(validity) => compute_min_max(
86+
values
87+
.iter()
88+
.zip(validity.bit_buffer().iter())
89+
.filter_map(|(value, valid)| valid.then_some(value)),
90+
),
91+
}
92+
}
93+
94+
fn compute_min_max<'a, T>(iter: impl Iterator<Item = &'a T>) -> Option<(T, T)>
6595
where
66-
T: Into<DecimalValue> + NativeDecimalType + Ord + Copy + 'a,
96+
T: NativeDecimalType,
6797
{
6898
match iter.minmax_by(|a, b| a.cmp(b)) {
6999
itertools::MinMaxResult::NoElements => None,
70-
itertools::MinMaxResult::OneElement(&x) => {
71-
let scalar = Scalar::decimal(x.into(), decimal_dtype, NonNullable);
72-
Some(MinMaxResult {
73-
min: scalar.clone(),
74-
max: scalar,
75-
})
76-
}
77-
itertools::MinMaxResult::MinMax(&min, &max) => Some(MinMaxResult {
78-
min: Scalar::decimal(min.into(), decimal_dtype, NonNullable),
79-
max: Scalar::decimal(max.into(), decimal_dtype, NonNullable),
80-
}),
100+
itertools::MinMaxResult::OneElement(&value) => Some((value, value)),
101+
itertools::MinMaxResult::MinMax(&min, &max) => Some((min, max)),
81102
}
82103
}

‎vortex-array/src/aggregate_fn/fns/min_max/mod.rs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
mod bool;
55
mod decimal;
6+
pub use decimal::valid_range;
67
mod extension;
78
mod fixed_size_list;
89
mod primitive;
@@ -535,6 +536,7 @@ mod tests {
535536
use std::sync::Arc;
536537
use std::sync::LazyLock;
537538

539+
use rstest::rstest;
538540
use vortex_buffer::BitBuffer;
539541
use vortex_buffer::Buffer;
540542
use vortex_buffer::buffer;
@@ -566,8 +568,10 @@ mod tests {
566568
use crate::arrays::NullArray;
567569
use crate::arrays::PrimitiveArray;
568570
use crate::arrays::VarBinArray;
571+
use crate::dtype::BigCast;
569572
use crate::dtype::DType;
570573
use crate::dtype::DecimalDType;
574+
use crate::dtype::DecimalType;
571575
use crate::dtype::Nullability;
572576
use crate::dtype::PType;
573577
use crate::dtype::extension::ExtDType;
@@ -576,6 +580,7 @@ mod tests {
576580
use crate::expr::stats::Precision;
577581
use crate::expr::stats::Stat;
578582
use crate::expr::stats::StatsProvider;
583+
use crate::match_each_decimal_value_type;
579584
use crate::scalar::DecimalValue;
580585
use crate::scalar::Scalar;
581586
use crate::scalar::ScalarValue;
@@ -995,6 +1000,66 @@ mod tests {
9951000
Ok(())
9961001
}
9971002

1003+
#[rstest]
1004+
#[case::non_nullable(vec![Some(-3), Some(7), Some(5)], false, Some((-3, 7)))]
1005+
#[case::all_valid(vec![Some(-3), Some(7), Some(5)], true, Some((-3, 7)))]
1006+
#[case::null_extremes(vec![None, Some(-3), Some(7), None], true, Some((-3, 7)))]
1007+
#[case::single_valid(vec![None, Some(5), None], true, Some((5, 5)))]
1008+
#[case::single_row(vec![Some(5)], false, Some((5, 5)))]
1009+
#[case::equal_values(vec![Some(5), Some(5)], true, Some((5, 5)))]
1010+
#[case::all_null(vec![None, None], true, None)]
1011+
#[case::empty_non_nullable(vec![], false, None)]
1012+
#[case::empty_nullable(vec![], true, None)]
1013+
fn test_decimal_valid_range(
1014+
#[case] values: Vec<Option<i8>>,
1015+
#[case] nullable: bool,
1016+
#[case] expected: Option<(i8, i8)>,
1017+
#[values(
1018+
DecimalType::I8,
1019+
DecimalType::I16,
1020+
DecimalType::I32,
1021+
DecimalType::I64,
1022+
DecimalType::I128,
1023+
DecimalType::I256
1024+
)]
1025+
storage: DecimalType,
1026+
) -> VortexResult<()> {
1027+
let mut ctx = SESSION.create_execution_ctx();
1028+
let decimal_dtype = DecimalDType::new(9, 2);
1029+
let validity = if nullable {
1030+
Validity::from_iter(values.iter().map(Option::is_some))
1031+
} else {
1032+
Validity::NonNullable
1033+
};
1034+
let expected = expected.map(|(min, max)| MinMaxResult {
1035+
min: Scalar::decimal(min.into(), decimal_dtype, Nullability::NonNullable),
1036+
max: Scalar::decimal(max.into(), decimal_dtype, Nullability::NonNullable),
1037+
});
1038+
1039+
match_each_decimal_value_type!(storage, |T| {
1040+
let buffer: Buffer<T> = values
1041+
.iter()
1042+
.enumerate()
1043+
.map(|(row, value)| match value {
1044+
Some(value) => <T as BigCast>::from(*value)
1045+
.vortex_expect("i8 fixture values fit every decimal storage type"),
1046+
None if row % 2 == 0 => T::MIN,
1047+
None => T::MAX,
1048+
})
1049+
.collect();
1050+
let array = DecimalArray::new(buffer, decimal_dtype, validity).into_array();
1051+
let actual = min_max(&array, &mut ctx, NumericalAggregateOpts::default())?;
1052+
assert_eq!(actual, expected);
1053+
if let Some(actual) = actual {
1054+
let dtype = DType::Decimal(decimal_dtype, Nullability::NonNullable);
1055+
assert_eq!(actual.min.dtype(), &dtype);
1056+
assert_eq!(actual.max.dtype(), &dtype);
1057+
}
1058+
1059+
Ok(())
1060+
})
1061+
}
1062+
9981063
#[test]
9991064
fn list_and_fixed_size_list_return_dtype() {
10001065
let element_dtype = DType::Primitive(PType::I32, Nullability::Nullable);

‎vortex-array/src/arrays/decimal/utils.rs‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ use vortex_error::vortex_ensure_eq;
1313
use vortex_mask::AllOr;
1414
use vortex_mask::Mask;
1515

16+
pub use crate::aggregate_fn::fns::min_max::valid_range;
1617
use crate::arrays::DecimalArray;
1718
use crate::arrays::decimal::DecimalArrayExt;
1819
use crate::buffer::BufferHandle;
@@ -318,6 +319,7 @@ mod tests {
318319
use super::check_precision;
319320
use super::converted_buffer;
320321
use super::copy_to_canonical_width;
322+
use super::valid_range;
321323
use crate::arrays::DecimalArray;
322324
use crate::dtype::BigCast;
323325
use crate::dtype::DecimalDType;
@@ -362,6 +364,82 @@ mod tests {
362364
.collect()
363365
}
364366

367+
#[rstest]
368+
#[case::all_valid(vec![5, -3, 7], vec![true, true, true], Some((-3, 7)))]
369+
#[case::partly_valid(vec![5, -3, 7], vec![true, true, false], Some((-3, 5)))]
370+
#[case::all_null(vec![5, -3, 7], vec![false, false, false], None)]
371+
#[case::empty(vec![], vec![], None)]
372+
#[case::one_valid(vec![5, -3, 7], vec![false, true, false], Some((-3, -3)))]
373+
#[case::outside_i8_precision(vec![-100, 100], vec![true, true], Some((-100, 100)))]
374+
fn test_valid_range(
375+
#[case] values: Vec<i8>,
376+
#[case] validity: Vec<bool>,
377+
#[case] expected: Option<(i8, i8)>,
378+
#[values(
379+
DecimalType::I8,
380+
DecimalType::I16,
381+
DecimalType::I32,
382+
DecimalType::I64,
383+
DecimalType::I128,
384+
DecimalType::I256
385+
)]
386+
storage: DecimalType,
387+
) -> VortexResult<()> {
388+
let validity = Mask::from_iter(validity);
389+
match_each_decimal_value_type!(storage, |T| {
390+
let values: Vec<T> = values
391+
.iter()
392+
.map(|&value| {
393+
<T as BigCast>::from(value).vortex_expect("i8 values fit every decimal type")
394+
})
395+
.collect();
396+
let expected = expected.map(|(min, max)| {
397+
(
398+
<T as BigCast>::from(min).vortex_expect("i8 values fit every decimal type"),
399+
<T as BigCast>::from(max).vortex_expect("i8 values fit every decimal type"),
400+
)
401+
});
402+
assert_eq!(valid_range(&values, &validity), expected);
403+
});
404+
405+
Ok(())
406+
}
407+
408+
#[rstest]
409+
fn test_valid_range_ignores_null_extremes(
410+
#[values(
411+
DecimalType::I8,
412+
DecimalType::I16,
413+
DecimalType::I32,
414+
DecimalType::I64,
415+
DecimalType::I128,
416+
DecimalType::I256
417+
)]
418+
storage: DecimalType,
419+
) -> VortexResult<()> {
420+
let validity = Mask::from_iter([false, true, true, false]);
421+
match_each_decimal_value_type!(storage, |T| {
422+
let min = <T as BigCast>::from(-3i8).vortex_expect("-3 fits every decimal type");
423+
let max = <T as BigCast>::from(7i8).vortex_expect("7 fits every decimal type");
424+
assert_eq!(
425+
valid_range(&[T::MIN, min, max, T::MAX], &validity),
426+
Some((min, max))
427+
);
428+
assert_eq!(
429+
valid_range(&[T::MIN, T::MAX], &Mask::AllTrue(2)),
430+
Some((T::MIN, T::MAX))
431+
);
432+
});
433+
434+
Ok(())
435+
}
436+
437+
#[test]
438+
#[should_panic(expected = "valid_range requires one validity bit per value")]
439+
fn test_valid_range_rejects_mismatched_mask() {
440+
valid_range(&[1i8, 2], &Mask::AllTrue(3));
441+
}
442+
365443
/// Each pair selects a different widening or narrowing conversion.
366444
#[rstest]
367445
fn converted_buffer_every_storage_pair(

0 commit comments

Comments
 (0)