Skip to content

Commit 1ba93b7

Browse files
committed
Reuse decimal min/max scanning for raw valid ranges
Signed-off-by: Connor Tsui <[email protected]>
1 parent 179fec7 commit 1ba93b7

3 files changed

Lines changed: 127 additions & 62 deletions

File tree

‎vortex-array/src/aggregate_fn/fns/min_max/decimal.rs‎

Lines changed: 54 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ use crate::ExecutionCtx;
1111
use crate::aggregate_fn::AggregateArgs;
1212
use crate::aggregate_fn::NumericalAggregateOpts;
1313
use crate::arrays::DecimalArray;
14-
use crate::dtype::DecimalDType;
1514
use crate::dtype::NativeDecimalType;
1615
use crate::dtype::Nullability::NonNullable;
1716
use crate::match_each_decimal_value_type;
@@ -38,45 +37,67 @@ fn compute_min_max_with_validity<D>(
3837
where
3938
D: Into<DecimalValue> + NativeDecimalType,
4039
{
40+
let validity = array
41+
.as_ref()
42+
.validity()?
43+
.execute_mask(array.as_ref().len(), ctx)?;
44+
if matches!(&validity, Mask::AllFalse(_)) {
45+
return Ok(None);
46+
}
47+
48+
let decimal_dtype = array.decimal_dtype();
49+
4150
Ok(
42-
match array
43-
.as_ref()
44-
.validity()?
45-
.execute_mask(array.as_ref().len(), ctx)?
46-
{
47-
Mask::AllTrue(_) => compute_min_max(array.buffer::<D>().iter(), array.decimal_dtype()),
48-
Mask::AllFalse(_) => None,
49-
Mask::Values(v) => compute_min_max(
50-
array
51-
.buffer::<D>()
52-
.iter()
53-
.zip(v.bit_buffer().iter())
54-
.filter_map(|(v, m)| m.then_some(v)),
55-
array.decimal_dtype(),
56-
),
57-
},
51+
valid_range(array.buffer::<D>().as_slice(), &validity).map(|(min, max)| {
52+
let min_scalar = Scalar::decimal(min.into(), decimal_dtype, NonNullable);
53+
let max_scalar = if min == max {
54+
min_scalar.clone()
55+
} else {
56+
Scalar::decimal(max.into(), decimal_dtype, NonNullable)
57+
};
58+
59+
MinMaxResult {
60+
min: min_scalar,
61+
max: max_scalar,
62+
}
63+
}),
5864
)
5965
}
6066

61-
fn compute_min_max<'a, T>(
62-
iter: impl Iterator<Item = &'a T>,
63-
decimal_dtype: DecimalDType,
64-
) -> Option<MinMaxResult>
67+
/// Returns the smallest and the largest valid value, or `None` when no value is valid.
68+
///
69+
/// Null slots can hold any value and do not contribute to the range. The scan allocates nothing
70+
/// and does not check decimal precision.
71+
///
72+
/// # Panics
73+
///
74+
/// Panics if `validity` and `values` have different lengths.
75+
pub fn valid_range<T: NativeDecimalType>(values: &[T], validity: &Mask) -> Option<(T, T)> {
76+
assert_eq!(
77+
validity.len(),
78+
values.len(),
79+
"valid_range requires one validity bit per value"
80+
);
81+
82+
match validity {
83+
Mask::AllTrue(_) => compute_min_max(values.iter()),
84+
Mask::AllFalse(_) => None,
85+
Mask::Values(validity) => compute_min_max(
86+
values
87+
.iter()
88+
.zip(validity.bit_buffer().iter())
89+
.filter_map(|(value, valid)| valid.then_some(value)),
90+
),
91+
}
92+
}
93+
94+
fn compute_min_max<'a, T>(iter: impl Iterator<Item = &'a T>) -> Option<(T, T)>
6595
where
66-
T: Into<DecimalValue> + NativeDecimalType + Ord + Copy + 'a,
96+
T: NativeDecimalType,
6797
{
6898
match iter.minmax_by(|a, b| a.cmp(b)) {
6999
itertools::MinMaxResult::NoElements => None,
70-
itertools::MinMaxResult::OneElement(&x) => {
71-
let scalar = Scalar::decimal(x.into(), decimal_dtype, NonNullable);
72-
Some(MinMaxResult {
73-
min: scalar.clone(),
74-
max: scalar,
75-
})
76-
}
77-
itertools::MinMaxResult::MinMax(&min, &max) => Some(MinMaxResult {
78-
min: Scalar::decimal(min.into(), decimal_dtype, NonNullable),
79-
max: Scalar::decimal(max.into(), decimal_dtype, NonNullable),
80-
}),
100+
itertools::MinMaxResult::OneElement(&value) => Some((value, value)),
101+
itertools::MinMaxResult::MinMax(&min, &max) => Some((min, max)),
81102
}
82103
}

‎vortex-array/src/aggregate_fn/fns/min_max/mod.rs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
mod bool;
55
mod decimal;
6+
pub use decimal::valid_range;
67
mod extension;
78
mod fixed_size_list;
89
mod primitive;
@@ -535,6 +536,7 @@ mod tests {
535536
use std::sync::Arc;
536537
use std::sync::LazyLock;
537538

539+
use rstest::rstest;
538540
use vortex_buffer::BitBuffer;
539541
use vortex_buffer::Buffer;
540542
use vortex_buffer::buffer;
@@ -566,8 +568,10 @@ mod tests {
566568
use crate::arrays::NullArray;
567569
use crate::arrays::PrimitiveArray;
568570
use crate::arrays::VarBinArray;
571+
use crate::dtype::BigCast;
569572
use crate::dtype::DType;
570573
use crate::dtype::DecimalDType;
574+
use crate::dtype::DecimalType;
571575
use crate::dtype::Nullability;
572576
use crate::dtype::PType;
573577
use crate::dtype::extension::ExtDType;
@@ -576,6 +580,7 @@ mod tests {
576580
use crate::expr::stats::Precision;
577581
use crate::expr::stats::Stat;
578582
use crate::expr::stats::StatsProvider;
583+
use crate::match_each_decimal_value_type;
579584
use crate::scalar::DecimalValue;
580585
use crate::scalar::Scalar;
581586
use crate::scalar::ScalarValue;
@@ -995,6 +1000,66 @@ mod tests {
9951000
Ok(())
9961001
}
9971002

1003+
#[rstest]
1004+
#[case::non_nullable(vec![Some(-3), Some(7), Some(5)], false, Some((-3, 7)))]
1005+
#[case::all_valid(vec![Some(-3), Some(7), Some(5)], true, Some((-3, 7)))]
1006+
#[case::null_extremes(vec![None, Some(-3), Some(7), None], true, Some((-3, 7)))]
1007+
#[case::single_valid(vec![None, Some(5), None], true, Some((5, 5)))]
1008+
#[case::single_row(vec![Some(5)], false, Some((5, 5)))]
1009+
#[case::equal_values(vec![Some(5), Some(5)], true, Some((5, 5)))]
1010+
#[case::all_null(vec![None, None], true, None)]
1011+
#[case::empty_non_nullable(vec![], false, None)]
1012+
#[case::empty_nullable(vec![], true, None)]
1013+
fn test_decimal_valid_range(
1014+
#[case] values: Vec<Option<i8>>,
1015+
#[case] nullable: bool,
1016+
#[case] expected: Option<(i8, i8)>,
1017+
#[values(
1018+
DecimalType::I8,
1019+
DecimalType::I16,
1020+
DecimalType::I32,
1021+
DecimalType::I64,
1022+
DecimalType::I128,
1023+
DecimalType::I256
1024+
)]
1025+
storage: DecimalType,
1026+
) -> VortexResult<()> {
1027+
let mut ctx = SESSION.create_execution_ctx();
1028+
let decimal_dtype = DecimalDType::new(9, 2);
1029+
let validity = if nullable {
1030+
Validity::from_iter(values.iter().map(Option::is_some))
1031+
} else {
1032+
Validity::NonNullable
1033+
};
1034+
let expected = expected.map(|(min, max)| MinMaxResult {
1035+
min: Scalar::decimal(min.into(), decimal_dtype, Nullability::NonNullable),
1036+
max: Scalar::decimal(max.into(), decimal_dtype, Nullability::NonNullable),
1037+
});
1038+
1039+
match_each_decimal_value_type!(storage, |T| {
1040+
let buffer: Buffer<T> = values
1041+
.iter()
1042+
.enumerate()
1043+
.map(|(row, value)| match value {
1044+
Some(value) => <T as BigCast>::from(*value)
1045+
.vortex_expect("i8 fixture values fit every decimal storage type"),
1046+
None if row % 2 == 0 => T::MIN,
1047+
None => T::MAX,
1048+
})
1049+
.collect();
1050+
let array = DecimalArray::new(buffer, decimal_dtype, validity).into_array();
1051+
let actual = min_max(&array, &mut ctx, NumericalAggregateOpts::default())?;
1052+
assert_eq!(actual, expected);
1053+
if let Some(actual) = actual {
1054+
let dtype = DType::Decimal(decimal_dtype, Nullability::NonNullable);
1055+
assert_eq!(actual.min.dtype(), &dtype);
1056+
assert_eq!(actual.max.dtype(), &dtype);
1057+
}
1058+
1059+
Ok(())
1060+
})
1061+
}
1062+
9981063
#[test]
9991064
fn list_and_fixed_size_list_return_dtype() {
10001065
let element_dtype = DType::Primitive(PType::I32, Nullability::Nullable);

‎vortex-array/src/arrays/decimal/utils.rs‎

Lines changed: 8 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ use vortex_error::vortex_ensure_eq;
1313
use vortex_mask::AllOr;
1414
use vortex_mask::Mask;
1515

16+
pub use crate::aggregate_fn::fns::min_max::valid_range;
1617
use crate::arrays::DecimalArray;
1718
use crate::arrays::decimal::DecimalArrayExt;
1819
use crate::buffer::BufferHandle;
@@ -196,34 +197,6 @@ fn narrow_checked_values<F: NativeDecimalType, W: NativeDecimalType>(
196197
buffer.freeze()
197198
}
198199

199-
/// Returns the smallest and the largest valid value, or `None` when no value is valid.
200-
///
201-
/// Null slots can hold any value and do not contribute to the range. The mask must have the same
202-
/// length as `values`. The scan allocates nothing and does not check decimal precision.
203-
pub fn valid_range<T: NativeDecimalType>(values: &[T], validity: &Mask) -> Option<(T, T)> {
204-
match validity {
205-
Mask::AllTrue(_) => {
206-
let (&first, rest) = values.split_first()?;
207-
Some(rest.iter().fold((first, first), |(min, max), &value| {
208-
(min.min(value), max.max(value))
209-
}))
210-
}
211-
Mask::AllFalse(_) => None,
212-
Mask::Values(validity) => {
213-
let mut range: Option<(T, T)> = None;
214-
validity.bit_buffer().for_each_set_index(|row| {
215-
let value = values[row];
216-
range = Some(match range {
217-
Some((min, max)) => (min.min(value), max.max(value)),
218-
None => (value, value),
219-
});
220-
});
221-
222-
range
223-
}
224-
}
225-
}
226-
227200
/// Checks that every valid value fits the precision of `decimal_dtype`.
228201
///
229202
/// Null slots are not checked, so they can hold any value. The check takes a materialized
@@ -397,7 +370,7 @@ mod tests {
397370
#[case::all_null(vec![5, -3, 7], vec![false, false, false], None)]
398371
#[case::empty(vec![], vec![], None)]
399372
#[case::one_valid(vec![5, -3, 7], vec![false, true, false], Some((-3, -3)))]
400-
#[case::outside_precision(vec![-100, 100], vec![true, true], Some((-100, 100)))]
373+
#[case::outside_i8_precision(vec![-100, 100], vec![true, true], Some((-100, 100)))]
401374
fn test_valid_range(
402375
#[case] values: Vec<i8>,
403376
#[case] validity: Vec<bool>,
@@ -461,6 +434,12 @@ mod tests {
461434
Ok(())
462435
}
463436

437+
#[test]
438+
#[should_panic(expected = "valid_range requires one validity bit per value")]
439+
fn test_valid_range_rejects_mismatched_mask() {
440+
valid_range(&[1i8, 2], &Mask::AllTrue(3));
441+
}
442+
464443
/// Each pair selects a different widening or narrowing conversion.
465444
#[rstest]
466445
fn converted_buffer_every_storage_pair(

0 commit comments

Comments
 (0)