Skip to content

Commit 1169c46

Browse files
committed
Ignore empty chunks in file extrema and flag states
Signed-off-by: Connor Tsui <[email protected]>
1 parent b91d0fc commit 1169c46

2 files changed

Lines changed: 49 additions & 6 deletions

File tree

‎vortex-layout/src/layouts/file_stats/field.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,21 @@ impl FieldAccumulator {
9797
array: &ArrayRef,
9898
ctx: &mut ExecutionCtx,
9999
) -> VortexResult<()> {
100-
if let Some(min_max) = &mut self.min_max {
100+
// Empty constants carry a scalar, but contribute no extremum or flag boundary.
101+
if !array.is_empty()
102+
&& let Some(min_max) = &mut self.min_max
103+
{
101104
min_max.push_chunk(array, ctx)?;
102105
}
103106
for accumulator in &mut self.accumulators {
107+
if array.is_empty()
108+
&& (accumulator.aggregate.is::<Min>()
109+
|| accumulator.aggregate.is::<Max>()
110+
|| accumulator.aggregate.is::<IsConstant>()
111+
|| accumulator.aggregate.is::<IsSorted>())
112+
{
113+
continue;
114+
}
104115
accumulator.push_chunk(array, ctx)?;
105116
}
106117
self.seen_input = true;

‎vortex-layout/src/layouts/file_stats/tests.rs‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ use vortex_array::aggregate_fn::fns::uncompressed_size_in_bytes::UncompressedSiz
2828
use vortex_array::aggregate_fn::kernels::DynAggregateKernel;
2929
use vortex_array::aggregate_fn::session::AggregateFnSessionExt;
3030
use vortex_array::array_session;
31-
#[cfg(target_pointer_width = "64")]
3231
use vortex_array::arrays::ConstantArray;
3332
use vortex_array::arrays::ListArray;
3433
use vortex_array::arrays::PrimitiveArray;
@@ -272,13 +271,15 @@ fn sum_distinguishes_missing_input_from_zero(
272271
Ok(())
273272
}
274273

275-
#[test]
276-
fn empty_chunk_preserves_counts() -> VortexResult<()> {
277-
let array = PrimitiveArray::empty::<f64>(Nullability::NonNullable).into_array();
274+
#[rstest]
275+
#[case::primitive(PrimitiveArray::empty::<f64>(Nullability::NonNullable).into_array())]
276+
#[case::constant(ConstantArray::new(Scalar::from(42f64), 0).into_array())]
277+
fn empty_chunk_preserves_counts(#[case] array: ArrayRef) -> VortexResult<()> {
278278
let requests = [
279279
NullCount.bind(EmptyOptions),
280280
NanCount.bind(EmptyOptions),
281281
UncompressedSizeInBytes.bind(EmptyOptions),
282+
Sum.bind(NumericalAggregateOpts::skip_nans()),
282283
Min.bind(NumericalAggregateOpts::skip_nans()),
283284
Max.bind(NumericalAggregateOpts::skip_nans()),
284285
IsConstant.bind(EmptyOptions),
@@ -297,9 +298,16 @@ fn empty_chunk_preserves_counts() -> VortexResult<()> {
297298
results.get_result(&requests[2]),
298299
Precision::Exact(Scalar::from(0u64))
299300
);
300-
for request in &requests[3..] {
301+
assert_eq!(
302+
results.get_result(&requests[3]),
303+
Precision::Exact(Scalar::primitive(0f64, Nullability::Nullable))
304+
);
305+
for request in &requests[4..] {
301306
assert!(results.get_result(request).is_absent());
302307
}
308+
for stat in [Stat::Min, Stat::Max, Stat::IsConstant, Stat::IsSorted] {
309+
assert!(array.statistics().get(stat).is_absent());
310+
}
303311
Ok(())
304312
}
305313

@@ -521,3 +529,27 @@ fn fused_min_max_reuses_exact_chunk_extrema() -> VortexResult<()> {
521529
);
522530
Ok(())
523531
}
532+
533+
#[rstest]
534+
#[case::min(vec![Min.bind(NumericalAggregateOpts::skip_nans())], vec![1, 2], vec![Scalar::primitive(1i32, Nullability::Nullable)])]
535+
#[case::max(vec![Max.bind(NumericalAggregateOpts::skip_nans())], vec![1, 2], vec![Scalar::primitive(2i32, Nullability::Nullable)])]
536+
#[case::fused(vec![Min.bind(NumericalAggregateOpts::skip_nans()), Max.bind(NumericalAggregateOpts::skip_nans())], vec![1, 2], vec![Scalar::primitive(1i32, Nullability::Nullable), Scalar::primitive(2i32, Nullability::Nullable)])]
537+
#[case::constant(vec![IsConstant.bind(EmptyOptions)], vec![1, 1], vec![Scalar::from(true)])]
538+
#[case::sorted(vec![IsSorted.bind(IsSortedOptions { strict: false })], vec![1, 2], vec![Scalar::from(true)])]
539+
#[case::strict_sorted(vec![IsSorted.bind(IsSortedOptions { strict: true })], vec![1, 2], vec![Scalar::from(true)])]
540+
fn empty_constants_do_not_add_values_or_boundaries(
541+
#[case] requests: Vec<AggregateFnRef>,
542+
#[case] values: Vec<i32>,
543+
#[case] expected: Vec<Scalar>,
544+
) -> VortexResult<()> {
545+
let chunks = [
546+
ConstantArray::new(Scalar::from(42i32), 0).into_array(),
547+
PrimitiveArray::from_iter(values).into_array(),
548+
ConstantArray::new(Scalar::from(-42i32), 0).into_array(),
549+
];
550+
let results = accumulate(chunks[0].dtype(), &requests, &chunks, 64)?;
551+
for (request, expected) in requests.iter().zip(expected) {
552+
assert_eq!(results.get_result(request), Precision::Exact(expected));
553+
}
554+
Ok(())
555+
}

0 commit comments

Comments
 (0)