Skip to content

Commit 54d91e0

Browse files
committed
refactor(stats): transfer aggregate results between representations
Signed-off-by: Connor Tsui <[email protected]>
1 parent 652fd9e commit 54d91e0

14 files changed

Lines changed: 378 additions & 102 deletions

File tree

‎encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ pub fn bitpack_encode(
101101
array.len(),
102102
0,
103103
)?;
104-
bitpacked.statistics().inherit_from(array.statistics());
104+
bitpacked.aggregations().inherit_from(array.aggregations());
105105
Ok(bitpacked)
106106
}
107107

@@ -120,7 +120,6 @@ pub unsafe fn bitpack_encode_unchecked(
120120
// SAFETY: non-negativity of input checked by caller.
121121
let packed = unsafe { bitpack_unchecked(&array, bit_width) };
122122

123-
let arr_ref = array.clone().into_array();
124123
let bitpacked = BitPacked::try_new(
125124
BufferHandle::new_host(packed),
126125
array.ptype(),
@@ -131,7 +130,7 @@ pub unsafe fn bitpack_encode_unchecked(
131130
0,
132131
)
133132
.vortex_expect("bitpacked array construction should succeed");
134-
bitpacked.statistics().inherit_from(arr_ref.statistics());
133+
bitpacked.aggregations().inherit_from(array.aggregations());
135134
Ok(bitpacked)
136135
}
137136

@@ -440,16 +439,20 @@ pub mod test_harness {
440439
}
441440

442441
#[cfg(test)]
443-
mod test {
442+
mod tests {
444443
use std::sync::LazyLock;
445444

446445
use rand::SeedableRng;
447446
use rand::rngs::StdRng;
448447
use vortex_array::VortexSessionExecute;
448+
use vortex_array::aggregate_fn::EmptyOptions;
449+
use vortex_array::aggregate_fn::fns::count::Count;
450+
use vortex_array::aggregate_fn::fns::uncompressed_size_in_bytes::UncompressedSizeInBytes;
449451
use vortex_array::arrays::ChunkedArray;
450452
use vortex_array::assert_arrays_eq;
451453
use vortex_array::builders::ArrayBuilder;
452454
use vortex_array::builders::PrimitiveBuilder;
455+
use vortex_array::expr::stats::Precision;
453456
use vortex_buffer::Buffer;
454457
use vortex_error::VortexError;
455458
use vortex_error::vortex_err;
@@ -466,6 +469,24 @@ mod test {
466469
session
467470
});
468471

472+
#[test]
473+
fn bitpack_preserves_logical_results_and_discards_physical_size() -> VortexResult<()> {
474+
let source = PrimitiveArray::from_iter([1u32, 2, 3]);
475+
let mut ctx = SESSION.create_execution_ctx();
476+
let count = Count.bind(NumericalAggregateOpts::skip_nans());
477+
let size = UncompressedSizeInBytes.bind(EmptyOptions);
478+
source.aggregations().compute_result(&count, &mut ctx)?;
479+
source.aggregations().compute_result(&size, &mut ctx)?;
480+
481+
let packed = bitpack_encode(&source, 2, None, &mut ctx)?;
482+
assert_eq!(
483+
packed.aggregations().get_result_as::<u64>(&count)?,
484+
Precision::Exact(3)
485+
);
486+
assert_eq!(packed.aggregations().get_result(&size), Precision::Absent);
487+
Ok(())
488+
}
489+
469490
#[test]
470491
fn test_best_bit_width() {
471492
// 10 1-bit values, 20 2-bit, etc.

‎encodings/zstd/src/zstd_buffers.rs‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ impl ZstdBuffers {
9797
ArrayParts::new(ZstdBuffers, array.dtype().clone(), array.len(), data)
9898
.with_slots(slots),
9999
)?;
100-
compressed.statistics().inherit_from(array.statistics());
100+
compressed.aggregations().inherit_from(array.aggregations());
101101
Ok(compressed)
102102
}
103103

@@ -558,13 +558,16 @@ mod tests {
558558
use vortex_array::ArrayRef;
559559
use vortex_array::IntoArray;
560560
use vortex_array::VortexSessionExecute;
561+
use vortex_array::aggregate_fn::AggregateFnVTableExt;
562+
use vortex_array::aggregate_fn::EmptyOptions;
563+
use vortex_array::aggregate_fn::NumericalAggregateOpts;
564+
use vortex_array::aggregate_fn::fns::count::Count;
565+
use vortex_array::aggregate_fn::fns::uncompressed_size_in_bytes::UncompressedSizeInBytes;
561566
use vortex_array::array_session;
562567
use vortex_array::arrays::PrimitiveArray;
563568
use vortex_array::arrays::VarBinViewArray;
564569
use vortex_array::assert_arrays_eq;
565570
use vortex_array::expr::stats::Precision;
566-
use vortex_array::expr::stats::Stat;
567-
use vortex_array::expr::stats::StatsProvider;
568571
use vortex_array::serde::SerializeOptions;
569572
use vortex_array::serde::SerializedArray;
570573
use vortex_array::session::ArraySessionExt;
@@ -659,13 +662,30 @@ mod tests {
659662
}
660663

661664
#[test]
662-
fn test_compress_inherits_stats() -> VortexResult<()> {
663-
let input = make_primitive_array();
664-
input.statistics().set(Stat::Min, Precision::exact(0i32));
665-
666-
let compressed = ZstdBuffers::compress(&input, 3, &array_session())?;
665+
fn compress_preserves_option_results_and_discards_physical_size() -> VortexResult<()> {
666+
let input = PrimitiveArray::from_iter([1.0f64, f64::NAN, 3.0]).into_array();
667+
let session = array_session();
668+
let mut ctx = session.create_execution_ctx();
669+
let skip = Count.bind(NumericalAggregateOpts::skip_nans());
670+
let include = Count.bind(NumericalAggregateOpts::include_nans());
671+
let size = UncompressedSizeInBytes.bind(EmptyOptions);
672+
for aggregate in [&skip, &include, &size] {
673+
input.aggregations().compute_result(aggregate, &mut ctx)?;
674+
}
667675

668-
assert!(!compressed.statistics().get(Stat::Min).is_absent());
676+
let compressed = ZstdBuffers::compress(&input, 3, &session)?;
677+
assert_eq!(
678+
compressed.aggregations().get_result_as::<u64>(&skip)?,
679+
Precision::Exact(2)
680+
);
681+
assert_eq!(
682+
compressed.aggregations().get_result_as::<u64>(&include)?,
683+
Precision::Exact(3)
684+
);
685+
assert_eq!(
686+
compressed.aggregations().get_result(&size),
687+
Precision::Absent
688+
);
669689
Ok(())
670690
}
671691

‎vortex-array/src/array/erased.rs‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -43,16 +43,15 @@ use crate::arrays::Constant;
4343
use crate::arrays::DictArray;
4444
use crate::arrays::FilterArray;
4545
use crate::arrays::SliceArray;
46+
use crate::arrays::slice::inherit_slice_results;
4647
use crate::buffer::BufferHandle;
4748
use crate::builders::ArrayBuilder;
4849
use crate::dtype::DType;
4950
use crate::expr::stats::Precision;
50-
use crate::expr::stats::Stat;
5151
use crate::legacy_session;
5252
use crate::matcher::Matcher;
5353
use crate::optimizer::ArrayOptimizer;
5454
use crate::scalar::Scalar;
55-
use crate::scalar::ScalarValue;
5655
use crate::stats::AggregationsRef;
5756
use crate::stats::StatsSetRef;
5857
use crate::validity::Validity;
@@ -235,19 +234,8 @@ impl ArrayRef {
235234
.into_array()
236235
.optimize()?;
237236

238-
// Propagate some stats from the original array to the sliced array.
239237
if !sliced.is::<Constant>() {
240-
self.statistics().with_iter(|iter| {
241-
sliced.statistics().inherit(iter.filter(|(stat, value)| {
242-
matches!(
243-
stat,
244-
Stat::IsConstant | Stat::IsSorted | Stat::IsStrictSorted
245-
) && value
246-
.as_ref()
247-
.as_exact()
248-
.is_some_and(|v| matches!(v, ScalarValue::Bool(true)))
249-
}));
250-
});
238+
inherit_slice_results(self, &sliced);
251239
}
252240

253241
Ok(sliced)

‎vortex-array/src/arrays/chunked/array.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -197,17 +197,17 @@ impl Array<Chunked> {
197197
}
198198
// This is the slow path that will be hit at most once per execution since the second one
199199
// *MUST* have execlusive access due to this copy.
200-
let stats = self.statistics().to_owned();
201200
let mut data = self.data().clone();
202201
data.next_builder_slot = next_builder_slot;
203202
// SAFETY: we only modified next_builder_slot which doesn't affect array invariants.
204-
unsafe {
203+
let array = unsafe {
205204
Array::from_parts_unchecked(
206205
ArrayParts::new(Chunked, self.dtype().clone(), self.len(), data)
207206
.with_slots(self.slots().iter().cloned().collect::<ArraySlots>()),
208207
)
209-
}
210-
.with_stats_set(stats)
208+
};
209+
array.aggregations().inherit_from(self.aggregations());
210+
array
211211
}
212212

213213
/// Constructs a new `ChunkedArray`.

‎vortex-array/src/arrays/chunked/tests.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ use vortex_session::VortexSession;
1212
use crate::Canonical;
1313
use crate::IntoArray;
1414
use crate::VortexSessionExecute;
15+
use crate::aggregate_fn::AggregateFnVTableExt;
16+
use crate::aggregate_fn::NumericalAggregateOpts;
17+
use crate::aggregate_fn::fns::count::Count;
1518
use crate::array_session;
1619
use crate::arrays::Chunked;
1720
use crate::arrays::ChunkedArray;
@@ -30,6 +33,7 @@ use crate::dtype::Nullability;
3033
use crate::dtype::PType;
3134
use crate::dtype::PType::I32;
3235
use crate::executor::execute_into_builder;
36+
use crate::expr::stats::Precision;
3337
use crate::validity::Validity;
3438

3539
static SESSION: LazyLock<VortexSession> = LazyLock::new(array_session);
@@ -437,3 +441,24 @@ pub fn pack_nested_lists() {
437441
canon_values.execute_scalar(1, &mut ctx).unwrap()
438442
);
439443
}
444+
445+
#[test]
446+
fn builder_cursor_preserves_generic_results() -> VortexResult<()> {
447+
let array = chunked_array();
448+
let source = array.clone();
449+
let mut ctx = SESSION.create_execution_ctx();
450+
let count = Count.bind(NumericalAggregateOpts::skip_nans());
451+
array.aggregations().compute_result(&count, &mut ctx)?;
452+
453+
let advanced = array.with_next_builder_slot(1);
454+
assert_eq!(
455+
advanced.aggregations().get_result_as::<u64>(&count)?,
456+
Precision::Exact(9)
457+
);
458+
assert_eq!(
459+
source.aggregations().get_result_as::<u64>(&count)?,
460+
Precision::Exact(9)
461+
);
462+
assert_arrays_eq!(advanced, source, &mut ctx);
463+
Ok(())
464+
}

‎vortex-array/src/arrays/dict/compute/slice.rs‎

Lines changed: 3 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,8 @@ use crate::arrays::DictArray;
1616
use crate::arrays::Primitive;
1717
use crate::arrays::dict::DictArraySlotsExt;
1818
use crate::arrays::slice::SliceReduce;
19-
use crate::expr::stats::Precision;
20-
use crate::expr::stats::Stat;
19+
use crate::arrays::slice::inherit_slice_results;
2120
use crate::scalar::Scalar;
22-
use crate::scalar::ScalarValue;
2321

2422
impl SliceReduce for Dict {
2523
fn slice(array: ArrayView<'_, Self>, range: Range<usize>) -> VortexResult<Option<ArrayRef>> {
@@ -30,8 +28,8 @@ impl SliceReduce for Dict {
3028
let sliced_code = if let Some(codes) = array.codes().as_typed::<Primitive>() {
3129
let sliced_code = <Primitive as SliceReduce>::slice(codes, range)?
3230
.vortex_expect("Primitive SliceReduce should always return Some");
33-
// Because we specialize the primitive branch here, we have to make sure to handle the stat inheritance
34-
inherit_slice_stats(array.codes(), &sliced_code);
31+
// The specialized primitive path bypasses ArrayRef::slice and its propagation rules.
32+
inherit_slice_results(array.codes(), &sliced_code);
3533
sliced_code
3634
} else {
3735
array.codes().slice(range)?
@@ -49,24 +47,6 @@ impl SliceReduce for Dict {
4947
}
5048
}
5149

52-
fn inherit_slice_stats(source: &ArrayRef, sliced: &ArrayRef) {
53-
source.statistics().with_iter(|iter| {
54-
sliced
55-
.statistics()
56-
.inherit(iter.filter(|(stat, value)| is_inheritable_true_slice_stat(*stat, value)));
57-
});
58-
}
59-
60-
fn is_inheritable_true_slice_stat(stat: Stat, value: &Precision<ScalarValue>) -> bool {
61-
matches!(
62-
stat,
63-
Stat::IsConstant | Stat::IsSorted | Stat::IsStrictSorted
64-
) && value
65-
.as_ref()
66-
.as_exact()
67-
.is_some_and(|value| matches!(value, ScalarValue::Bool(true)))
68-
}
69-
7050
fn slice_constant_code(
7151
array: ArrayView<'_, Dict>,
7252
code: &Scalar,

‎vortex-array/src/arrays/dict/take.rs‎

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@ use crate::ArrayRef;
88
use crate::Canonical;
99
use crate::ExecutionCtx;
1010
use crate::IntoArray;
11+
use crate::aggregate_fn::fns::is_constant::IS_CONSTANT;
12+
use crate::aggregate_fn::fns::max::MAX_SKIP_NANS;
13+
use crate::aggregate_fn::fns::min::MIN_SKIP_NANS;
1114
use crate::array::ArrayView;
1215
use crate::array::VTable;
1316
use crate::arrays::ConstantArray;
1417
use crate::arrays::dict::DictArraySlotsExt;
1518
use crate::expr::stats::Precision;
16-
use crate::expr::stats::Stat;
17-
use crate::expr::stats::StatsProvider;
18-
use crate::expr::stats::StatsProviderExt;
1919
use crate::kernel::ExecuteParentKernel;
2020
use crate::matcher::Matcher;
2121
use crate::optimizer::rules::ArrayParentReduceRule;
@@ -100,7 +100,7 @@ where
100100
}
101101
let result = <V as TakeReduce>::take(array, parent.codes())?;
102102
if let Some(taken) = &result {
103-
propagate_take_stats(array.array(), taken, parent.codes())?;
103+
propagate_take_results(array.array(), taken, parent.codes())?;
104104
}
105105
Ok(result)
106106
}
@@ -131,13 +131,13 @@ where
131131
}
132132
let result = <V as TakeExecute>::take(array, parent.codes(), ctx)?;
133133
if let Some(taken) = &result {
134-
propagate_take_stats(array.array(), taken, parent.codes())?;
134+
propagate_take_results(array.array(), taken, parent.codes())?;
135135
}
136136
Ok(result)
137137
}
138138
}
139139

140-
pub(crate) fn propagate_take_stats(
140+
pub(crate) fn propagate_take_results(
141141
source: &ArrayRef,
142142
target: &ArrayRef,
143143
indices: &ArrayRef,
@@ -146,21 +146,24 @@ pub(crate) fn propagate_take_stats(
146146
indices.validity()?,
147147
Validity::NonNullable | Validity::AllValid
148148
);
149-
if indices_all_valid
150-
&& source.statistics().get_as::<bool>(Stat::IsConstant) == Precision::Exact(true)
149+
if !target.is_empty()
150+
&& indices_all_valid
151+
&& source.aggregations().get_result(&IS_CONSTANT) == Precision::Exact(true.into())
151152
{
152153
target
153-
.statistics()
154-
.set(Stat::IsConstant, Precision::exact(true));
154+
.aggregations()
155+
.insert_result(IS_CONSTANT.clone(), Precision::Exact(true.into()));
155156
}
156-
for stat in [Stat::Min, Stat::Max] {
157-
let value = source
158-
.statistics()
159-
.get(stat)
160-
.into_inexact()
161-
.and_then(Scalar::into_value);
162-
if !value.is_absent() && !target.statistics().get(stat).is_exact() {
163-
target.statistics().set(stat, value);
157+
for aggregate in [&*MIN_SKIP_NANS, &*MAX_SKIP_NANS] {
158+
let value = source.aggregations().get_result(aggregate).into_inexact();
159+
if value
160+
.as_ref()
161+
.into_inner()
162+
.is_some_and(|value| !value.is_null())
163+
{
164+
target
165+
.aggregations()
166+
.insert_result(aggregate.clone(), value);
164167
}
165168
}
166169
Ok(())

0 commit comments

Comments
 (0)