Skip to content

Commit e7c69fb

Browse files
committed
Use aggregate results in core statistics helpers
Core helpers and stat expressions should share the aggregate cache without requiring a fixed Stat slot. Preserve metadata-only lookup and recover partials through each aggregate's explicit contract, including fused MinMax results. Signed-off-by: Connor Tsui <[email protected]>
1 parent 4e9cde3 commit e7c69fb

13 files changed

Lines changed: 474 additions & 168 deletions

File tree

‎vortex-array/src/aggregate_fn/fns/count/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ impl AggregateFnVTable for Count {
119119
let mut count = batch.valid_count(ctx)? as u64;
120120
// NaN values are excluded from the count of a float input when they are skipped.
121121
if args.options.skip_nans && args.dtype.is_float() {
122-
// `nan_count` shortcircuits on an exact `Stat::NaNCount` before scanning the batch.
122+
// `nan_count` uses an exact cached aggregate result before scanning the batch.
123123
count = count.saturating_sub(nan_count(batch, ctx)? as u64);
124124
}
125125
*state += count;

‎vortex-array/src/aggregate_fn/fns/is_constant/mod.rs‎

Lines changed: 34 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ pub mod primitive;
1111
mod struct_;
1212
mod varbin;
1313

14+
use std::sync::LazyLock;
15+
1416
use vortex_error::VortexExpect;
1517
use vortex_error::VortexResult;
1618
use vortex_error::vortex_bail;
@@ -33,9 +35,15 @@ use crate::IntoArray;
3335
use crate::aggregate_fn::Accumulator;
3436
use crate::aggregate_fn::AggregateArgs;
3537
use crate::aggregate_fn::AggregateFnId;
38+
use crate::aggregate_fn::AggregateFnRef;
3639
use crate::aggregate_fn::AggregateFnVTable;
40+
use crate::aggregate_fn::AggregateFnVTableExt;
3741
use crate::aggregate_fn::DynAccumulator;
3842
use crate::aggregate_fn::EmptyOptions;
43+
use crate::aggregate_fn::fns::max::MAX_SKIP_NANS;
44+
use crate::aggregate_fn::fns::min::MIN_SKIP_NANS;
45+
use crate::aggregate_fn::fns::nan_count::NAN_COUNT;
46+
use crate::aggregate_fn::fns::nan_count::NanCount;
3947
use crate::arrays::Constant;
4048
use crate::arrays::Null;
4149
use crate::builtins::ArrayBuiltins;
@@ -44,9 +52,6 @@ use crate::dtype::FieldNames;
4452
use crate::dtype::Nullability;
4553
use crate::dtype::StructFields;
4654
use crate::expr::stats::Precision;
47-
use crate::expr::stats::Stat;
48-
use crate::expr::stats::StatsProvider;
49-
use crate::expr::stats::StatsProviderExt;
5055
use crate::scalar::Scalar;
5156
use crate::scalar_fn::fns::operators::Operator;
5257

@@ -81,17 +86,15 @@ fn arrays_value_equal(a: &ArrayRef, b: &ArrayRef, ctx: &mut ExecutionCtx) -> Vor
8186
Ok(eq_result.true_count() == valid_count)
8287
}
8388

89+
static IS_CONSTANT: LazyLock<AggregateFnRef> = LazyLock::new(|| IsConstant.bind(EmptyOptions));
90+
8491
/// Compute whether an array has constant values.
8592
///
86-
/// An array is constant IFF at least one of the following conditions apply:
87-
/// 1. It has at least one element (**Note** - an empty array isn't constant).
88-
/// 2. It's encoded as a [`ConstantArray`](crate::arrays::ConstantArray) or [`NullArray`](crate::arrays::NullArray)
89-
/// 3. Has an exact statistic attached to it, saying its constant.
90-
/// 4. Is all invalid.
91-
/// 5. Is all valid AND has minimum and maximum statistics that are equal.
93+
/// Empty arrays are not constant. Cached results, constant encodings, validity facts, and equal
94+
/// cached extrema can avoid scanning the values.
9295
pub fn is_constant(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
93-
// Short-circuit using cached array statistics.
94-
if let Precision::Exact(value) = array.statistics().get_as::<bool>(Stat::IsConstant) {
96+
// Short-circuit using the cached aggregate result.
97+
if let Precision::Exact(value) = array.aggregations().get_result_as::<bool>(&IS_CONSTANT)? {
9598
return Ok(value);
9699
}
97100

@@ -103,24 +106,24 @@ pub fn is_constant(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<boo
103106
// Array of length 1 is always constant.
104107
if array.len() == 1 {
105108
array
106-
.statistics()
107-
.set(Stat::IsConstant, Precision::Exact(true.into()));
109+
.aggregations()
110+
.insert_result(IS_CONSTANT.clone(), Precision::Exact(true.into()));
108111
return Ok(true);
109112
}
110113

111114
// Constant and null arrays are always constant.
112115
if array.is::<Constant>() || array.is::<Null>() {
113116
array
114-
.statistics()
115-
.set(Stat::IsConstant, Precision::Exact(true.into()));
117+
.aggregations()
118+
.insert_result(IS_CONSTANT.clone(), Precision::Exact(true.into()));
116119
return Ok(true);
117120
}
118121

119122
let all_invalid = array.all_invalid(ctx)?;
120123
if all_invalid {
121124
array
122-
.statistics()
123-
.set(Stat::IsConstant, Precision::Exact(true.into()));
125+
.aggregations()
126+
.insert_result(IS_CONSTANT.clone(), Precision::Exact(true.into()));
124127
return Ok(true);
125128
}
126129

@@ -129,24 +132,26 @@ pub fn is_constant(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<boo
129132
// If we have some nulls but not all nulls, array can't be constant.
130133
if !all_valid && !all_invalid {
131134
array
132-
.statistics()
133-
.set(Stat::IsConstant, Precision::Exact(false.into()));
135+
.aggregations()
136+
.insert_result(IS_CONSTANT.clone(), Precision::Exact(false.into()));
134137
return Ok(false);
135138
}
136139

137140
// We already know here that the array is all valid, so we check for min/max stats.
138-
let min_stat = array.statistics().get(Stat::Min);
139-
let max_stat = array.statistics().get(Stat::Max);
141+
let min_stat = array.aggregations().get_result(&MIN_SKIP_NANS);
142+
let max_stat = array.aggregations().get_result(&MAX_SKIP_NANS);
140143

141144
if let Precision::Exact(min) = min_stat.as_ref()
142145
&& let Precision::Exact(max) = max_stat.as_ref()
143146
&& min == max
144-
&& (Stat::NaNCount.dtype(array.dtype()).is_none()
145-
|| array.statistics().get_as::<u64>(Stat::NaNCount) == Precision::exact(0u64))
147+
&& (NanCount
148+
.return_dtype(&EmptyOptions, array.dtype())
149+
.is_none()
150+
|| array.aggregations().get_result_as::<u64>(&NAN_COUNT)? == Precision::exact(0u64))
146151
{
147152
array
148-
.statistics()
149-
.set(Stat::IsConstant, Precision::Exact(true.into()));
153+
.aggregations()
154+
.insert_result(IS_CONSTANT.clone(), Precision::Exact(true.into()));
150155
return Ok(true);
151156
}
152157

@@ -166,10 +171,10 @@ pub fn is_constant(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<boo
166171

167172
let result = result_scalar.as_bool().value().unwrap_or(false);
168173

169-
// Cache the computed is_constant as a statistic.
174+
// Retain the finalized result without retaining the comparison value.
170175
array
171-
.statistics()
172-
.set(Stat::IsConstant, Precision::Exact(result.into()));
176+
.aggregations()
177+
.insert_result(IS_CONSTANT.clone(), Precision::Exact(result.into()));
173178

174179
Ok(result)
175180
}
@@ -248,8 +253,7 @@ impl IsConstantPartial {
248253
}
249254
}
250255

251-
static NAMES: std::sync::LazyLock<FieldNames> =
252-
std::sync::LazyLock::new(|| FieldNames::from(["is_constant", "value"]));
256+
static NAMES: LazyLock<FieldNames> = LazyLock::new(|| FieldNames::from(["is_constant", "value"]));
253257

254258
pub fn make_is_constant_partial_dtype(element_dtype: &DType) -> DType {
255259
DType::Struct(

‎vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs‎

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ mod varbin;
1010
use std::fmt;
1111
use std::fmt::Display;
1212
use std::fmt::Formatter;
13+
use std::sync::LazyLock;
1314

1415
use vortex_error::VortexExpect;
1516
use vortex_error::VortexResult;
@@ -29,7 +30,9 @@ use crate::IntoArray;
2930
use crate::aggregate_fn::Accumulator;
3031
use crate::aggregate_fn::AggregateArgs;
3132
use crate::aggregate_fn::AggregateFnId;
33+
use crate::aggregate_fn::AggregateFnRef;
3234
use crate::aggregate_fn::AggregateFnVTable;
35+
use crate::aggregate_fn::AggregateFnVTableExt;
3336
use crate::aggregate_fn::DynAccumulator;
3437
use crate::arrays::Constant;
3538
use crate::arrays::Null;
@@ -39,8 +42,6 @@ use crate::dtype::FieldNames;
3942
use crate::dtype::Nullability;
4043
use crate::dtype::StructFields;
4144
use crate::expr::stats::Precision;
42-
use crate::expr::stats::Stat;
43-
use crate::expr::stats::StatsProviderExt;
4445
use crate::scalar::Scalar;
4546

4647
/// Options for the `is_sorted` aggregate function.
@@ -56,6 +57,11 @@ impl Display for IsSortedOptions {
5657
}
5758
}
5859

60+
static IS_SORTED: LazyLock<AggregateFnRef> =
61+
LazyLock::new(|| IsSorted.bind(IsSortedOptions { strict: false }));
62+
static IS_STRICT_SORTED: LazyLock<AggregateFnRef> =
63+
LazyLock::new(|| IsSorted.bind(IsSortedOptions { strict: true }));
64+
5965
/// Compute whether an array is sorted in non-decreasing order.
6066
///
6167
/// Returns `true` for empty arrays and arrays of length 1.
@@ -73,14 +79,14 @@ pub fn is_strict_sorted(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResul
7379
}
7480

7581
fn is_sorted_impl(array: &ArrayRef, strict: bool, ctx: &mut ExecutionCtx) -> VortexResult<bool> {
76-
let stat = if strict {
77-
Stat::IsStrictSorted
82+
let aggregate = if strict {
83+
&*IS_STRICT_SORTED
7884
} else {
79-
Stat::IsSorted
85+
&*IS_SORTED
8086
};
8187

82-
// Short-circuit using cached array statistics.
83-
if let Precision::Exact(value) = array.statistics().get_as::<bool>(stat) {
88+
// Short-circuit using the cached aggregate result.
89+
if let Precision::Exact(value) = array.aggregations().get_result_as::<bool>(aggregate)? {
8490
return Ok(value);
8591
}
8692

@@ -137,26 +143,26 @@ fn is_sorted_impl(array: &ArrayRef, strict: bool, ctx: &mut ExecutionCtx) -> Vor
137143

138144
let result = result_scalar.as_bool().value().unwrap_or(false);
139145

140-
// Cache the computed result as statistics.
146+
// Retain the finalized flags without retaining the stream boundary values.
141147
cache_is_sorted(array, strict, result);
142148

143149
Ok(result)
144150
}
145151

146152
fn cache_is_sorted(array: &ArrayRef, strict: bool, result: bool) {
147-
let array_stats = array.statistics();
153+
let aggregations = array.aggregations();
148154
if strict {
149155
if result {
150-
array_stats.set(Stat::IsSorted, Precision::Exact(true.into()));
151-
array_stats.set(Stat::IsStrictSorted, Precision::Exact(true.into()));
156+
aggregations.insert_result(IS_SORTED.clone(), Precision::Exact(true.into()));
157+
aggregations.insert_result(IS_STRICT_SORTED.clone(), Precision::Exact(true.into()));
152158
} else {
153-
array_stats.set(Stat::IsStrictSorted, Precision::Exact(false.into()));
159+
aggregations.insert_result(IS_STRICT_SORTED.clone(), Precision::Exact(false.into()));
154160
}
155161
} else if result {
156-
array_stats.set(Stat::IsSorted, Precision::Exact(true.into()));
162+
aggregations.insert_result(IS_SORTED.clone(), Precision::Exact(true.into()));
157163
} else {
158-
array_stats.set(Stat::IsSorted, Precision::Exact(false.into()));
159-
array_stats.set(Stat::IsStrictSorted, Precision::Exact(false.into()));
164+
aggregations.insert_result(IS_SORTED.clone(), Precision::Exact(false.into()));
165+
aggregations.insert_result(IS_STRICT_SORTED.clone(), Precision::Exact(false.into()));
160166
}
161167
}
162168

@@ -220,9 +226,8 @@ impl IsSortedPartial {
220226
}
221227
}
222228

223-
static NAMES: std::sync::LazyLock<FieldNames> = std::sync::LazyLock::new(|| {
224-
FieldNames::from(["is_sorted", "strict", "first_value", "last_value"])
225-
});
229+
static NAMES: LazyLock<FieldNames> =
230+
LazyLock::new(|| FieldNames::from(["is_sorted", "strict", "first_value", "last_value"]));
226231

227232
pub fn make_is_sorted_partial_dtype(element_dtype: &DType) -> DType {
228233
DType::Struct(

‎vortex-array/src/aggregate_fn/fns/max/mod.rs‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4+
use std::sync::LazyLock;
5+
46
use vortex_error::VortexExpect;
57
use vortex_error::VortexResult;
68
use vortex_session::VortexSession;
@@ -15,20 +17,24 @@ use crate::aggregate_fn::AggregateFnId;
1517
use crate::aggregate_fn::AggregateFnRef;
1618
use crate::aggregate_fn::AggregateFnSatisfaction;
1719
use crate::aggregate_fn::AggregateFnVTable;
20+
use crate::aggregate_fn::AggregateFnVTableExt;
1821
use crate::aggregate_fn::NumericalAggregateOpts;
1922
use crate::aggregate_fn::fns::bounded_max::BoundedMax;
2023
use crate::aggregate_fn::fns::min_max::MinMax;
2124
use crate::aggregate_fn::fns::min_max::min_max;
2225
use crate::aggregate_fn::fns::min_max::nan_scalar;
2326
use crate::aggregate_fn::fns::min_max::scalar_is_nan;
27+
use crate::aggregate_fn::fns::nan_count::NAN_COUNT;
2428
use crate::dtype::DType;
2529
use crate::expr::stats::Precision;
26-
use crate::expr::stats::Stat;
27-
use crate::expr::stats::StatsProvider;
28-
use crate::expr::stats::StatsProviderExt;
2930
use crate::partial_ord::partial_max;
3031
use crate::scalar::Scalar;
3132

33+
pub(crate) static MAX_SKIP_NANS: LazyLock<AggregateFnRef> =
34+
LazyLock::new(|| Max.bind(NumericalAggregateOpts::skip_nans()));
35+
pub(crate) static MAX_INCLUDE_NANS: LazyLock<AggregateFnRef> =
36+
LazyLock::new(|| Max.bind(NumericalAggregateOpts::include_nans()));
37+
3238
/// Compute the maximum non-null value of an array.
3339
///
3440
/// NaN handling for float inputs is controlled by [`NumericalAggregateOpts`]: with `skip_nans` (the
@@ -195,11 +201,11 @@ impl AggregateFnVTable for Max {
195201
if args.options.skip_nans || !args.dtype.is_float() {
196202
return Ok(false);
197203
}
198-
match batch.statistics().get_as::<u64>(Stat::NaNCount) {
204+
match batch.aggregations().get_result_as::<u64>(&NAN_COUNT)? {
199205
Precision::Exact(0) => {
200206
// NaN-free batch: the cached NaN-skipping maximum (if any) is valid. `to_scalar`
201207
// re-casts to the result dtype, so the cached scalar can merge as-is.
202-
if let Some(max) = batch.statistics().get(Stat::Max).as_exact() {
208+
if let Some(max) = batch.aggregations().get_result(&MAX_SKIP_NANS).as_exact() {
203209
partial.merge(args, max);
204210
return Ok(true);
205211
}

‎vortex-array/src/aggregate_fn/fns/min/mod.rs‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4+
use std::sync::LazyLock;
5+
46
use vortex_error::VortexExpect;
57
use vortex_error::VortexResult;
68
use vortex_session::VortexSession;
@@ -15,20 +17,24 @@ use crate::aggregate_fn::AggregateFnId;
1517
use crate::aggregate_fn::AggregateFnRef;
1618
use crate::aggregate_fn::AggregateFnSatisfaction;
1719
use crate::aggregate_fn::AggregateFnVTable;
20+
use crate::aggregate_fn::AggregateFnVTableExt;
1821
use crate::aggregate_fn::NumericalAggregateOpts;
1922
use crate::aggregate_fn::fns::bounded_min::BoundedMin;
2023
use crate::aggregate_fn::fns::min_max::MinMax;
2124
use crate::aggregate_fn::fns::min_max::min_max;
2225
use crate::aggregate_fn::fns::min_max::nan_scalar;
2326
use crate::aggregate_fn::fns::min_max::scalar_is_nan;
27+
use crate::aggregate_fn::fns::nan_count::NAN_COUNT;
2428
use crate::dtype::DType;
2529
use crate::expr::stats::Precision;
26-
use crate::expr::stats::Stat;
27-
use crate::expr::stats::StatsProvider;
28-
use crate::expr::stats::StatsProviderExt;
2930
use crate::partial_ord::partial_min;
3031
use crate::scalar::Scalar;
3132

33+
pub(crate) static MIN_SKIP_NANS: LazyLock<AggregateFnRef> =
34+
LazyLock::new(|| Min.bind(NumericalAggregateOpts::skip_nans()));
35+
pub(crate) static MIN_INCLUDE_NANS: LazyLock<AggregateFnRef> =
36+
LazyLock::new(|| Min.bind(NumericalAggregateOpts::include_nans()));
37+
3238
/// Compute the minimum non-null value of an array.
3339
///
3440
/// NaN handling for float inputs is controlled by [`NumericalAggregateOpts`]: with `skip_nans` (the
@@ -195,11 +201,11 @@ impl AggregateFnVTable for Min {
195201
if args.options.skip_nans || !args.dtype.is_float() {
196202
return Ok(false);
197203
}
198-
match batch.statistics().get_as::<u64>(Stat::NaNCount) {
204+
match batch.aggregations().get_result_as::<u64>(&NAN_COUNT)? {
199205
Precision::Exact(0) => {
200206
// NaN-free batch: the cached NaN-skipping minimum (if any) is valid. `to_scalar`
201207
// re-casts to the result dtype, so the cached scalar can merge as-is.
202-
if let Some(min) = batch.statistics().get(Stat::Min).as_exact() {
208+
if let Some(min) = batch.aggregations().get_result(&MIN_SKIP_NANS).as_exact() {
203209
partial.merge(args, min);
204210
return Ok(true);
205211
}

0 commit comments

Comments
 (0)