Skip to content

Commit 241723d

Browse files
committed
refactor(aggregate): replace storage lifters with ExtensionAggregate<A, E>
Replace the StorageLifter trait and ExtensionDelegate with a single ExtensionAggregate<A, E> adapter, which defines aggregate A over extension type E by running A over E's storage. It is an aggregate of its own, with id `<A>[<E>]`, so supporting a new pair is one session registration: session.aggregate_fns().register(ExtensionAggregate::new(Sum, Timestamp)); Restore the original storage delegation in min_max, is_constant, is_sorted, uncompressed_size_in_bytes and the zoned bloom filter. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01Hp7ZGEG8mrip6qkAtK4iWD Signed-off-by: Claude <[email protected]>
1 parent 0683b35 commit 241723d

8 files changed

Lines changed: 171 additions & 457 deletions

File tree

Lines changed: 139 additions & 165 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4-
//! Generic adapter for aggregates over an extension dtype that are computed by running an
5-
//! aggregate over the extension's storage dtype.
4+
//! Generic adapter that defines an aggregate over one extension type by running an existing
5+
//! aggregate over the extension's storage.
66
77
use vortex_error::VortexResult;
88
use vortex_error::vortex_bail;
@@ -22,195 +22,105 @@ use crate::aggregate_fn::AggregateFnVTable;
2222
use crate::aggregate_fn::DynAccumulator;
2323
use crate::arrays::ConstantArray;
2424
use crate::arrays::Extension;
25-
use crate::arrays::ExtensionArray;
2625
use crate::arrays::extension::ExtensionArrayExt;
2726
use crate::dtype::DType;
28-
use crate::dtype::extension::ExtDTypeRef;
27+
use crate::dtype::extension::ExtVTable;
2928
use crate::scalar::Scalar;
3029

31-
type StorageOptions<L> = <<L as StorageLifter>::Storage as AggregateFnVTable>::Options;
32-
33-
/// Declare an aggregate over an extension dtype in terms of an aggregate over its storage dtype.
30+
/// The aggregate `A` applied to extension type `E` through its storage.
3431
///
35-
/// Wrap an implementation in [`ExtensionDelegate`] to obtain an [`AggregateFnVTable`]. Every batch
36-
/// is unwrapped to its storage array and accumulated by [`Self::Storage`], so storage-encoding
37-
/// kernels registered for the storage aggregate still fire. Partial states are the storage
38-
/// aggregate's partials; only the final result is lifted back with [`Self::lift_scalar`] and
39-
/// [`Self::lift_array`].
32+
/// Inputs must have an extension dtype of type `E`. Every batch is unwrapped to its storage array
33+
/// and accumulated by `A`, so storage-encoding kernels registered for `A` still fire. Partials and
34+
/// results are those of `A` over the storage dtype.
4035
///
41-
/// The defaults return the storage result unchanged, which suits aggregates whose result does not
42-
/// depend on the input dtype, e.g. `is_constant` or `is_sorted`.
43-
pub trait StorageLifter: 'static + Send + Sync + Clone {
44-
/// The aggregate computed over the storage dtype.
45-
type Storage: AggregateFnVTable;
46-
47-
/// Stable identifier of the lifted aggregate.
48-
///
49-
/// It must differ from the storage aggregate's id: the lifted partials and results have
50-
/// different dtypes, so kernels and cached statistics of one cannot stand in for the other.
51-
fn id(&self) -> AggregateFnId;
52-
53-
/// Construct the storage aggregate vtable.
54-
fn storage(&self) -> Self::Storage;
55-
56-
/// Compute the final storage result over a whole storage array.
57-
///
58-
/// Used by [`ExtensionDelegate::aggregate`]. Override it to reuse a storage entry point that
59-
/// consults cached statistics, e.g. `is_constant`.
60-
fn aggregate_storage(
61-
&self,
62-
options: &StorageOptions<Self>,
63-
storage: &ArrayRef,
64-
ctx: &mut ExecutionCtx,
65-
) -> VortexResult<Scalar> {
66-
let mut acc = Accumulator::try_new(self.storage(), options.clone(), storage.dtype().clone())?;
67-
acc.accumulate(storage, ctx)?;
68-
acc.finish()
69-
}
70-
71-
/// The return dtype of the lifted aggregate, given the storage aggregate's return dtype.
72-
fn lift_return_dtype(&self, ext_dtype: &ExtDTypeRef, storage_dtype: DType) -> Option<DType> {
73-
let _ = ext_dtype;
74-
Some(storage_dtype)
75-
}
76-
77-
/// Lift a storage result scalar into a result of the lifted return dtype.
78-
fn lift_scalar(&self, ext_dtype: &ExtDTypeRef, storage_result: Scalar) -> VortexResult<Scalar> {
79-
let _ = ext_dtype;
80-
Ok(storage_result)
81-
}
82-
83-
/// Lift an array of storage results into an array of the lifted return dtype.
84-
fn lift_array(
85-
&self,
86-
ext_dtype: &ExtDTypeRef,
87-
storage_results: ArrayRef,
88-
) -> VortexResult<ArrayRef> {
89-
let _ = ext_dtype;
90-
Ok(storage_results)
91-
}
92-
}
93-
94-
/// Adapter that exposes any [`StorageLifter`] as an [`AggregateFnVTable`] over extension dtypes.
36+
/// The adapter is an aggregate of its own, with id `<A>[<E>]`, so supporting a new pair is one
37+
/// registration:
38+
///
39+
/// ```
40+
/// # use vortex_array::aggregate_fn::ExtensionAggregate;
41+
/// # use vortex_array::aggregate_fn::fns::sum::Sum;
42+
/// # use vortex_array::aggregate_fn::session::AggregateFnSessionExt;
43+
/// # use vortex_array::extension::datetime::Timestamp;
44+
/// # let session = vortex_array::array_session();
45+
/// session.aggregate_fns().register(ExtensionAggregate::new(Sum, Timestamp));
46+
/// ```
47+
///
48+
/// Only register a pair when `A` over the storage is meaningful for `E`, e.g. when `E` orders and
49+
/// compares like its storage.
9550
#[derive(Clone, Debug)]
96-
pub struct ExtensionDelegate<L: StorageLifter>(pub L);
97-
98-
impl<L: StorageLifter> ExtensionDelegate<L> {
99-
/// Construct a new extension delegate vtable.
100-
pub fn new(lifter: L) -> Self {
101-
Self(lifter)
102-
}
51+
pub struct ExtensionAggregate<A: AggregateFnVTable, E: ExtVTable> {
52+
aggregate: A,
53+
ext: E,
54+
id: AggregateFnId,
55+
}
10356

104-
/// Aggregate an extension `array` into the lifted final result.
105-
///
106-
/// This is the entry point for aggregates that handle extension inputs inside their own
107-
/// `accumulate` by delegating to the storage array.
108-
pub fn aggregate(
109-
&self,
110-
options: &StorageOptions<L>,
111-
array: &ExtensionArray,
112-
ctx: &mut ExecutionCtx,
113-
) -> VortexResult<Scalar> {
114-
let storage_result = self
115-
.0
116-
.aggregate_storage(options, array.storage_array(), ctx)?;
117-
self.0.lift_scalar(array.ext_dtype(), storage_result)
57+
impl<A: AggregateFnVTable, E: ExtVTable> ExtensionAggregate<A, E> {
58+
/// Define `aggregate` over extension type `ext`.
59+
pub fn new(aggregate: A, ext: E) -> Self {
60+
let id = AggregateFnId::new(&format!("{}[{}]", aggregate.id(), ext.id()));
61+
Self { aggregate, ext, id }
11862
}
11963

120-
/// Accumulate an extension `array` straight into a partial state of the storage aggregate.
121-
///
122-
/// This suits aggregates whose partial state does not depend on the input dtype, e.g. a Bloom
123-
/// filter, which can then fold extension batches into the partial they already hold.
124-
pub fn accumulate_storage(
125-
&self,
126-
options: &StorageOptions<L>,
127-
partial: &mut <L::Storage as AggregateFnVTable>::Partial,
128-
array: &ExtensionArray,
129-
ctx: &mut ExecutionCtx,
130-
) -> VortexResult<()> {
131-
let storage = self.0.storage();
132-
let dtypes =
133-
AggregateDTypes::try_new(&storage, options, array.storage_array().dtype().clone())?;
134-
let args = dtypes.args(options);
135-
if storage.try_accumulate(args, partial, array.storage_array(), ctx)? {
136-
return Ok(());
137-
}
138-
let columnar = array.storage_array().clone().execute::<Columnar>(ctx)?;
139-
storage.accumulate(args, partial, &columnar, ctx)
64+
/// The storage dtype of `dtype`, if it is an extension dtype of type `E`.
65+
fn storage_dtype<'a>(&self, dtype: &'a DType) -> Option<&'a DType> {
66+
dtype
67+
.as_extension_opt()
68+
.filter(|ext_dtype| ext_dtype.is::<E>())
69+
.map(|ext_dtype| ext_dtype.storage_dtype())
14070
}
14171

142-
/// Derive the storage aggregate's dtypes from the lifted aggregate's.
143-
///
144-
/// The partial dtypes are shared, so only the storage return dtype is left to resolve.
145-
fn storage_dtypes<'a>(
146-
&self,
147-
args: AggregateArgs<'a, StorageOptions<L>>,
148-
) -> VortexResult<(&'a ExtDTypeRef, AggregateDTypes)> {
149-
let ext_dtype = ext_dtype(args.dtype)?;
150-
let storage = self.0.storage();
151-
let storage_dtype = ext_dtype.storage_dtype().clone();
152-
let return_dtype = storage
153-
.return_dtype(args.options, &storage_dtype)
154-
.ok_or_else(|| {
155-
vortex_err!(
156-
"Aggregate function {} cannot be applied to dtype {}",
157-
storage.id(),
158-
storage_dtype
159-
)
160-
})?;
161-
Ok((
162-
ext_dtype,
163-
AggregateDTypes::new(storage_dtype, return_dtype, args.partial_dtype.clone()),
72+
/// Derive the storage aggregate's dtypes from this aggregate's, which share the partial and
73+
/// return dtypes.
74+
fn storage_dtypes(&self, args: AggregateArgs<'_, A::Options>) -> VortexResult<AggregateDTypes> {
75+
let storage_dtype = self.storage_dtype(args.dtype).ok_or_else(|| {
76+
vortex_err!(
77+
"Aggregate function {} expects extension type {}, got {}",
78+
self.id,
79+
self.ext.id(),
80+
args.dtype
81+
)
82+
})?;
83+
Ok(AggregateDTypes::new(
84+
storage_dtype.clone(),
85+
args.return_dtype.clone(),
86+
args.partial_dtype.clone(),
16487
))
16588
}
16689
}
16790

168-
fn ext_dtype(dtype: &DType) -> VortexResult<&ExtDTypeRef> {
169-
dtype
170-
.as_extension_opt()
171-
.ok_or_else(|| vortex_err!("Extension delegate expects an extension dtype, got {dtype}"))
172-
}
173-
174-
impl<L: StorageLifter> AggregateFnVTable for ExtensionDelegate<L> {
175-
type Options = StorageOptions<L>;
91+
impl<A: AggregateFnVTable, E: ExtVTable> AggregateFnVTable for ExtensionAggregate<A, E> {
92+
type Options = A::Options;
17693
// The storage aggregate is held as a fully-fledged `Accumulator` so that storage batches
177-
// consult the kernel registry and the cached statistics of the storage aggregate.
178-
type Partial = Accumulator<L::Storage>;
94+
// consult the kernel registry and the cached statistics of `A`.
95+
type Partial = Accumulator<A>;
17996

18097
fn id(&self) -> AggregateFnId {
181-
self.0.id()
98+
self.id
18299
}
183100

184101
fn serialize(&self, options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
185-
self.0.storage().serialize(options)
102+
self.aggregate.serialize(options)
186103
}
187104

188105
fn deserialize(&self, metadata: &[u8], session: &VortexSession) -> VortexResult<Self::Options> {
189-
self.0.storage().deserialize(metadata, session)
106+
self.aggregate.deserialize(metadata, session)
190107
}
191108

192109
fn return_dtype(&self, options: &Self::Options, input_dtype: &DType) -> Option<DType> {
193-
let ext_dtype = input_dtype.as_extension_opt()?;
194-
let storage_return = self
195-
.0
196-
.storage()
197-
.return_dtype(options, ext_dtype.storage_dtype())?;
198-
self.0.lift_return_dtype(ext_dtype, storage_return)
110+
self.aggregate
111+
.return_dtype(options, self.storage_dtype(input_dtype)?)
199112
}
200113

201114
fn partial_dtype(&self, options: &Self::Options, input_dtype: &DType) -> Option<DType> {
202-
let ext_dtype = input_dtype.as_extension_opt()?;
203-
self.0
204-
.storage()
205-
.partial_dtype(options, ext_dtype.storage_dtype())
115+
self.aggregate
116+
.partial_dtype(options, self.storage_dtype(input_dtype)?)
206117
}
207118

208119
fn empty_partial(&self, args: AggregateArgs<'_, Self::Options>) -> VortexResult<Self::Partial> {
209-
let (_, dtypes) = self.storage_dtypes(args)?;
210120
Ok(Accumulator::from_dtypes(
211-
self.0.storage(),
121+
self.aggregate.clone(),
212122
args.options.clone(),
213-
dtypes,
123+
self.storage_dtypes(args)?,
214124
))
215125
}
216126

@@ -283,7 +193,8 @@ impl<L: StorageLifter> AggregateFnVTable for ExtensionDelegate<L> {
283193
}
284194
Columnar::Canonical(_) => {
285195
vortex_bail!(
286-
"Extension delegate expects an extension array, got {}",
196+
"Aggregate function {} expects an extension array, got {}",
197+
self.id,
287198
batch.dtype()
288199
)
289200
}
@@ -295,20 +206,83 @@ impl<L: StorageLifter> AggregateFnVTable for ExtensionDelegate<L> {
295206
args: AggregateArgs<'_, Self::Options>,
296207
states: ArrayRef,
297208
) -> VortexResult<ArrayRef> {
298-
let (ext_dtype, dtypes) = self.storage_dtypes(args)?;
299-
let storage_results = self
300-
.0
301-
.storage()
302-
.finalize(dtypes.args(args.options), states)?;
303-
self.0.lift_array(ext_dtype, storage_results)
209+
let dtypes = self.storage_dtypes(args)?;
210+
self.aggregate.finalize(dtypes.args(args.options), states)
304211
}
305212

306213
fn finalize_scalar(
307214
&self,
308-
args: AggregateArgs<'_, Self::Options>,
215+
_args: AggregateArgs<'_, Self::Options>,
309216
partial: &Self::Partial,
310217
) -> VortexResult<Scalar> {
311-
self.0
312-
.lift_scalar(ext_dtype(args.dtype)?, partial.final_scalar()?)
218+
partial.final_scalar()
219+
}
220+
}
221+
222+
#[cfg(test)]
223+
mod tests {
224+
use vortex_error::VortexResult;
225+
226+
use super::ExtensionAggregate;
227+
use crate::IntoArray;
228+
use crate::VortexSessionExecute;
229+
use crate::aggregate_fn::Accumulator;
230+
use crate::aggregate_fn::AggregateFnVTable;
231+
use crate::aggregate_fn::DynAccumulator;
232+
use crate::aggregate_fn::NumericalAggregateOpts;
233+
use crate::aggregate_fn::fns::sum::Sum;
234+
use crate::aggregate_fn::session::AggregateFnSessionExt;
235+
use crate::arrays::ExtensionArray;
236+
use crate::arrays::PrimitiveArray;
237+
use crate::dtype::DType;
238+
use crate::dtype::Nullability;
239+
use crate::extension::datetime::Date;
240+
use crate::extension::datetime::TimeUnit;
241+
use crate::extension::datetime::Timestamp;
242+
243+
#[test]
244+
fn sums_extension_through_storage() -> VortexResult<()> {
245+
let session = crate::array_session();
246+
let aggregate = ExtensionAggregate::new(Sum, Timestamp);
247+
session.aggregate_fns().register(aggregate.clone());
248+
assert!(
249+
session
250+
.aggregate_fns()
251+
.find_plugin(&aggregate.id())
252+
.is_some()
253+
);
254+
255+
let ext_dtype = Timestamp::new(TimeUnit::Milliseconds, Nullability::Nullable).erased();
256+
let storage =
257+
PrimitiveArray::from_option_iter([Some(3_000i64), None, Some(1_000)]).into_array();
258+
let array = ExtensionArray::new(ext_dtype.clone(), storage.clone()).into_array();
259+
260+
let mut ctx = session.create_execution_ctx();
261+
let mut acc = Accumulator::try_new(
262+
aggregate,
263+
NumericalAggregateOpts::default(),
264+
DType::Extension(ext_dtype),
265+
)?;
266+
acc.accumulate(&array, &mut ctx)?;
267+
268+
let mut expected = Accumulator::try_new(
269+
Sum,
270+
NumericalAggregateOpts::default(),
271+
storage.dtype().clone(),
272+
)?;
273+
expected.accumulate(&storage, &mut ctx)?;
274+
275+
assert_eq!(acc.finish()?, expected.finish()?);
276+
Ok(())
277+
}
278+
279+
#[test]
280+
fn rejects_other_extension_types() {
281+
let date = DType::Extension(Date::new(TimeUnit::Days, Nullability::Nullable).erased());
282+
assert!(
283+
ExtensionAggregate::new(Sum, Timestamp)
284+
.return_dtype(&NumericalAggregateOpts::default(), &date)
285+
.is_none()
286+
);
313287
}
314288
}

0 commit comments

Comments
 (0)