11// SPDX-License-Identifier: Apache-2.0
22// SPDX-FileCopyrightText: Copyright the Vortex contributors
33
4- //! Generic adapter for aggregates over an extension dtype that are computed by running an
5- //! aggregate over the extension's storage dtype .
4+ //! Generic adapter that defines an aggregate over one extension type by running an existing
5+ //! aggregate over the extension's storage.
66
77use vortex_error:: VortexResult ;
88use vortex_error:: vortex_bail;
@@ -22,195 +22,105 @@ use crate::aggregate_fn::AggregateFnVTable;
2222use crate :: aggregate_fn:: DynAccumulator ;
2323use crate :: arrays:: ConstantArray ;
2424use crate :: arrays:: Extension ;
25- use crate :: arrays:: ExtensionArray ;
2625use crate :: arrays:: extension:: ExtensionArrayExt ;
2726use crate :: dtype:: DType ;
28- use crate :: dtype:: extension:: ExtDTypeRef ;
27+ use crate :: dtype:: extension:: ExtVTable ;
2928use crate :: scalar:: Scalar ;
3029
31- type StorageOptions < L > = <<L as StorageLifter >:: Storage as AggregateFnVTable >:: Options ;
32-
33- /// Declare an aggregate over an extension dtype in terms of an aggregate over its storage dtype.
30+ /// The aggregate `A` applied to extension type `E` through its storage.
3431///
35- /// Wrap an implementation in [`ExtensionDelegate`] to obtain an [`AggregateFnVTable`]. Every batch
36- /// is unwrapped to its storage array and accumulated by [`Self::Storage`], so storage-encoding
37- /// kernels registered for the storage aggregate still fire. Partial states are the storage
38- /// aggregate's partials; only the final result is lifted back with [`Self::lift_scalar`] and
39- /// [`Self::lift_array`].
32+ /// Inputs must have an extension dtype of type `E`. Every batch is unwrapped to its storage array
33+ /// and accumulated by `A`, so storage-encoding kernels registered for `A` still fire. Partials and
34+ /// results are those of `A` over the storage dtype.
4035///
41- /// The defaults return the storage result unchanged, which suits aggregates whose result does not
42- /// depend on the input dtype, e.g. `is_constant` or `is_sorted`.
43- pub trait StorageLifter : ' static + Send + Sync + Clone {
44- /// The aggregate computed over the storage dtype.
45- type Storage : AggregateFnVTable ;
46-
47- /// Stable identifier of the lifted aggregate.
48- ///
49- /// It must differ from the storage aggregate's id: the lifted partials and results have
50- /// different dtypes, so kernels and cached statistics of one cannot stand in for the other.
51- fn id ( & self ) -> AggregateFnId ;
52-
53- /// Construct the storage aggregate vtable.
54- fn storage ( & self ) -> Self :: Storage ;
55-
56- /// Compute the final storage result over a whole storage array.
57- ///
58- /// Used by [`ExtensionDelegate::aggregate`]. Override it to reuse a storage entry point that
59- /// consults cached statistics, e.g. `is_constant`.
60- fn aggregate_storage (
61- & self ,
62- options : & StorageOptions < Self > ,
63- storage : & ArrayRef ,
64- ctx : & mut ExecutionCtx ,
65- ) -> VortexResult < Scalar > {
66- let mut acc = Accumulator :: try_new ( self . storage ( ) , options. clone ( ) , storage. dtype ( ) . clone ( ) ) ?;
67- acc. accumulate ( storage, ctx) ?;
68- acc. finish ( )
69- }
70-
71- /// The return dtype of the lifted aggregate, given the storage aggregate's return dtype.
72- fn lift_return_dtype ( & self , ext_dtype : & ExtDTypeRef , storage_dtype : DType ) -> Option < DType > {
73- let _ = ext_dtype;
74- Some ( storage_dtype)
75- }
76-
77- /// Lift a storage result scalar into a result of the lifted return dtype.
78- fn lift_scalar ( & self , ext_dtype : & ExtDTypeRef , storage_result : Scalar ) -> VortexResult < Scalar > {
79- let _ = ext_dtype;
80- Ok ( storage_result)
81- }
82-
83- /// Lift an array of storage results into an array of the lifted return dtype.
84- fn lift_array (
85- & self ,
86- ext_dtype : & ExtDTypeRef ,
87- storage_results : ArrayRef ,
88- ) -> VortexResult < ArrayRef > {
89- let _ = ext_dtype;
90- Ok ( storage_results)
91- }
92- }
93-
94- /// Adapter that exposes any [`StorageLifter`] as an [`AggregateFnVTable`] over extension dtypes.
36+ /// The adapter is an aggregate of its own, with id `<A>[<E>]`, so supporting a new pair is one
37+ /// registration:
38+ ///
39+ /// ```
40+ /// # use vortex_array::aggregate_fn::ExtensionAggregate;
41+ /// # use vortex_array::aggregate_fn::fns::sum::Sum;
42+ /// # use vortex_array::aggregate_fn::session::AggregateFnSessionExt;
43+ /// # use vortex_array::extension::datetime::Timestamp;
44+ /// # let session = vortex_array::array_session();
45+ /// session.aggregate_fns().register(ExtensionAggregate::new(Sum, Timestamp));
46+ /// ```
47+ ///
48+ /// Only register a pair when `A` over the storage is meaningful for `E`, e.g. when `E` orders and
49+ /// compares like its storage.
9550#[ derive( Clone , Debug ) ]
96- pub struct ExtensionDelegate < L : StorageLifter > ( pub L ) ;
97-
98- impl < L : StorageLifter > ExtensionDelegate < L > {
99- /// Construct a new extension delegate vtable.
100- pub fn new ( lifter : L ) -> Self {
101- Self ( lifter)
102- }
51+ pub struct ExtensionAggregate < A : AggregateFnVTable , E : ExtVTable > {
52+ aggregate : A ,
53+ ext : E ,
54+ id : AggregateFnId ,
55+ }
10356
104- /// Aggregate an extension `array` into the lifted final result.
105- ///
106- /// This is the entry point for aggregates that handle extension inputs inside their own
107- /// `accumulate` by delegating to the storage array.
108- pub fn aggregate (
109- & self ,
110- options : & StorageOptions < L > ,
111- array : & ExtensionArray ,
112- ctx : & mut ExecutionCtx ,
113- ) -> VortexResult < Scalar > {
114- let storage_result = self
115- . 0
116- . aggregate_storage ( options, array. storage_array ( ) , ctx) ?;
117- self . 0 . lift_scalar ( array. ext_dtype ( ) , storage_result)
57+ impl < A : AggregateFnVTable , E : ExtVTable > ExtensionAggregate < A , E > {
58+ /// Define `aggregate` over extension type `ext`.
59+ pub fn new ( aggregate : A , ext : E ) -> Self {
60+ let id = AggregateFnId :: new ( & format ! ( "{}[{}]" , aggregate. id( ) , ext. id( ) ) ) ;
61+ Self { aggregate, ext, id }
11862 }
11963
120- /// Accumulate an extension `array` straight into a partial state of the storage aggregate.
121- ///
122- /// This suits aggregates whose partial state does not depend on the input dtype, e.g. a Bloom
123- /// filter, which can then fold extension batches into the partial they already hold.
124- pub fn accumulate_storage (
125- & self ,
126- options : & StorageOptions < L > ,
127- partial : & mut <L :: Storage as AggregateFnVTable >:: Partial ,
128- array : & ExtensionArray ,
129- ctx : & mut ExecutionCtx ,
130- ) -> VortexResult < ( ) > {
131- let storage = self . 0 . storage ( ) ;
132- let dtypes =
133- AggregateDTypes :: try_new ( & storage, options, array. storage_array ( ) . dtype ( ) . clone ( ) ) ?;
134- let args = dtypes. args ( options) ;
135- if storage. try_accumulate ( args, partial, array. storage_array ( ) , ctx) ? {
136- return Ok ( ( ) ) ;
137- }
138- let columnar = array. storage_array ( ) . clone ( ) . execute :: < Columnar > ( ctx) ?;
139- storage. accumulate ( args, partial, & columnar, ctx)
64+ /// The storage dtype of `dtype`, if it is an extension dtype of type `E`.
65+ fn storage_dtype < ' a > ( & self , dtype : & ' a DType ) -> Option < & ' a DType > {
66+ dtype
67+ . as_extension_opt ( )
68+ . filter ( |ext_dtype| ext_dtype. is :: < E > ( ) )
69+ . map ( |ext_dtype| ext_dtype. storage_dtype ( ) )
14070 }
14171
142- /// Derive the storage aggregate's dtypes from the lifted aggregate's.
143- ///
144- /// The partial dtypes are shared, so only the storage return dtype is left to resolve.
145- fn storage_dtypes < ' a > (
146- & self ,
147- args : AggregateArgs < ' a , StorageOptions < L > > ,
148- ) -> VortexResult < ( & ' a ExtDTypeRef , AggregateDTypes ) > {
149- let ext_dtype = ext_dtype ( args. dtype ) ?;
150- let storage = self . 0 . storage ( ) ;
151- let storage_dtype = ext_dtype. storage_dtype ( ) . clone ( ) ;
152- let return_dtype = storage
153- . return_dtype ( args. options , & storage_dtype)
154- . ok_or_else ( || {
155- vortex_err ! (
156- "Aggregate function {} cannot be applied to dtype {}" ,
157- storage. id( ) ,
158- storage_dtype
159- )
160- } ) ?;
161- Ok ( (
162- ext_dtype,
163- AggregateDTypes :: new ( storage_dtype, return_dtype, args. partial_dtype . clone ( ) ) ,
72+ /// Derive the storage aggregate's dtypes from this aggregate's, which share the partial and
73+ /// return dtypes.
74+ fn storage_dtypes ( & self , args : AggregateArgs < ' _ , A :: Options > ) -> VortexResult < AggregateDTypes > {
75+ let storage_dtype = self . storage_dtype ( args. dtype ) . ok_or_else ( || {
76+ vortex_err ! (
77+ "Aggregate function {} expects extension type {}, got {}" ,
78+ self . id,
79+ self . ext. id( ) ,
80+ args. dtype
81+ )
82+ } ) ?;
83+ Ok ( AggregateDTypes :: new (
84+ storage_dtype. clone ( ) ,
85+ args. return_dtype . clone ( ) ,
86+ args. partial_dtype . clone ( ) ,
16487 ) )
16588 }
16689}
16790
168- fn ext_dtype ( dtype : & DType ) -> VortexResult < & ExtDTypeRef > {
169- dtype
170- . as_extension_opt ( )
171- . ok_or_else ( || vortex_err ! ( "Extension delegate expects an extension dtype, got {dtype}" ) )
172- }
173-
174- impl < L : StorageLifter > AggregateFnVTable for ExtensionDelegate < L > {
175- type Options = StorageOptions < L > ;
91+ impl < A : AggregateFnVTable , E : ExtVTable > AggregateFnVTable for ExtensionAggregate < A , E > {
92+ type Options = A :: Options ;
17693 // The storage aggregate is held as a fully-fledged `Accumulator` so that storage batches
177- // consult the kernel registry and the cached statistics of the storage aggregate .
178- type Partial = Accumulator < L :: Storage > ;
94+ // consult the kernel registry and the cached statistics of `A` .
95+ type Partial = Accumulator < A > ;
17996
18097 fn id ( & self ) -> AggregateFnId {
181- self . 0 . id ( )
98+ self . id
18299 }
183100
184101 fn serialize ( & self , options : & Self :: Options ) -> VortexResult < Option < Vec < u8 > > > {
185- self . 0 . storage ( ) . serialize ( options)
102+ self . aggregate . serialize ( options)
186103 }
187104
188105 fn deserialize ( & self , metadata : & [ u8 ] , session : & VortexSession ) -> VortexResult < Self :: Options > {
189- self . 0 . storage ( ) . deserialize ( metadata, session)
106+ self . aggregate . deserialize ( metadata, session)
190107 }
191108
192109 fn return_dtype ( & self , options : & Self :: Options , input_dtype : & DType ) -> Option < DType > {
193- let ext_dtype = input_dtype. as_extension_opt ( ) ?;
194- let storage_return = self
195- . 0
196- . storage ( )
197- . return_dtype ( options, ext_dtype. storage_dtype ( ) ) ?;
198- self . 0 . lift_return_dtype ( ext_dtype, storage_return)
110+ self . aggregate
111+ . return_dtype ( options, self . storage_dtype ( input_dtype) ?)
199112 }
200113
201114 fn partial_dtype ( & self , options : & Self :: Options , input_dtype : & DType ) -> Option < DType > {
202- let ext_dtype = input_dtype. as_extension_opt ( ) ?;
203- self . 0
204- . storage ( )
205- . partial_dtype ( options, ext_dtype. storage_dtype ( ) )
115+ self . aggregate
116+ . partial_dtype ( options, self . storage_dtype ( input_dtype) ?)
206117 }
207118
208119 fn empty_partial ( & self , args : AggregateArgs < ' _ , Self :: Options > ) -> VortexResult < Self :: Partial > {
209- let ( _, dtypes) = self . storage_dtypes ( args) ?;
210120 Ok ( Accumulator :: from_dtypes (
211- self . 0 . storage ( ) ,
121+ self . aggregate . clone ( ) ,
212122 args. options . clone ( ) ,
213- dtypes ,
123+ self . storage_dtypes ( args ) ? ,
214124 ) )
215125 }
216126
@@ -283,7 +193,8 @@ impl<L: StorageLifter> AggregateFnVTable for ExtensionDelegate<L> {
283193 }
284194 Columnar :: Canonical ( _) => {
285195 vortex_bail ! (
286- "Extension delegate expects an extension array, got {}" ,
196+ "Aggregate function {} expects an extension array, got {}" ,
197+ self . id,
287198 batch. dtype( )
288199 )
289200 }
@@ -295,20 +206,83 @@ impl<L: StorageLifter> AggregateFnVTable for ExtensionDelegate<L> {
295206 args : AggregateArgs < ' _ , Self :: Options > ,
296207 states : ArrayRef ,
297208 ) -> VortexResult < ArrayRef > {
298- let ( ext_dtype, dtypes) = self . storage_dtypes ( args) ?;
299- let storage_results = self
300- . 0
301- . storage ( )
302- . finalize ( dtypes. args ( args. options ) , states) ?;
303- self . 0 . lift_array ( ext_dtype, storage_results)
209+ let dtypes = self . storage_dtypes ( args) ?;
210+ self . aggregate . finalize ( dtypes. args ( args. options ) , states)
304211 }
305212
306213 fn finalize_scalar (
307214 & self ,
308- args : AggregateArgs < ' _ , Self :: Options > ,
215+ _args : AggregateArgs < ' _ , Self :: Options > ,
309216 partial : & Self :: Partial ,
310217 ) -> VortexResult < Scalar > {
311- self . 0
312- . lift_scalar ( ext_dtype ( args. dtype ) ?, partial. final_scalar ( ) ?)
218+ partial. final_scalar ( )
219+ }
220+ }
221+
222+ #[ cfg( test) ]
223+ mod tests {
224+ use vortex_error:: VortexResult ;
225+
226+ use super :: ExtensionAggregate ;
227+ use crate :: IntoArray ;
228+ use crate :: VortexSessionExecute ;
229+ use crate :: aggregate_fn:: Accumulator ;
230+ use crate :: aggregate_fn:: AggregateFnVTable ;
231+ use crate :: aggregate_fn:: DynAccumulator ;
232+ use crate :: aggregate_fn:: NumericalAggregateOpts ;
233+ use crate :: aggregate_fn:: fns:: sum:: Sum ;
234+ use crate :: aggregate_fn:: session:: AggregateFnSessionExt ;
235+ use crate :: arrays:: ExtensionArray ;
236+ use crate :: arrays:: PrimitiveArray ;
237+ use crate :: dtype:: DType ;
238+ use crate :: dtype:: Nullability ;
239+ use crate :: extension:: datetime:: Date ;
240+ use crate :: extension:: datetime:: TimeUnit ;
241+ use crate :: extension:: datetime:: Timestamp ;
242+
243+ #[ test]
244+ fn sums_extension_through_storage ( ) -> VortexResult < ( ) > {
245+ let session = crate :: array_session ( ) ;
246+ let aggregate = ExtensionAggregate :: new ( Sum , Timestamp ) ;
247+ session. aggregate_fns ( ) . register ( aggregate. clone ( ) ) ;
248+ assert ! (
249+ session
250+ . aggregate_fns( )
251+ . find_plugin( & aggregate. id( ) )
252+ . is_some( )
253+ ) ;
254+
255+ let ext_dtype = Timestamp :: new ( TimeUnit :: Milliseconds , Nullability :: Nullable ) . erased ( ) ;
256+ let storage =
257+ PrimitiveArray :: from_option_iter ( [ Some ( 3_000i64 ) , None , Some ( 1_000 ) ] ) . into_array ( ) ;
258+ let array = ExtensionArray :: new ( ext_dtype. clone ( ) , storage. clone ( ) ) . into_array ( ) ;
259+
260+ let mut ctx = session. create_execution_ctx ( ) ;
261+ let mut acc = Accumulator :: try_new (
262+ aggregate,
263+ NumericalAggregateOpts :: default ( ) ,
264+ DType :: Extension ( ext_dtype) ,
265+ ) ?;
266+ acc. accumulate ( & array, & mut ctx) ?;
267+
268+ let mut expected = Accumulator :: try_new (
269+ Sum ,
270+ NumericalAggregateOpts :: default ( ) ,
271+ storage. dtype ( ) . clone ( ) ,
272+ ) ?;
273+ expected. accumulate ( & storage, & mut ctx) ?;
274+
275+ assert_eq ! ( acc. finish( ) ?, expected. finish( ) ?) ;
276+ Ok ( ( ) )
277+ }
278+
279+ #[ test]
280+ fn rejects_other_extension_types ( ) {
281+ let date = DType :: Extension ( Date :: new ( TimeUnit :: Days , Nullability :: Nullable ) . erased ( ) ) ;
282+ assert ! (
283+ ExtensionAggregate :: new( Sum , Timestamp )
284+ . return_dtype( & NumericalAggregateOpts :: default ( ) , & date)
285+ . is_none( )
286+ ) ;
313287 }
314288}
0 commit comments