44//! Generic adapter that defines an aggregate over one extension type by running an existing
55//! aggregate over the extension's storage.
66
7+ use std:: mem;
8+
79use vortex_error:: VortexResult ;
810use vortex_error:: vortex_bail;
911use vortex_error:: vortex_err;
@@ -70,30 +72,56 @@ impl<A: AggregateFnVTable, E: ExtVTable> ExtensionAggregate<A, E> {
7072 . map ( |ext_dtype| ext_dtype. storage_dtype ( ) )
7173 }
7274
73- /// Derive the storage aggregate's dtypes from this aggregate's, which share the partial and
74- /// return dtypes.
75- fn storage_dtypes ( & self , args : AggregateArgs < ' _ , A :: Options > ) -> VortexResult < AggregateDTypes > {
76- let storage_dtype = self . storage_dtype ( args. dtype ) . ok_or_else ( || {
75+ /// The arguments of the storage aggregate `A`.
76+ ///
77+ /// `A` shares this aggregate's options, return and partial dtypes, and its input dtype is
78+ /// borrowed from the extension dtype, so this allocates nothing.
79+ fn storage_args < ' a > (
80+ & self ,
81+ args : AggregateArgs < ' a , A :: Options > ,
82+ ) -> VortexResult < AggregateArgs < ' a , A :: Options > > {
83+ let dtype = self . storage_dtype ( args. dtype ) . ok_or_else ( || {
7784 vortex_err ! (
7885 "Aggregate function {} expects extension type {}, got {}" ,
7986 self . id,
8087 self . ext. id( ) ,
8188 args. dtype
8289 )
8390 } ) ?;
84- Ok ( AggregateDTypes :: new (
85- storage_dtype. clone ( ) ,
86- args. return_dtype . clone ( ) ,
87- args. partial_dtype . clone ( ) ,
88- ) )
91+ Ok ( AggregateArgs { dtype, ..args } )
92+ }
93+
94+ /// Accumulate a storage batch into `state` through an [`Accumulator`] of `A`, so that the
95+ /// kernels and cached statistics of `A` on the storage encoding still apply.
96+ fn accumulate_storage (
97+ & self ,
98+ args : AggregateArgs < ' _ , A :: Options > ,
99+ state : & mut A :: Partial ,
100+ storage : & ArrayRef ,
101+ ctx : & mut ExecutionCtx ,
102+ ) -> VortexResult < ( ) > {
103+ let mut accumulator = Accumulator :: from_dtypes (
104+ self . aggregate . clone ( ) ,
105+ args. options . clone ( ) ,
106+ AggregateDTypes :: new (
107+ args. dtype . clone ( ) ,
108+ args. return_dtype . clone ( ) ,
109+ args. partial_dtype . clone ( ) ,
110+ ) ,
111+ ) ;
112+ accumulator. accumulate ( storage, ctx) ?;
113+ let Some ( batch) = accumulator. take_partial ( ) else {
114+ return Ok ( ( ) ) ;
115+ } ;
116+ let running = mem:: replace ( state, self . aggregate . empty_partial ( args) ?) ;
117+ * state = self . aggregate . merge_partials ( args, running, batch) ?;
118+ Ok ( ( ) )
89119 }
90120}
91121
92122impl < A : AggregateFnVTable , E : ExtVTable > AggregateFnVTable for ExtensionAggregate < A , E > {
93123 type Options = A :: Options ;
94- // The storage aggregate is held as a fully-fledged `Accumulator` so that storage batches
95- // consult the kernel registry and the cached statistics of `A`.
96- type Partial = Accumulator < A > ;
124+ type Partial = A :: Partial ;
97125
98126 fn id ( & self ) -> AggregateFnId {
99127 self . id
@@ -118,79 +146,78 @@ impl<A: AggregateFnVTable, E: ExtVTable> AggregateFnVTable for ExtensionAggregat
118146 }
119147
120148 fn empty_partial ( & self , args : AggregateArgs < ' _ , Self :: Options > ) -> VortexResult < Self :: Partial > {
121- Ok ( Accumulator :: from_dtypes (
122- self . aggregate . clone ( ) ,
123- args. options . clone ( ) ,
124- self . storage_dtypes ( args) ?,
125- ) )
149+ self . aggregate . empty_partial ( self . storage_args ( args) ?)
126150 }
127151
128152 fn partial_from_scalar (
129153 & self ,
130154 args : AggregateArgs < ' _ , Self :: Options > ,
131155 scalar : & Scalar ,
132156 ) -> VortexResult < Self :: Partial > {
133- let mut partial = self . empty_partial ( args) ?;
134- partial. combine_partials ( scalar) ?;
135- Ok ( partial)
157+ self . aggregate
158+ . partial_from_scalar ( self . storage_args ( args) ?, scalar)
136159 }
137160
138161 fn merge_partials (
139162 & self ,
140- _args : AggregateArgs < ' _ , Self :: Options > ,
141- mut first : Self :: Partial ,
142- mut second : Self :: Partial ,
163+ args : AggregateArgs < ' _ , Self :: Options > ,
164+ first : Self :: Partial ,
165+ second : Self :: Partial ,
143166 ) -> VortexResult < Self :: Partial > {
144- first . merge_from ( & mut second ) ? ;
145- Ok ( first)
167+ self . aggregate
168+ . merge_partials ( self . storage_args ( args ) ? , first, second )
146169 }
147170
148171 fn to_scalar (
149172 & self ,
150- _args : AggregateArgs < ' _ , Self :: Options > ,
173+ args : AggregateArgs < ' _ , Self :: Options > ,
151174 partial : & Self :: Partial ,
152175 ) -> VortexResult < Scalar > {
153- partial . partial_scalar ( )
176+ self . aggregate . to_scalar ( self . storage_args ( args ) ? , partial )
154177 }
155178
156179 fn is_saturated (
157180 & self ,
158- _args : AggregateArgs < ' _ , Self :: Options > ,
181+ args : AggregateArgs < ' _ , Self :: Options > ,
159182 partial : & Self :: Partial ,
160183 ) -> bool {
161- partial. is_saturated ( )
184+ self . storage_args ( args)
185+ . is_ok_and ( |args| self . aggregate . is_saturated ( args, partial) )
162186 }
163187
164188 /// Unwrap extension arrays to their storage before decoding, so the storage accumulator sees
165189 /// the storage encoding.
166190 fn try_accumulate (
167191 & self ,
168- _args : AggregateArgs < ' _ , Self :: Options > ,
192+ args : AggregateArgs < ' _ , Self :: Options > ,
169193 state : & mut Self :: Partial ,
170194 batch : & ArrayRef ,
171195 ctx : & mut ExecutionCtx ,
172196 ) -> VortexResult < bool > {
173197 let Some ( array) = batch. as_opt :: < Extension > ( ) else {
174198 return Ok ( false ) ;
175199 } ;
176- state . accumulate ( array. storage_array ( ) , ctx) ?;
200+ self . accumulate_storage ( self . storage_args ( args ) ? , state , array. storage_array ( ) , ctx) ?;
177201 Ok ( true )
178202 }
179203
180204 fn accumulate (
181205 & self ,
182- _args : AggregateArgs < ' _ , Self :: Options > ,
206+ args : AggregateArgs < ' _ , Self :: Options > ,
183207 state : & mut Self :: Partial ,
184208 batch : & Columnar ,
185209 ctx : & mut ExecutionCtx ,
186210 ) -> VortexResult < ( ) > {
211+ let args = self . storage_args ( args) ?;
187212 match batch {
188213 Columnar :: Canonical ( Canonical :: Extension ( array) ) => {
189- state . accumulate ( array. storage_array ( ) , ctx)
214+ self . accumulate_storage ( args , state , array. storage_array ( ) , ctx)
190215 }
191216 Columnar :: Constant ( constant) => {
192217 let storage = constant. scalar ( ) . as_extension ( ) . to_storage_scalar ( ) ;
193- state. accumulate (
218+ self . accumulate_storage (
219+ args,
220+ state,
194221 & ConstantArray :: new ( storage, constant. len ( ) ) . into_array ( ) ,
195222 ctx,
196223 )
@@ -210,16 +237,16 @@ impl<A: AggregateFnVTable, E: ExtVTable> AggregateFnVTable for ExtensionAggregat
210237 args : AggregateArgs < ' _ , Self :: Options > ,
211238 states : ArrayRef ,
212239 ) -> VortexResult < ArrayRef > {
213- let dtypes = self . storage_dtypes ( args) ?;
214- self . aggregate . finalize ( dtypes. args ( args. options ) , states)
240+ self . aggregate . finalize ( self . storage_args ( args) ?, states)
215241 }
216242
217243 fn finalize_scalar (
218244 & self ,
219- _args : AggregateArgs < ' _ , Self :: Options > ,
245+ args : AggregateArgs < ' _ , Self :: Options > ,
220246 partial : & Self :: Partial ,
221247 ) -> VortexResult < Scalar > {
222- partial. final_scalar ( )
248+ self . aggregate
249+ . finalize_scalar ( self . storage_args ( args) ?, partial)
223250 }
224251}
225252
@@ -280,6 +307,40 @@ mod tests {
280307 Ok ( ( ) )
281308 }
282309
310+ #[ test]
311+ fn combines_flushed_partials ( ) -> VortexResult < ( ) > {
312+ let session = crate :: array_session ( ) ;
313+ let mut ctx = session. create_execution_ctx ( ) ;
314+ let ext_dtype = Timestamp :: new ( TimeUnit :: Milliseconds , Nullability :: Nullable ) . erased ( ) ;
315+ let batch = |values : [ Option < i64 > ; 2 ] | {
316+ ExtensionArray :: new (
317+ ext_dtype. clone ( ) ,
318+ PrimitiveArray :: from_option_iter ( values) . into_array ( ) ,
319+ )
320+ . into_array ( )
321+ } ;
322+ let accumulator = || {
323+ Accumulator :: try_new (
324+ ExtensionAggregate :: new ( Sum , Timestamp ) ,
325+ NumericalAggregateOpts :: default ( ) ,
326+ DType :: Extension ( ext_dtype. clone ( ) ) ,
327+ )
328+ } ;
329+
330+ let mut first = accumulator ( ) ?;
331+ first. accumulate ( & batch ( [ Some ( 3_000 ) , None ] ) , & mut ctx) ?;
332+ let mut second = accumulator ( ) ?;
333+ second. accumulate ( & batch ( [ Some ( 1_000 ) , Some ( 500 ) ] ) , & mut ctx) ?;
334+ first. combine_partials ( & second. flush ( ) ?) ?;
335+
336+ let mut expected = accumulator ( ) ?;
337+ expected. accumulate ( & batch ( [ Some ( 3_000 ) , None ] ) , & mut ctx) ?;
338+ expected. accumulate ( & batch ( [ Some ( 1_000 ) , Some ( 500 ) ] ) , & mut ctx) ?;
339+
340+ assert_eq ! ( first. finish( ) ?, expected. finish( ) ?) ;
341+ Ok ( ( ) )
342+ }
343+
283344 #[ test]
284345 fn rejects_other_extension_types ( ) {
285346 let date = DType :: Extension ( Date :: new ( TimeUnit :: Days , Nullability :: Nullable ) . erased ( ) ) ;
0 commit comments