11// SPDX-License-Identifier: Apache-2.0
22// SPDX-FileCopyrightText: Copyright the Vortex contributors
33
4- //! [`ArrayPlugin`] implementation for [`ALP`].
4+ //! [`ArrayPlugin`] implementations for [`ALP`].
5+ //!
6+ //! [`ALP`] itself reads and writes the current `vortex.alp` form. [`ALPPatchedPlugin`] reads an
7+ //! ALP node with interior patches as a `PatchedArray` wrapping a patchless ALP array, which keeps
8+ //! previously written datasets readable at no cost.
59
610use prost:: Message ;
711use vortex_array:: Array ;
@@ -14,16 +18,19 @@ use vortex_array::ArraySerialization;
1418use vortex_array:: IntoArray ;
1519use vortex_array:: VTable ;
1620use vortex_array:: VortexSessionExecute ;
21+ use vortex_array:: arrays:: Patched ;
1722use vortex_array:: dtype:: DType ;
1823use vortex_array:: dtype:: PType ;
1924use vortex_array:: patches:: Patches ;
2025use vortex_error:: VortexResult ;
2126use vortex_error:: vortex_bail;
2227use vortex_error:: vortex_ensure_eq;
28+ use vortex_error:: vortex_err;
2329use vortex_session:: VortexSession ;
2430
2531use super :: ALP ;
2632use super :: ALPArrayExt ;
33+ use super :: ALPArrayOwnedExt ;
2734use super :: ALPData ;
2835use super :: ALPMetadata ;
2936use crate :: alp:: Exponents ;
@@ -125,3 +132,226 @@ impl ArrayPlugin for ALP {
125132 . into_array ( ) )
126133 }
127134}
135+
136+ /// Custom deserialization plugin that converts an ALP array with interior
137+ /// patches into a PatchedArray holding an ALP array.
138+ #[ derive( Debug , Clone ) ]
139+ pub ( crate ) struct ALPPatchedPlugin ;
140+
141+ impl ArrayPlugin for ALPPatchedPlugin {
142+ fn id ( & self ) -> ArrayId {
143+ // We reuse the existing `ALP` ID so that we can take over its
144+ // deserialization pathway.
145+ // TODO(joe): dedup method name
146+ VTable :: id ( & ALP )
147+ }
148+
149+ fn serialize (
150+ & self ,
151+ array : & ArrayRef ,
152+ session : & VortexSession ,
153+ ) -> VortexResult < Option < ArraySerialization > > {
154+ // Delegate to ALP's metadata serde
155+ ArrayPlugin :: serialize ( & ALP , array, session)
156+ }
157+
158+ fn deserialize (
159+ & self ,
160+ parts : ArrayDeserialization < ' _ > ,
161+ session : & VortexSession ,
162+ ) -> VortexResult < ArrayRef > {
163+ vortex_ensure_eq ! (
164+ parts. serialized_id,
165+ self . id( ) ,
166+ "ALP plugin does not recognize serialized ID"
167+ ) ;
168+ let alp_array = ArrayPlugin :: deserialize ( & ALP , parts, session) ?
169+ . try_downcast :: < ALP > ( )
170+ . map_err ( |_| vortex_err ! ( "ALP plugin returned an array that is not vortex.alp" ) ) ?;
171+
172+ // Check if there are interior patches to externalize.
173+ let Some ( patches) = alp_array. patches ( ) else {
174+ return Ok ( alp_array. into_array ( ) ) ;
175+ } ;
176+
177+ // Extract components and create a new ALP array without patches.
178+ let ( encoded, exponents, _) = alp_array. into_parts ( ) ;
179+
180+ let alp_without_patches = ALP :: try_new ( encoded, exponents, None ) ?. into_array ( ) ;
181+
182+ let patched = Patched :: from_array_and_patches (
183+ alp_without_patches,
184+ & patches,
185+ & mut session. create_execution_ctx ( ) ,
186+ ) ?;
187+
188+ Ok ( patched. into_array ( ) )
189+ }
190+
191+ fn is_supported_encoding ( & self , id : & ArrayId ) -> bool {
192+ // TODO(joe): dedup method name
193+ id == VTable :: id ( & Patched ) || id == VTable :: id ( & ALP )
194+ }
195+ }
196+
197+ #[ cfg( test) ]
198+ mod tests {
199+ use std:: f64:: consts:: PI ;
200+ use std:: sync:: LazyLock ;
201+
202+ use vortex_array:: ArrayDeserialization ;
203+ use vortex_array:: ArrayPlugin ;
204+ use vortex_array:: IntoArray ;
205+ use vortex_array:: VortexSessionExecute ;
206+ use vortex_array:: arrays:: PatchedArray ;
207+ use vortex_array:: arrays:: PrimitiveArray ;
208+ use vortex_array:: arrays:: patched:: PatchedArraySlotsExt ;
209+ use vortex_array:: assert_arrays_eq;
210+ use vortex_array:: buffer:: BufferHandle ;
211+ use vortex_array:: session:: ArraySessionExt ;
212+ use vortex_error:: VortexResult ;
213+ use vortex_error:: vortex_err;
214+ use vortex_session:: VortexSession ;
215+
216+ use super :: ALPPatchedPlugin ;
217+ use crate :: ALP ;
218+ use crate :: ALPArray ;
219+ use crate :: ALPArrayExt ;
220+ use crate :: alp_encode;
221+
222+ static SESSION : LazyLock < VortexSession > = LazyLock :: new ( || {
223+ let session = VortexSession :: empty ( ) ;
224+ session. arrays ( ) . register ( ALPPatchedPlugin ) ;
225+ session
226+ } ) ;
227+
228+ #[ test]
229+ fn test_decode_alp_patches ( ) -> VortexResult < ( ) > {
230+ // Create values where some don't encode cleanly with ALP, causing patches.
231+ // PI doesn't encode cleanly.
232+ let values: Vec < f64 > = ( 0 ..100 )
233+ . map ( |i| if i % 4 == 3 { PI } else { i as f64 } )
234+ . collect ( ) ;
235+
236+ let parray = PrimitiveArray :: from_iter ( values) ;
237+ let alp_encoded = alp_encode ( parray. as_view ( ) , None , & mut SESSION . create_execution_ctx ( ) ) ?;
238+
239+ assert ! (
240+ alp_encoded. patches( ) . is_some( ) ,
241+ "Expected ALP array to have patches"
242+ ) ;
243+
244+ let array = alp_encoded. as_array ( ) ;
245+
246+ let serialization = SESSION . array_serialize ( array) ?. unwrap ( ) ;
247+ let children = array. children ( ) ;
248+ let buffers = array
249+ . buffers ( )
250+ . into_iter ( )
251+ . map ( BufferHandle :: new_host)
252+ . collect :: < Vec < _ > > ( ) ;
253+
254+ let deserialized = ALPPatchedPlugin . deserialize (
255+ ArrayDeserialization :: new (
256+ ALPPatchedPlugin . id ( ) ,
257+ array. dtype ( ) ,
258+ array. len ( ) ,
259+ & serialization. metadata ,
260+ & buffers,
261+ & children,
262+ ) ,
263+ & SESSION ,
264+ ) ?;
265+
266+ assert_arrays_eq ! ( deserialized, parray, & mut SESSION . create_execution_ctx( ) ) ;
267+
268+ let patched: PatchedArray = deserialized
269+ . try_downcast ( )
270+ . map_err ( |a| vortex_err ! ( "Expected Patched, got {}" , a. encoding_id( ) ) ) ?;
271+
272+ let inner_alp: ALPArray = patched
273+ . inner ( )
274+ . clone ( )
275+ . try_downcast ( )
276+ . map_err ( |a| vortex_err ! ( "Expected inner ALP, got {}" , a. encoding_id( ) ) ) ?;
277+
278+ assert ! (
279+ inner_alp. patches( ) . is_none( ) ,
280+ "Inner ALP should NOT have patches"
281+ ) ;
282+
283+ Ok ( ( ) )
284+ }
285+
286+ #[ test]
287+ fn alp_without_patches_stays_alp ( ) -> VortexResult < ( ) > {
288+ // Values that encode cleanly without patches.
289+ let values: Vec < f64 > = ( 0 ..100 ) . map ( |i| i as f64 ) . collect ( ) ;
290+ let parray = PrimitiveArray :: from_iter ( values) ;
291+ let alp_encoded = alp_encode ( parray. as_view ( ) , None , & mut SESSION . create_execution_ctx ( ) ) ?;
292+
293+ assert ! (
294+ alp_encoded. patches( ) . is_none( ) ,
295+ "Expected ALP array without patches"
296+ ) ;
297+
298+ let array = alp_encoded. as_array ( ) ;
299+
300+ let serialization = SESSION . array_serialize ( array) ?. unwrap ( ) ;
301+ let children = array. children ( ) ;
302+ let buffers = array
303+ . buffers ( )
304+ . into_iter ( )
305+ . map ( BufferHandle :: new_host)
306+ . collect :: < Vec < _ > > ( ) ;
307+
308+ let deserialized = ALPPatchedPlugin . deserialize (
309+ ArrayDeserialization :: new (
310+ ALPPatchedPlugin . id ( ) ,
311+ array. dtype ( ) ,
312+ array. len ( ) ,
313+ & serialization. metadata ,
314+ & buffers,
315+ & children,
316+ ) ,
317+ & SESSION ,
318+ ) ?;
319+
320+ assert_arrays_eq ! ( deserialized, parray, & mut SESSION . create_execution_ctx( ) ) ;
321+
322+ let result = deserialized
323+ . try_downcast :: < ALP > ( )
324+ . map_err ( |a| vortex_err ! ( "Expected deserialized ALP, got {}" , a. encoding_id( ) ) ) ?;
325+
326+ assert ! ( result. patches( ) . is_none( ) , "Result should not have patches" ) ;
327+
328+ Ok ( ( ) )
329+ }
330+
331+ #[ test]
332+ #[ should_panic( expected = "index out of bounds" ) ]
333+ fn primitive_array_returns_error ( ) {
334+ let array = PrimitiveArray :: from_iter ( [ 1.0f64 , 2.0 , 3.0 ] ) . into_array ( ) ;
335+
336+ let serialization = SESSION . array_serialize ( & array) . unwrap ( ) . unwrap ( ) ;
337+ let children = array. children ( ) ;
338+ let buffers = array
339+ . buffers ( )
340+ . into_iter ( )
341+ . map ( BufferHandle :: new_host)
342+ . collect :: < Vec < _ > > ( ) ;
343+
344+ // This panics because PrimitiveArray has no children and ALP requires encoded child.
345+ let _result = ALPPatchedPlugin . deserialize (
346+ ArrayDeserialization :: new (
347+ ALPPatchedPlugin . id ( ) ,
348+ array. dtype ( ) ,
349+ array. len ( ) ,
350+ & serialization. metadata ,
351+ & buffers,
352+ & children,
353+ ) ,
354+ & SESSION ,
355+ ) ;
356+ }
357+ }
0 commit comments