Skip to content

Commit 503854f

Browse files
committed
use directory modules for vtables with a plugin and merge the alp plugins
Signed-off-by: Matt Katz <[email protected]>
1 parent 29697ed commit 503854f

17 files changed

Lines changed: 233 additions & 251 deletions

File tree

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ use crate::alp::decompress::execute_decompress;
5050
use crate::alp::rules::RULES;
5151

5252
mod plugin;
53+
pub(crate) use plugin::ALPPatchedPlugin;
5354

5455
/// A [`ALP`]-encoded Vortex array.
5556
pub type ALPArray = Array<ALP>;

‎encodings/alp/src/alp/array/plugin.rs‎

Lines changed: 231 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4-
//! [`ArrayPlugin`] implementation for [`ALP`].
4+
//! [`ArrayPlugin`] implementations for [`ALP`].
5+
//!
6+
//! [`ALP`] itself reads and writes the current `vortex.alp` form. [`ALPPatchedPlugin`] reads an
7+
//! ALP node with interior patches as a `PatchedArray` wrapping a patchless ALP array, which keeps
8+
//! previously written datasets readable at no cost.
59
610
use prost::Message;
711
use vortex_array::Array;
@@ -14,16 +18,19 @@ use vortex_array::ArraySerialization;
1418
use vortex_array::IntoArray;
1519
use vortex_array::VTable;
1620
use vortex_array::VortexSessionExecute;
21+
use vortex_array::arrays::Patched;
1722
use vortex_array::dtype::DType;
1823
use vortex_array::dtype::PType;
1924
use vortex_array::patches::Patches;
2025
use vortex_error::VortexResult;
2126
use vortex_error::vortex_bail;
2227
use vortex_error::vortex_ensure_eq;
28+
use vortex_error::vortex_err;
2329
use vortex_session::VortexSession;
2430

2531
use super::ALP;
2632
use super::ALPArrayExt;
33+
use super::ALPArrayOwnedExt;
2734
use super::ALPData;
2835
use super::ALPMetadata;
2936
use crate::alp::Exponents;
@@ -125,3 +132,226 @@ impl ArrayPlugin for ALP {
125132
.into_array())
126133
}
127134
}
135+
136+
/// Custom deserialization plugin that converts an ALP array with interior
137+
/// patches into a PatchedArray holding an ALP array.
138+
#[derive(Debug, Clone)]
139+
pub(crate) struct ALPPatchedPlugin;
140+
141+
impl ArrayPlugin for ALPPatchedPlugin {
142+
fn id(&self) -> ArrayId {
143+
// We reuse the existing `ALP` ID so that we can take over its
144+
// deserialization pathway.
145+
// TODO(joe): dedup method name
146+
VTable::id(&ALP)
147+
}
148+
149+
fn serialize(
150+
&self,
151+
array: &ArrayRef,
152+
session: &VortexSession,
153+
) -> VortexResult<Option<ArraySerialization>> {
154+
// Delegate to ALP's metadata serde
155+
ArrayPlugin::serialize(&ALP, array, session)
156+
}
157+
158+
fn deserialize(
159+
&self,
160+
parts: ArrayDeserialization<'_>,
161+
session: &VortexSession,
162+
) -> VortexResult<ArrayRef> {
163+
vortex_ensure_eq!(
164+
parts.serialized_id,
165+
self.id(),
166+
"ALP plugin does not recognize serialized ID"
167+
);
168+
let alp_array = ArrayPlugin::deserialize(&ALP, parts, session)?
169+
.try_downcast::<ALP>()
170+
.map_err(|_| vortex_err!("ALP plugin returned an array that is not vortex.alp"))?;
171+
172+
// Check if there are interior patches to externalize.
173+
let Some(patches) = alp_array.patches() else {
174+
return Ok(alp_array.into_array());
175+
};
176+
177+
// Extract components and create a new ALP array without patches.
178+
let (encoded, exponents, _) = alp_array.into_parts();
179+
180+
let alp_without_patches = ALP::try_new(encoded, exponents, None)?.into_array();
181+
182+
let patched = Patched::from_array_and_patches(
183+
alp_without_patches,
184+
&patches,
185+
&mut session.create_execution_ctx(),
186+
)?;
187+
188+
Ok(patched.into_array())
189+
}
190+
191+
fn is_supported_encoding(&self, id: &ArrayId) -> bool {
192+
// TODO(joe): dedup method name
193+
id == VTable::id(&Patched) || id == VTable::id(&ALP)
194+
}
195+
}
196+
197+
#[cfg(test)]
198+
mod tests {
199+
use std::f64::consts::PI;
200+
use std::sync::LazyLock;
201+
202+
use vortex_array::ArrayDeserialization;
203+
use vortex_array::ArrayPlugin;
204+
use vortex_array::IntoArray;
205+
use vortex_array::VortexSessionExecute;
206+
use vortex_array::arrays::PatchedArray;
207+
use vortex_array::arrays::PrimitiveArray;
208+
use vortex_array::arrays::patched::PatchedArraySlotsExt;
209+
use vortex_array::assert_arrays_eq;
210+
use vortex_array::buffer::BufferHandle;
211+
use vortex_array::session::ArraySessionExt;
212+
use vortex_error::VortexResult;
213+
use vortex_error::vortex_err;
214+
use vortex_session::VortexSession;
215+
216+
use super::ALPPatchedPlugin;
217+
use crate::ALP;
218+
use crate::ALPArray;
219+
use crate::ALPArrayExt;
220+
use crate::alp_encode;
221+
222+
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
223+
let session = VortexSession::empty();
224+
session.arrays().register(ALPPatchedPlugin);
225+
session
226+
});
227+
228+
#[test]
229+
fn test_decode_alp_patches() -> VortexResult<()> {
230+
// Create values where some don't encode cleanly with ALP, causing patches.
231+
// PI doesn't encode cleanly.
232+
let values: Vec<f64> = (0..100)
233+
.map(|i| if i % 4 == 3 { PI } else { i as f64 })
234+
.collect();
235+
236+
let parray = PrimitiveArray::from_iter(values);
237+
let alp_encoded = alp_encode(parray.as_view(), None, &mut SESSION.create_execution_ctx())?;
238+
239+
assert!(
240+
alp_encoded.patches().is_some(),
241+
"Expected ALP array to have patches"
242+
);
243+
244+
let array = alp_encoded.as_array();
245+
246+
let serialization = SESSION.array_serialize(array)?.unwrap();
247+
let children = array.children();
248+
let buffers = array
249+
.buffers()
250+
.into_iter()
251+
.map(BufferHandle::new_host)
252+
.collect::<Vec<_>>();
253+
254+
let deserialized = ALPPatchedPlugin.deserialize(
255+
ArrayDeserialization::new(
256+
ALPPatchedPlugin.id(),
257+
array.dtype(),
258+
array.len(),
259+
&serialization.metadata,
260+
&buffers,
261+
&children,
262+
),
263+
&SESSION,
264+
)?;
265+
266+
assert_arrays_eq!(deserialized, parray, &mut SESSION.create_execution_ctx());
267+
268+
let patched: PatchedArray = deserialized
269+
.try_downcast()
270+
.map_err(|a| vortex_err!("Expected Patched, got {}", a.encoding_id()))?;
271+
272+
let inner_alp: ALPArray = patched
273+
.inner()
274+
.clone()
275+
.try_downcast()
276+
.map_err(|a| vortex_err!("Expected inner ALP, got {}", a.encoding_id()))?;
277+
278+
assert!(
279+
inner_alp.patches().is_none(),
280+
"Inner ALP should NOT have patches"
281+
);
282+
283+
Ok(())
284+
}
285+
286+
#[test]
287+
fn alp_without_patches_stays_alp() -> VortexResult<()> {
288+
// Values that encode cleanly without patches.
289+
let values: Vec<f64> = (0..100).map(|i| i as f64).collect();
290+
let parray = PrimitiveArray::from_iter(values);
291+
let alp_encoded = alp_encode(parray.as_view(), None, &mut SESSION.create_execution_ctx())?;
292+
293+
assert!(
294+
alp_encoded.patches().is_none(),
295+
"Expected ALP array without patches"
296+
);
297+
298+
let array = alp_encoded.as_array();
299+
300+
let serialization = SESSION.array_serialize(array)?.unwrap();
301+
let children = array.children();
302+
let buffers = array
303+
.buffers()
304+
.into_iter()
305+
.map(BufferHandle::new_host)
306+
.collect::<Vec<_>>();
307+
308+
let deserialized = ALPPatchedPlugin.deserialize(
309+
ArrayDeserialization::new(
310+
ALPPatchedPlugin.id(),
311+
array.dtype(),
312+
array.len(),
313+
&serialization.metadata,
314+
&buffers,
315+
&children,
316+
),
317+
&SESSION,
318+
)?;
319+
320+
assert_arrays_eq!(deserialized, parray, &mut SESSION.create_execution_ctx());
321+
322+
let result = deserialized
323+
.try_downcast::<ALP>()
324+
.map_err(|a| vortex_err!("Expected deserialized ALP, got {}", a.encoding_id()))?;
325+
326+
assert!(result.patches().is_none(), "Result should not have patches");
327+
328+
Ok(())
329+
}
330+
331+
#[test]
332+
#[should_panic(expected = "index out of bounds")]
333+
fn primitive_array_returns_error() {
334+
let array = PrimitiveArray::from_iter([1.0f64, 2.0, 3.0]).into_array();
335+
336+
let serialization = SESSION.array_serialize(&array).unwrap().unwrap();
337+
let children = array.children();
338+
let buffers = array
339+
.buffers()
340+
.into_iter()
341+
.map(BufferHandle::new_host)
342+
.collect::<Vec<_>>();
343+
344+
// This panics because PrimitiveArray has no children and ALP requires encoded child.
345+
let _result = ALPPatchedPlugin.deserialize(
346+
ArrayDeserialization::new(
347+
ALPPatchedPlugin.id(),
348+
array.dtype(),
349+
array.len(),
350+
&serialization.metadata,
351+
&buffers,
352+
&children,
353+
),
354+
&SESSION,
355+
);
356+
}
357+
}

‎encodings/alp/src/alp/mod.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,9 @@ mod compress;
1515
pub(crate) mod compute;
1616
mod decompress;
1717
mod ops;
18-
mod plugin;
1918
mod rules;
2019

21-
pub(crate) use plugin::ALPPatchedPlugin;
20+
pub(crate) use array::ALPPatchedPlugin;
2221

2322
pub(crate) fn initialize(session: &VortexSession) {
2423
rules::initialize(session);

0 commit comments

Comments
 (0)