Skip to content

Commit 709eaa1

Browse files
authored
Move FoR serde logic from VTable to Plugin (#10104)
First PR of a stack prototyping a FoR encoding with one reference per 1024-element chunk, following the plugin approach used for `DecimalByteParts` and #9937. ## Summary Move metadata, serialization and deserialization of `fastlanes.for` into a new `FoRPlugin`. The VTable keeps only the required serde methods, which now return an error, plus in-memory validation. `vortex_fastlanes::initialize` registers `FoRPlugin` instead of the vtable-derived plugin. The wire format is unchanged: the metadata is still just the reference `ScalarValue` proto bytes, with a single `encoded` child. Users who register `FoR` directly on a session must now register `FoRPlugin` to get serde. --------- Signed-off-by: Matt Katz <[email protected]>
1 parent 52e6407 commit 709eaa1

4 files changed

Lines changed: 179 additions & 33 deletions

File tree

‎encodings/fastlanes/src/for/mod.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ pub use array::FoRSlots;
99

1010
pub(crate) mod compute;
1111

12+
mod plugin;
13+
pub use plugin::FoRPlugin;
14+
1215
mod vtable;
1316
pub use vtable::FoR;
1417
pub use vtable::FoRArray;
Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
//! ArrayPlugin implementation for FoR.
5+
6+
use vortex_array::Array;
7+
use vortex_array::ArrayDeserialization;
8+
use vortex_array::ArrayId;
9+
use vortex_array::ArrayParts;
10+
use vortex_array::ArrayPlugin;
11+
use vortex_array::ArrayRef;
12+
use vortex_array::ArraySerialization;
13+
use vortex_array::IntoArray;
14+
use vortex_array::VTable;
15+
use vortex_array::scalar::Scalar;
16+
use vortex_array::scalar::ScalarValue;
17+
use vortex_array::smallvec::smallvec;
18+
use vortex_error::VortexResult;
19+
use vortex_error::vortex_bail;
20+
use vortex_error::vortex_ensure;
21+
use vortex_error::vortex_err;
22+
use vortex_session::VortexSession;
23+
24+
use crate::FoR;
25+
use crate::FoRData;
26+
use crate::r#for::array::FoRArrayExt;
27+
28+
/// Serde for the [`FoR`] array.
29+
///
30+
/// Register this plugin, or call [`crate::initialize`], to enable serde. Direct registration of
31+
/// [`FoR`] does not support serde.
32+
#[derive(Clone, Debug)]
33+
pub struct FoRPlugin;
34+
35+
impl ArrayPlugin for FoRPlugin {
36+
fn id(&self) -> ArrayId {
37+
VTable::id(&FoR)
38+
}
39+
40+
fn serialize(
41+
&self,
42+
array: &ArrayRef,
43+
_session: &VortexSession,
44+
) -> VortexResult<Option<ArraySerialization>> {
45+
let view = array
46+
.as_opt::<FoR>()
47+
.ok_or_else(|| vortex_err!("FoR plugin cannot serialize {}", array.encoding_id()))?;
48+
// Note that we **only** serialize the optional scalar value (not including the dtype).
49+
let metadata = ScalarValue::to_proto_bytes(view.reference_scalar().value());
50+
Ok(Some(ArraySerialization::from_array(
51+
self.id(),
52+
array,
53+
metadata,
54+
)))
55+
}
56+
57+
fn deserialize(
58+
&self,
59+
parts: ArrayDeserialization<'_>,
60+
session: &VortexSession,
61+
) -> VortexResult<ArrayRef> {
62+
vortex_ensure!(
63+
parts.serialized_id == self.id(),
64+
"FoR plugin does not recognize serialized ID {}",
65+
parts.serialized_id
66+
);
67+
vortex_ensure!(
68+
parts.buffers.is_empty(),
69+
"FoRArray expects 0 buffers, got {}",
70+
parts.buffers.len()
71+
);
72+
if parts.children.len() != 1 {
73+
vortex_bail!(
74+
"Expected 1 child for FoR encoding, found {}",
75+
parts.children.len()
76+
)
77+
}
78+
79+
let scalar_value = ScalarValue::from_proto_bytes(parts.metadata, parts.dtype, session)?;
80+
let reference = Scalar::try_new(parts.dtype.clone(), scalar_value)?;
81+
let encoded = parts.children.get(0, parts.dtype, parts.len)?;
82+
let slots = smallvec![Some(encoded)];
83+
84+
let data = FoRData::try_new(reference)?;
85+
Ok(Array::try_from_parts(
86+
ArrayParts::new(FoR, parts.dtype.clone(), parts.len, data).with_slots(slots),
87+
)?
88+
.into_array())
89+
}
90+
}
91+
92+
#[cfg(test)]
93+
mod tests {
94+
use std::sync::LazyLock;
95+
96+
use rstest::rstest;
97+
use vortex_array::ArrayContext;
98+
use vortex_array::VortexSessionExecute;
99+
use vortex_array::arrays::PrimitiveArray;
100+
use vortex_array::assert_arrays_eq;
101+
use vortex_array::serde::SerializeOptions;
102+
use vortex_array::serde::SerializedArray;
103+
use vortex_array::session::ArraySessionExt;
104+
use vortex_buffer::ByteBufferMut;
105+
use vortex_session::registry::ReadContext;
106+
107+
use super::*;
108+
use crate::FoRArray;
109+
110+
static SESSION: LazyLock<VortexSession> = LazyLock::new(|| {
111+
let session = vortex_array::array_session();
112+
crate::initialize(&session);
113+
session
114+
});
115+
116+
fn roundtrip(array: &ArrayRef) -> VortexResult<ArrayRef> {
117+
let array_ctx = ArrayContext::empty();
118+
let buffers = array.serialize(&array_ctx, &SESSION, &SerializeOptions::default())?;
119+
let mut bytes = ByteBufferMut::empty();
120+
for buffer in buffers {
121+
bytes.extend_from_slice(&buffer);
122+
}
123+
SerializedArray::try_from(bytes.freeze())?.decode(
124+
array.dtype(),
125+
array.len(),
126+
&ReadContext::new(array_ctx.to_ids()),
127+
&SESSION,
128+
)
129+
}
130+
131+
#[rstest]
132+
#[case::signed(FoR::encode(PrimitiveArray::from_iter([-5i32, 0, 7, 100]), &mut SESSION.create_execution_ctx()))]
133+
#[case::unsigned(FoR::encode(PrimitiveArray::from_iter(1_000u64..3_000), &mut SESSION.create_execution_ctx()))]
134+
#[case::nullable(FoR::encode(
135+
PrimitiveArray::from_option_iter([Some(10i16), None, Some(12)]),
136+
&mut SESSION.create_execution_ctx(),
137+
))]
138+
fn serde_roundtrip(#[case] array: VortexResult<FoRArray>) -> VortexResult<()> {
139+
let array = array?;
140+
let serialization = SESSION
141+
.array_serialize(array.as_array())?
142+
.ok_or_else(|| vortex_err!("FoR must serialize"))?;
143+
assert_eq!(serialization.serialized_id, VTable::id(&FoR));
144+
assert_eq!(
145+
serialization.metadata,
146+
ScalarValue::to_proto_bytes::<Vec<u8>>(array.reference_scalar().value())
147+
);
148+
149+
let read = roundtrip(array.as_array())?;
150+
assert_eq!(read.encoding_id(), VTable::id(&FoR));
151+
assert_arrays_eq!(read, array, &mut SESSION.create_execution_ctx());
152+
Ok(())
153+
}
154+
155+
#[test]
156+
fn vtable_serde_requires_plugin() -> VortexResult<()> {
157+
let array = FoR::encode(
158+
PrimitiveArray::from_iter([1u8, 2, 3]),
159+
&mut SESSION.create_execution_ctx(),
160+
)?;
161+
let session = vortex_array::array_session();
162+
session.arrays().register(FoR);
163+
assert!(session.array_serialize(&array.into_array()).is_err());
164+
Ok(())
165+
}
166+
}

‎encodings/fastlanes/src/for/vtable/mod.rs‎

Lines changed: 9 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ use vortex_array::arrays::PrimitiveArray;
2020
use vortex_array::buffer::BufferHandle;
2121
use vortex_array::dtype::DType;
2222
use vortex_array::scalar::Scalar;
23-
use vortex_array::scalar::ScalarValue;
2423
use vortex_array::serde::ArrayChildren;
2524
use vortex_array::smallvec::smallvec;
2625
use vortex_array::vtable::VTable;
@@ -33,7 +32,6 @@ use vortex_session::VortexSession;
3332
use vortex_session::registry::CachedId;
3433

3534
use crate::FoRData;
36-
use crate::r#for::array::FoRArrayExt;
3735
use crate::r#for::array::FoRSlots;
3836
use crate::r#for::array::FoRSlotsView;
3937
use crate::r#for::array::for_decompress::decompress;
@@ -111,43 +109,22 @@ impl VTable for FoR {
111109
}
112110

113111
fn serialize(
114-
array: ArrayView<'_, Self>,
112+
_array: ArrayView<'_, Self>,
115113
_session: &VortexSession,
116114
) -> VortexResult<Option<Vec<u8>>> {
117-
// Note that we **only** serialize the optional scalar value (not including the dtype).
118-
Ok(Some(ScalarValue::to_proto_bytes(
119-
array.reference_scalar().value(),
120-
)))
115+
vortex_bail!("FoR serialization requires FoRPlugin")
121116
}
122117

123118
fn deserialize(
124119
&self,
125-
dtype: &DType,
126-
len: usize,
127-
metadata: &[u8],
128-
buffers: &[BufferHandle],
129-
children: &dyn ArrayChildren,
130-
session: &VortexSession,
120+
_dtype: &DType,
121+
_len: usize,
122+
_metadata: &[u8],
123+
_buffers: &[BufferHandle],
124+
_children: &dyn ArrayChildren,
125+
_session: &VortexSession,
131126
) -> VortexResult<ArrayParts<Self>> {
132-
vortex_ensure!(
133-
buffers.is_empty(),
134-
"FoRArray expects 0 buffers, got {}",
135-
buffers.len()
136-
);
137-
if children.len() != 1 {
138-
vortex_bail!(
139-
"Expected 1 child for FoR encoding, found {}",
140-
children.len()
141-
)
142-
}
143-
144-
let scalar_value = ScalarValue::from_proto_bytes(metadata, dtype, session)?;
145-
let reference = Scalar::try_new(dtype.clone(), scalar_value)?;
146-
let encoded = children.get(0, dtype, len)?;
147-
let slots = smallvec![Some(encoded)];
148-
149-
let data = FoRData::try_new(reference)?;
150-
Ok(ArrayParts::new(self.clone(), dtype.clone(), len, data).with_slots(slots))
127+
vortex_bail!("FoR deserialization requires FoRPlugin")
151128
}
152129

153130
fn reduce_parent(

‎encodings/fastlanes/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ pub fn initialize(session: &VortexSession) {
8787
session.arrays().register(BitPacked);
8888
}
8989
session.arrays().register(Delta);
90-
session.arrays().register(FoR);
90+
session.arrays().register(FoRPlugin);
9191
session.arrays().register(RLE);
9292
session.arrays().register(TransposedBool);
9393
bitpacking::initialize(session);

0 commit comments

Comments
 (0)