Skip to content

Commit ea94807

Browse files
authored
Separate the read and write IDs of array plugins (#11057)
Tracking Issue: #10701 Bottom of the decimal `Widen` stack. #11002 builds on it. No release gate. ## Summary A plugin can need another encoding's wire form without taking over its reader. `serialized_ids()` currently serves both jobs, so a `Widen` writer cannot emit a narrow decimal node while preserving the decimal reader. ## Changes Reader registration now uses `read_ids()`, while serialization validates its output against `write_ids()`. A writing session does not need a reader for every ID it writes. Existing plugins keep their current IDs, and tests cover reader ownership and the two validation contracts. `LayoutPlugin` is unchanged. ## API Changes Replaces `ArrayPlugin::serialized_ids()` with required `read_ids()` and `write_ids()` methods, with no defaults. Read IDs stay ordered oldest to newest. --------- Signed-off-by: Connor Tsui <[email protected]>
1 parent 7e8fb3a commit ea94807

10 files changed

Lines changed: 230 additions & 59 deletions

File tree

‎encodings/alp/src/alp/plugin.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,14 @@ impl ArrayPlugin for ALPPatchedPlugin {
3838
ArrayVTable::id(&ALP)
3939
}
4040

41+
fn read_ids(&self) -> Vec<ArrayId> {
42+
vec![self.id()]
43+
}
44+
45+
fn write_ids(&self) -> Vec<ArrayId> {
46+
vec![self.id()]
47+
}
48+
4149
fn serialize(
4250
&self,
4351
array: &ArrayRef,

‎encodings/decimal-byte-parts/src/decimal_byte_parts/plugin/mod.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,11 @@ impl ArrayPlugin for DecimalBytePartsPlugin {
5555
VTable::id(&DecimalByteParts)
5656
}
5757

58-
fn serialized_ids(&self) -> Vec<ArrayId> {
58+
fn read_ids(&self) -> Vec<ArrayId> {
59+
vec![decimal_byte_parts_v1_id(), decimal_byte_parts_v2_id()]
60+
}
61+
62+
fn write_ids(&self) -> Vec<ArrayId> {
5963
vec![decimal_byte_parts_v1_id(), decimal_byte_parts_v2_id()]
6064
}
6165

‎encodings/fastlanes/src/bitpacking/plugin/mod.rs‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,11 @@ impl ArrayPlugin for BitPackedPlugin {
6464
ArrayVTable::id(&BitPacked)
6565
}
6666

67-
fn serialized_ids(&self) -> Vec<ArrayId> {
67+
fn read_ids(&self) -> Vec<ArrayId> {
68+
vec![bitpacked_v1_id(), bitpacked_v2_id()]
69+
}
70+
71+
fn write_ids(&self) -> Vec<ArrayId> {
6872
vec![bitpacked_v1_id(), bitpacked_v2_id()]
6973
}
7074

@@ -114,8 +118,12 @@ impl ArrayPlugin for BitPackedPatchedPlugin {
114118
ArrayVTable::id(&BitPacked)
115119
}
116120

117-
fn serialized_ids(&self) -> Vec<ArrayId> {
118-
BitPackedPlugin.serialized_ids()
121+
fn read_ids(&self) -> Vec<ArrayId> {
122+
BitPackedPlugin.read_ids()
123+
}
124+
125+
fn write_ids(&self) -> Vec<ArrayId> {
126+
BitPackedPlugin.write_ids()
119127
}
120128

121129
fn serialize(
@@ -133,7 +141,7 @@ impl ArrayPlugin for BitPackedPatchedPlugin {
133141
session: &VortexSession,
134142
) -> VortexResult<ArrayRef> {
135143
vortex_ensure!(
136-
self.serialized_ids().contains(&parts.serialized_id),
144+
self.read_ids().contains(&parts.serialized_id),
137145
"BitPacked plugin does not recognize serialized ID {}",
138146
parts.serialized_id,
139147
);

‎encodings/fastlanes/src/for_/plugin/mod.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,11 @@ impl ArrayPlugin for FoRPlugin {
5252
VTable::id(&FoR)
5353
}
5454

55-
fn serialized_ids(&self) -> Vec<ArrayId> {
55+
fn read_ids(&self) -> Vec<ArrayId> {
56+
vec![for_v1_id(), for_v2_id()]
57+
}
58+
59+
fn write_ids(&self) -> Vec<ArrayId> {
5660
vec![for_v1_id(), for_v2_id()]
5761
}
5862

‎vortex-array/src/array/plugin.rs‎

Lines changed: 30 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ pub type ArrayPluginRef = Arc<dyn ArrayPlugin>;
2626
/// The wire representation produced by an in-memory array's serializer.
2727
///
2828
/// A serializer may reuse the in-memory array's buffers and children with [`Self::from_array`],
29-
/// or return different parts when an older wire representation requires a lossless structural
30-
/// downgrade.
29+
/// or return the wire representation of another encoding. The node must hold the logical values
30+
/// of the array under the dtype supplied by the containing format.
3131
#[derive(Clone, Debug)]
3232
pub struct ArraySerialization {
3333
/// The concrete array ID to write on the wire.
@@ -101,32 +101,35 @@ impl<'a> ArrayDeserialization<'a> {
101101

102102
/// Registry trait for serializing and deserializing an in-memory array representation.
103103
///
104-
/// A plugin has one [`id`](Self::id) for the in-memory representation and one or more
105-
/// [`serialized_ids`](Self::serialized_ids) for wire representations. Its serializer chooses the
106-
/// wire representation, and the serialization context validates that the chosen ID is permitted
107-
/// before it is written.
104+
/// A plugin writes one in-memory encoding, identified by [`id`](Self::id). It declares the wire
105+
/// IDs that it reads and writes separately, so it can write another encoding's wire form without
106+
/// replacing that encoding's reader. The serialization context validates that the chosen wire ID
107+
/// is permitted before it is written.
108108
///
109-
/// Every serialized ID is also registered for deserialization. A current plugin may therefore
110-
/// deserialize several historical IDs into the same in-memory representation. A reader that
111-
/// predates a newer ID has no registration for it and reports it as unknown instead of silently
112-
/// interpreting an unsupported representation.
109+
/// A plugin can retain historical read IDs after it stops writing them. A reader that predates a
110+
/// newer ID has no registration for it and reports it as unknown.
113111
pub trait ArrayPlugin: 'static + Send + Sync {
114-
/// Returns the ID of the in-memory array representation handled by this plugin.
112+
/// Returns the ID of the in-memory encoding that this plugin writes.
115113
fn id(&self) -> ArrayId;
116114

117-
/// Returns the serialized array IDs understood by this plugin, ordered oldest to newest.
115+
/// Returns the wire IDs that [`deserialize`](Self::deserialize) accepts, oldest to newest.
118116
///
119-
/// The default uses the in-memory ID as the sole wire ID. Override this for an in-memory array
120-
/// that has multiple serialized variants. IDs retained only for reading may also be included;
121-
/// the single serializer need not select them.
122-
fn serialized_ids(&self) -> Vec<ArrayId> {
123-
vec![self.id()]
124-
}
117+
/// [`ArraySession::register`](crate::session::ArraySession::register) makes this plugin the
118+
/// reader of each ID. An ID can stay in this list after the plugin stops writing it, so that
119+
/// old files stay readable.
120+
fn read_ids(&self) -> Vec<ArrayId>;
121+
122+
/// Returns the wire IDs that [`serialize`](Self::serialize) may return.
123+
///
124+
/// An ID in this list does not have to be in [`read_ids`](Self::read_ids). A plugin can write
125+
/// its arrays in the wire form of another encoding. The plugin that owns that form reads it.
126+
/// The node must hold the logical values of the array under the dtype of the array.
127+
fn write_ids(&self) -> Vec<ArrayId>;
125128

126129
/// Serialize `array` to its wire representation.
127130
///
128131
/// This function is called only for arrays whose in-memory encoding matches [`id`](Self::id).
129-
/// The returned ID must be declared by [`serialized_ids`](Self::serialized_ids). Return
132+
/// The returned ID must be declared by [`write_ids`](Self::write_ids). Return
130133
/// `Ok(None)` when the array cannot be serialized.
131134
fn serialize(
132135
&self,
@@ -167,6 +170,14 @@ impl<V: VTable> ArrayPlugin for V {
167170
VTable::id(self)
168171
}
169172

173+
fn read_ids(&self) -> Vec<ArrayId> {
174+
vec![self.id()]
175+
}
176+
177+
fn write_ids(&self) -> Vec<ArrayId> {
178+
vec![self.id()]
179+
}
180+
170181
fn serialize(
171182
&self,
172183
array: &ArrayRef,

‎vortex-array/src/arrays/scalar_fn/plugin.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,14 @@ impl<V: ScalarFnVTable + ScalarFnArrayVTable> ArrayPlugin for ScalarFnArrayPlugi
6262
self.0.id()
6363
}
6464

65+
fn read_ids(&self) -> Vec<ArrayId> {
66+
vec![self.id()]
67+
}
68+
69+
fn write_ids(&self) -> Vec<ArrayId> {
70+
vec![self.id()]
71+
}
72+
6573
fn serialize(
6674
&self,
6775
array: &ArrayRef,

‎vortex-array/src/serde.rs‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -818,7 +818,11 @@ mod tests {
818818
old_primitive_id()
819819
}
820820

821-
fn serialized_ids(&self) -> Vec<ArrayId> {
821+
fn read_ids(&self) -> Vec<ArrayId> {
822+
vec![old_primitive_id(), new_primitive_id()]
823+
}
824+
825+
fn write_ids(&self) -> Vec<ArrayId> {
822826
vec![old_primitive_id(), new_primitive_id()]
823827
}
824828

@@ -883,8 +887,12 @@ mod tests {
883887
VersionedPrimitivePlugin.id()
884888
}
885889

886-
fn serialized_ids(&self) -> Vec<ArrayId> {
887-
VersionedPrimitivePlugin.serialized_ids()
890+
fn read_ids(&self) -> Vec<ArrayId> {
891+
VersionedPrimitivePlugin.read_ids()
892+
}
893+
894+
fn write_ids(&self) -> Vec<ArrayId> {
895+
VersionedPrimitivePlugin.write_ids()
888896
}
889897

890898
fn serialize(

‎vortex-array/src/session/mod.rs‎

Lines changed: 11 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4+
//! Session registries for array readers and writers.
5+
//!
6+
//! Writers are keyed by in-memory encoding IDs, while readers are keyed by wire IDs. Keeping the
7+
//! registries separate lets a writer reuse another encoding's wire form without replacing its
8+
//! reader.
9+
410
use std::any::Any;
511
use std::sync::Arc;
612

@@ -62,14 +68,14 @@ impl ArraySession {
6268
&self.registry
6369
}
6470

65-
/// Register an in-memory array plugin and all of its recognized serialized IDs.
71+
/// Register an in-memory array writer and its declared wire readers.
6672
///
6773
/// This replaces any serializer with the same in-memory ID and any deserializer registered
68-
/// under one of [`ArrayPlugin::serialized_ids`].
74+
/// under one of [`ArrayPlugin::read_ids`].
6975
pub fn register<P: ArrayPlugin>(&self, plugin: P) {
7076
let plugin = Arc::new(plugin) as ArrayPluginRef;
7177
self.serializers.insert(plugin.id(), Arc::clone(&plugin));
72-
for serialized_id in plugin.serialized_ids() {
78+
for serialized_id in plugin.read_ids() {
7379
self.registry.insert(serialized_id, Arc::clone(&plugin));
7480
}
7581
}
@@ -144,9 +150,7 @@ pub trait ArraySessionExt: SessionExt {
144150
return Ok(None);
145151
};
146152
vortex_ensure!(
147-
plugin
148-
.serialized_ids()
149-
.contains(&serialization.serialized_id),
153+
plugin.write_ids().contains(&serialization.serialized_id),
150154
"array serializer {} produced undeclared serialized ID {}",
151155
array.encoding_id(),
152156
serialization.serialized_id,
@@ -158,27 +162,4 @@ pub trait ArraySessionExt: SessionExt {
158162
impl<S: SessionExt> ArraySessionExt for S {}
159163

160164
#[cfg(test)]
161-
mod tests {
162-
use vortex_session::VortexSession;
163-
164-
use crate::ArrayVTable;
165-
use crate::arrays::Bool;
166-
use crate::session::ArraySession;
167-
use crate::session::ArraySessionExt;
168-
169-
#[test]
170-
fn array_session_default_registers_encodings() {
171-
let session = VortexSession::empty().with::<ArraySession>();
172-
173-
assert!(session.arrays().registry().contains_key(&Bool.id()));
174-
assert!(session.arrays().serializer(&Bool.id()).is_some());
175-
}
176-
177-
#[test]
178-
fn empty_array_session_registers_no_encodings() {
179-
let session = VortexSession::empty().with_some(ArraySession::empty());
180-
181-
assert!(!session.arrays().registry().contains_key(&Bool.id()));
182-
assert!(session.arrays().serializer(&Bool.id()).is_none());
183-
}
184-
}
165+
mod tests;

0 commit comments

Comments
 (0)