Skip to content

Commit 27542d3

Browse files
authored
Add a VortexSession for scalar value deserialization (#6772)
## Summary Tracking Issue: #6771 Adds a `VortexSession` parameter to the scalar value deserialization functions, and updates all callers to pass a session. We want this in order to support adding an `Array` variant to the `ScalarValue` enum. That is coming in a followup PR (draft is at #6717). This PR just makes it easier to review what is going on. ## API Changes Adds a `VortexSession` parameter in a few places. ## Testing This changes doesn't actually make use of the session yet, so there is no logical changes (yet). --------- Signed-off-by: Connor Tsui <[email protected]>
1 parent a641b2a commit 27542d3

14 files changed

Lines changed: 80 additions & 39 deletions

File tree

‎encodings/fastlanes/public-api.lock‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -494,7 +494,7 @@ pub fn vortex_fastlanes::FoRVTable::child(array: &vortex_fastlanes::FoRArray, id
494494

495495
pub fn vortex_fastlanes::FoRVTable::child_name(_array: &vortex_fastlanes::FoRArray, idx: usize) -> alloc::string::String
496496

497-
pub fn vortex_fastlanes::FoRVTable::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
497+
pub fn vortex_fastlanes::FoRVTable::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
498498

499499
pub fn vortex_fastlanes::FoRVTable::dtype(array: &vortex_fastlanes::FoRArray) -> &vortex_array::dtype::DType
500500

‎encodings/fastlanes/src/for/vtable/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -132,9 +132,9 @@ impl VTable for FoRVTable {
132132
dtype: &DType,
133133
_len: usize,
134134
_buffers: &[BufferHandle],
135-
_session: &VortexSession,
135+
session: &VortexSession,
136136
) -> VortexResult<Self::Metadata> {
137-
let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype)?;
137+
let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype, session)?;
138138
Scalar::try_new(dtype.clone(), scalar_value)
139139
}
140140

‎encodings/sequence/public-api.lock‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ pub fn vortex_sequence::SequenceVTable::child(_array: &vortex_sequence::Sequence
124124

125125
pub fn vortex_sequence::SequenceVTable::child_name(_array: &vortex_sequence::SequenceArray, idx: usize) -> alloc::string::String
126126

127-
pub fn vortex_sequence::SequenceVTable::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
127+
pub fn vortex_sequence::SequenceVTable::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
128128

129129
pub fn vortex_sequence::SequenceVTable::dtype(array: &vortex_sequence::SequenceArray) -> &vortex_array::dtype::DType
130130

‎encodings/sequence/src/array.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -322,7 +322,7 @@ impl VTable for SequenceVTable {
322322
dtype: &DType,
323323
_len: usize,
324324
_buffers: &[BufferHandle],
325-
_session: &VortexSession,
325+
session: &VortexSession,
326326
) -> VortexResult<Self::Metadata> {
327327
let prost =
328328
<ProstMetadata<ProstSequenceMetadata> as DeserializeMetadata>::deserialize(bytes)?;
@@ -336,6 +336,7 @@ impl VTable for SequenceVTable {
336336
.as_ref()
337337
.ok_or_else(|| vortex_err!("base required"))?,
338338
&DType::Primitive(ptype, NonNullable),
339+
session,
339340
)?
340341
.as_primitive()
341342
.pvalue()
@@ -347,6 +348,7 @@ impl VTable for SequenceVTable {
347348
.as_ref()
348349
.ok_or_else(|| vortex_err!("multiplier required"))?,
349350
&DType::Primitive(ptype, NonNullable),
351+
session,
350352
)?
351353
.as_primitive()
352354
.pvalue()

‎encodings/sparse/public-api.lock‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ pub fn vortex_sparse::SparseVTable::child(array: &vortex_sparse::SparseArray, id
122122

123123
pub fn vortex_sparse::SparseVTable::child_name(_array: &vortex_sparse::SparseArray, idx: usize) -> alloc::string::String
124124

125-
pub fn vortex_sparse::SparseVTable::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
125+
pub fn vortex_sparse::SparseVTable::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
126126

127127
pub fn vortex_sparse::SparseVTable::dtype(array: &vortex_sparse::SparseArray) -> &vortex_array::dtype::DType
128128

‎encodings/sparse/src/lib.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ impl VTable for SparseVTable {
163163
dtype: &DType,
164164
_len: usize,
165165
buffers: &[BufferHandle],
166-
_session: &VortexSession,
166+
session: &VortexSession,
167167
) -> VortexResult<Self::Metadata> {
168168
let prost_patches =
169169
<ProstMetadata<ProstPatchesMetadata> as DeserializeMetadata>::deserialize(bytes)?;
@@ -175,7 +175,7 @@ impl VTable for SparseVTable {
175175
}
176176
let scalar_bytes: &[u8] = &buffers[0].clone().try_to_host_sync()?;
177177

178-
let scalar_value = ScalarValue::from_proto_bytes(scalar_bytes, dtype)?;
178+
let scalar_value = ScalarValue::from_proto_bytes(scalar_bytes, dtype, session)?;
179179
let fill_value = Scalar::try_new(dtype.clone(), scalar_value)?;
180180

181181
Ok(SparseMetadata {

‎vortex-array/public-api.lock‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -834,7 +834,7 @@ pub fn vortex_array::arrays::ConstantVTable::child(_array: &vortex_array::arrays
834834

835835
pub fn vortex_array::arrays::ConstantVTable::child_name(_array: &vortex_array::arrays::ConstantArray, idx: usize) -> alloc::string::String
836836

837-
pub fn vortex_array::arrays::ConstantVTable::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
837+
pub fn vortex_array::arrays::ConstantVTable::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
838838

839839
pub fn vortex_array::arrays::ConstantVTable::dtype(array: &vortex_array::arrays::ConstantArray) -> &vortex_array::dtype::DType
840840

@@ -11542,9 +11542,9 @@ pub fn vortex_array::scalar::ScalarValue::into_utf8(self) -> vortex_buffer::stri
1154211542

1154311543
impl vortex_array::scalar::ScalarValue
1154411544

11545-
pub fn vortex_array::scalar::ScalarValue::from_proto(value: &vortex_proto::scalar::ScalarValue, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult<core::option::Option<Self>>
11545+
pub fn vortex_array::scalar::ScalarValue::from_proto(value: &vortex_proto::scalar::ScalarValue, dtype: &vortex_array::dtype::DType, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<Self>>
1154611546

11547-
pub fn vortex_array::scalar::ScalarValue::from_proto_bytes(bytes: &[u8], dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult<core::option::Option<Self>>
11547+
pub fn vortex_array::scalar::ScalarValue::from_proto_bytes(bytes: &[u8], dtype: &vortex_array::dtype::DType, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<Self>>
1154811548

1154911549
impl vortex_array::scalar::ScalarValue
1155011550

@@ -12250,7 +12250,7 @@ impl vortex_array::scalar::Scalar
1225012250

1225112251
pub fn vortex_array::scalar::Scalar::from_proto(value: &vortex_proto::scalar::Scalar, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self>
1225212252

12253-
pub fn vortex_array::scalar::Scalar::from_proto_value(value: &vortex_proto::scalar::ScalarValue, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult<Self>
12253+
pub fn vortex_array::scalar::Scalar::from_proto_value(value: &vortex_proto::scalar::ScalarValue, dtype: &vortex_array::dtype::DType, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self>
1225412254

1225512255
impl vortex_array::scalar::Scalar
1225612256

@@ -15980,7 +15980,7 @@ pub fn vortex_array::stats::StatsSet::merge_unordered(self, other: &Self, dtype:
1598015980

1598115981
impl vortex_array::stats::StatsSet
1598215982

15983-
pub fn vortex_array::stats::StatsSet::from_flatbuffer<'a>(fb: &vortex_flatbuffers::array::ArrayStats<'a>, array_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult<Self>
15983+
pub fn vortex_array::stats::StatsSet::from_flatbuffer<'a>(fb: &vortex_flatbuffers::array::ArrayStats<'a>, array_dtype: &vortex_array::dtype::DType, session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self>
1598415984

1598515985
impl core::clone::Clone for vortex_array::stats::StatsSet
1598615986

@@ -16666,7 +16666,7 @@ pub fn vortex_array::arrays::ConstantVTable::child(_array: &vortex_array::arrays
1666616666

1666716667
pub fn vortex_array::arrays::ConstantVTable::child_name(_array: &vortex_array::arrays::ConstantArray, idx: usize) -> alloc::string::String
1666816668

16669-
pub fn vortex_array::arrays::ConstantVTable::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
16669+
pub fn vortex_array::arrays::ConstantVTable::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Metadata>
1667016670

1667116671
pub fn vortex_array::arrays::ConstantVTable::dtype(array: &vortex_array::arrays::ConstantArray) -> &vortex_array::dtype::DType
1667216672

‎vortex-array/src/arrays/constant/vtable/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ impl VTable for ConstantVTable {
133133
dtype: &DType,
134134
_len: usize,
135135
buffers: &[BufferHandle],
136-
_session: &VortexSession,
136+
session: &VortexSession,
137137
) -> VortexResult<Self::Metadata> {
138138
vortex_ensure!(
139139
buffers.len() == 1,
@@ -144,7 +144,7 @@ impl VTable for ConstantVTable {
144144
let buffer = buffers[0].clone().try_to_host_sync()?;
145145
let bytes: &[u8] = buffer.as_ref();
146146

147-
let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype)?;
147+
let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype, session)?;
148148
let scalar = Scalar::try_new(dtype.clone(), scalar_value)?;
149149

150150
Ok(scalar)

‎vortex-array/src/scalar/proto.rs‎

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -167,8 +167,12 @@ impl Scalar {
167167
/// # Errors
168168
///
169169
/// Returns an error if type validation fails.
170-
pub fn from_proto_value(value: &pb::ScalarValue, dtype: &DType) -> VortexResult<Self> {
171-
let scalar_value = ScalarValue::from_proto(value, dtype)?;
170+
pub fn from_proto_value(
171+
value: &pb::ScalarValue,
172+
dtype: &DType,
173+
session: &VortexSession,
174+
) -> VortexResult<Self> {
175+
let scalar_value = ScalarValue::from_proto(value, dtype, session)?;
172176

173177
Scalar::try_new(dtype.clone(), scalar_value)
174178
}
@@ -192,7 +196,7 @@ impl Scalar {
192196
.as_ref()
193197
.ok_or_else(|| vortex_err!(Serde: "Scalar missing value"))?;
194198

195-
let value: Option<ScalarValue> = ScalarValue::from_proto(pb_scalar_value, &dtype)?;
199+
let value: Option<ScalarValue> = ScalarValue::from_proto(pb_scalar_value, &dtype, session)?;
196200

197201
Scalar::try_new(dtype, value)
198202
}
@@ -207,9 +211,13 @@ impl ScalarValue {
207211
/// # Errors
208212
///
209213
/// Returns an error if decoding or type validation fails.
210-
pub fn from_proto_bytes(bytes: &[u8], dtype: &DType) -> VortexResult<Option<Self>> {
214+
pub fn from_proto_bytes(
215+
bytes: &[u8],
216+
dtype: &DType,
217+
session: &VortexSession,
218+
) -> VortexResult<Option<Self>> {
211219
let proto = pb::ScalarValue::decode(bytes)?;
212-
Self::from_proto(&proto, dtype)
220+
Self::from_proto(&proto, dtype, session)
213221
}
214222

215223
/// Creates a [`ScalarValue`] from its [protobuf](pb::ScalarValue) representation.
@@ -220,7 +228,11 @@ impl ScalarValue {
220228
/// # Errors
221229
///
222230
/// Returns an error if the protobuf value cannot be converted to the given [`DType`].
223-
pub fn from_proto(value: &pb::ScalarValue, dtype: &DType) -> VortexResult<Option<Self>> {
231+
pub fn from_proto(
232+
value: &pb::ScalarValue,
233+
dtype: &DType,
234+
session: &VortexSession,
235+
) -> VortexResult<Option<Self>> {
224236
let kind = value
225237
.kind
226238
.as_ref()
@@ -242,7 +254,7 @@ impl ScalarValue {
242254
Kind::F64Value(v) => f64_from_proto(*v, dtype)?,
243255
Kind::StringValue(s) => string_from_proto(s, dtype)?,
244256
Kind::BytesValue(b) => bytes_from_proto(b, dtype)?,
245-
Kind::ListValue(v) => list_from_proto(v, dtype)?,
257+
Kind::ListValue(v) => list_from_proto(v, dtype, session)?,
246258
}))
247259
}
248260
}
@@ -415,14 +427,22 @@ fn bytes_from_proto(bytes: &[u8], dtype: &DType) -> VortexResult<ScalarValue> {
415427
}
416428

417429
/// Deserialize a [`ScalarValue::List`] from a protobuf `ListValue`.
418-
fn list_from_proto(v: &ListValue, dtype: &DType) -> VortexResult<ScalarValue> {
430+
fn list_from_proto(
431+
v: &ListValue,
432+
dtype: &DType,
433+
session: &VortexSession,
434+
) -> VortexResult<ScalarValue> {
419435
let element_dtype = dtype
420436
.as_list_element_opt()
421437
.ok_or_else(|| vortex_err!(Serde: "expected List dtype for ListValue, got {dtype}"))?;
422438

423439
let mut values = Vec::with_capacity(v.values.len());
424440
for elem in v.values.iter() {
425-
values.push(ScalarValue::from_proto(elem, element_dtype.as_ref())?);
441+
values.push(ScalarValue::from_proto(
442+
elem,
443+
element_dtype.as_ref(),
444+
session,
445+
)?);
426446
}
427447

428448
Ok(ScalarValue::List(values))
@@ -604,6 +624,7 @@ mod tests {
604624
let scalar_value = ScalarValue::from_proto(
605625
&pb_scalar_value,
606626
&DType::Primitive(PType::U64, Nullability::NonNullable),
627+
&session(),
607628
)
608629
.unwrap();
609630
assert_eq!(
@@ -615,6 +636,7 @@ mod tests {
615636
let scalar_value_f16 = ScalarValue::from_proto(
616637
&pb_scalar_value,
617638
&DType::Primitive(PType::F16, Nullability::Nullable),
639+
&session(),
618640
)
619641
.unwrap();
620642

@@ -651,6 +673,7 @@ mod tests {
651673
let read_back = ScalarValue::from_proto(
652674
&pb_value,
653675
&DType::Primitive(PType::F16, Nullability::NonNullable),
676+
&session(),
654677
)
655678
.unwrap();
656679

@@ -729,7 +752,7 @@ mod tests {
729752

730753
for (name, value, dtype) in exact_roundtrip_cases {
731754
let pb_value = ScalarValue::to_proto(value.as_ref());
732-
let read_back = ScalarValue::from_proto(&pb_value, &dtype).unwrap();
755+
let read_back = ScalarValue::from_proto(&pb_value, &dtype, &session()).unwrap();
733756

734757
let original_debug = format!("{value:?}");
735758
let roundtrip_debug = format!("{read_back:?}");
@@ -764,7 +787,7 @@ mod tests {
764787

765788
for (name, value, dtype, expected) in unsigned_cases {
766789
let pb_value = ScalarValue::to_proto(Some(&value));
767-
let read_back = ScalarValue::from_proto(&pb_value, &dtype).unwrap();
790+
let read_back = ScalarValue::from_proto(&pb_value, &dtype, &session()).unwrap();
768791

769792
match read_back.as_ref() {
770793
Some(ScalarValue::Primitive(pv)) => {
@@ -808,7 +831,7 @@ mod tests {
808831

809832
for (name, value, dtype, expected) in signed_cases {
810833
let pb_value = ScalarValue::to_proto(Some(&value));
811-
let read_back = ScalarValue::from_proto(&pb_value, &dtype).unwrap();
834+
let read_back = ScalarValue::from_proto(&pb_value, &dtype, &session()).unwrap();
812835

813836
match read_back.as_ref() {
814837
Some(ScalarValue::Primitive(pv)) => {
@@ -837,7 +860,8 @@ mod tests {
837860
assert_eq!(
838861
Scalar::from_proto_value(
839862
&pb::ScalarValue::from(&v),
840-
&DType::Primitive(PType::U64, Nullability::Nullable)
863+
&DType::Primitive(PType::U64, Nullability::Nullable),
864+
&session()
841865
)
842866
.unwrap(),
843867
Scalar::primitive(0u64, Nullability::Nullable)
@@ -852,7 +876,8 @@ mod tests {
852876
assert_eq!(
853877
Scalar::from_proto_value(
854878
&pb::ScalarValue::from(&v),
855-
&DType::Primitive(PType::I64, Nullability::Nullable)
879+
&DType::Primitive(PType::I64, Nullability::Nullable),
880+
&session()
856881
)
857882
.unwrap(),
858883
Scalar::primitive(0i64, Nullability::Nullable)

‎vortex-array/src/serde.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,7 @@ impl ArrayParts {
377377
if let Some(stats) = self.flatbuffer().stats() {
378378
decoded
379379
.statistics()
380-
.set_iter(StatsSet::from_flatbuffer(&stats, dtype)?.into_iter());
380+
.set_iter(StatsSet::from_flatbuffer(&stats, dtype, session)?.into_iter());
381381
}
382382

383383
Ok(decoded)

0 commit comments

Comments
 (0)