Skip to content

Commit e6e8eeb

Browse files
committed
Remove serialized Union nullability
Signed-off-by: Connor Tsui <[email protected]>
1 parent 8916c4e commit e6e8eeb

9 files changed

Lines changed: 17 additions & 217 deletions

File tree

‎vortex-array/src/dtype/serde/flatbuffers.rs‎

Lines changed: 1 addition & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ use itertools::Itertools;
1111
use vortex_error::VortexError;
1212
use vortex_error::VortexResult;
1313
use vortex_error::vortex_bail;
14-
use vortex_error::vortex_ensure;
1514
use vortex_error::vortex_err;
1615
use vortex_flatbuffers::FlatBuffer;
1716
use vortex_flatbuffers::FlatBufferRoot;
@@ -22,7 +21,6 @@ use vortex_session::VortexSession;
2221
use crate::dtype::DType;
2322
use crate::dtype::DecimalDType;
2423
use crate::dtype::FieldDType;
25-
use crate::dtype::Nullability;
2624
use crate::dtype::PType;
2725
use crate::dtype::StructFields;
2826
use crate::dtype::UnionVariants;
@@ -236,16 +234,6 @@ impl TryFrom<ViewedDType> for DType {
236234
.ok_or_else(|| vortex_err!("failed to parse union from flatbuffer"))?;
237235
let variants =
238236
UnionVariants::from_fb(fb_union, vfdt.buffer().clone(), vfdt.session.clone())?;
239-
240-
let serialized_nullability: Nullability = fb_union.nullable().into();
241-
let derived_nullability = variants.derived_nullability();
242-
vortex_ensure!(
243-
serialized_nullability == derived_nullability,
244-
"Serialized Union nullability does not match its variants: serialized={:?}, derived={:?}",
245-
serialized_nullability,
246-
derived_nullability
247-
);
248-
249237
Ok(Self::Union(variants))
250238
}
251239
fb::Type::Variant => {
@@ -414,7 +402,6 @@ impl WriteFlatBuffer for DType {
414402
names,
415403
dtypes,
416404
type_ids,
417-
nullable: uv.derived_nullability().into(),
418405
},
419406
)
420407
.as_union_value()
@@ -594,7 +581,7 @@ mod test {
594581
}
595582

596583
#[test]
597-
fn test_union_round_trip_flatbuffer_with_nullability() {
584+
fn test_union_round_trip_flatbuffer_with_nullable_variant() {
598585
let dtype = DType::Union(
599586
UnionVariants::new(
600587
["null_variant", "str"].into(),
@@ -605,53 +592,6 @@ mod test {
605592
roundtrip_dtype(dtype);
606593
}
607594

608-
#[test]
609-
fn test_union_flatbuffer_rejects_mismatched_serialized_nullability() {
610-
use flatbuffers::FlatBufferBuilder;
611-
use vortex_buffer::ByteBuffer;
612-
use vortex_flatbuffers::WriteFlatBuffer;
613-
614-
let mut fbb = FlatBufferBuilder::new();
615-
let name = fbb.create_string("value");
616-
let names = fbb.create_vector(&[name]);
617-
let child = DType::Utf8(Nullability::Nullable)
618-
.write_flatbuffer(&mut fbb)
619-
.unwrap();
620-
let dtypes = fbb.create_vector(&[child]);
621-
let type_ids = fbb.create_vector::<i8>(&[0]);
622-
let union_table = fb::Union::create(
623-
&mut fbb,
624-
&fb::UnionArgs {
625-
names: Some(names),
626-
dtypes: Some(dtypes),
627-
type_ids: Some(type_ids),
628-
nullable: false,
629-
},
630-
);
631-
let dtype = fb::DType::create(
632-
&mut fbb,
633-
&fb::DTypeArgs {
634-
type_type: fb::Type::Union,
635-
type_: Some(union_table.as_union_value()),
636-
},
637-
);
638-
fbb.finish_minimal(dtype);
639-
let (vec, start) = fbb.collapse();
640-
let end = vec.len();
641-
let buffer = FlatBuffer::align_from(ByteBuffer::from(vec).slice(start..end));
642-
let root_fb = root::<fb::DType>(&buffer).unwrap();
643-
let view = ViewedDType::from_fb_loc(root_fb._tab.loc(), buffer, SESSION.clone());
644-
645-
let result = DType::try_from(view);
646-
assert!(result.is_err());
647-
assert!(
648-
result
649-
.unwrap_err()
650-
.to_string()
651-
.contains("Serialized Union nullability does not match its variants")
652-
);
653-
}
654-
655595
#[test]
656596
fn test_union_round_trip_flatbuffer_with_type_id_indirection() {
657597
let dtype = DType::Union(
@@ -755,7 +695,6 @@ mod test {
755695
names: Some(names),
756696
dtypes: Some(dtypes),
757697
type_ids: Some(type_ids),
758-
nullable: false,
759698
},
760699
);
761700

‎vortex-array/src/dtype/serde/mod.rs‎

Lines changed: 4 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,10 @@ mod test {
190190
);
191191

192192
let json = serde_json::to_string(&dtype).unwrap();
193+
assert_eq!(
194+
json,
195+
r#"{"Union":{"names":["value"],"dtypes":[{"Utf8":true}],"type_ids":[0]}}"#
196+
);
193197
let mut deserializer = serde_json::Deserializer::from_str(&json);
194198
let deserialized: DType = DTypeSerde::<DType>::new(&SESSION)
195199
.deserialize(&mut deserializer)
@@ -198,28 +202,4 @@ mod test {
198202
assert_eq!(deserialized, dtype);
199203
assert_eq!(deserialized.nullability(), Nullability::Nullable);
200204
}
201-
202-
#[test]
203-
fn test_serde_union_rejects_mismatched_serialized_nullability() {
204-
let json = r#"{
205-
"Union": [
206-
{
207-
"names": ["value"],
208-
"dtypes": [{"Utf8": false}],
209-
"type_ids": [0]
210-
},
211-
true
212-
]
213-
}"#;
214-
let mut deserializer = serde_json::Deserializer::from_str(json);
215-
let result = DTypeSerde::<DType>::new(&SESSION).deserialize(&mut deserializer);
216-
217-
assert!(result.is_err());
218-
assert!(
219-
result
220-
.unwrap_err()
221-
.to_string()
222-
.contains("Serialized Union nullability does not match its variants")
223-
);
224-
}
225205
}

‎vortex-array/src/dtype/serde/proto.rs‎

Lines changed: 1 addition & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,11 @@ use std::sync::Arc;
55

66
use vortex_error::VortexError;
77
use vortex_error::VortexResult;
8-
use vortex_error::vortex_ensure;
98
use vortex_error::vortex_err;
109
use vortex_session::VortexSession;
1110

1211
use crate::dtype::DType;
1312
use crate::dtype::DecimalDType;
14-
use crate::dtype::Nullability;
1513
use crate::dtype::PType;
1614
use crate::dtype::StructFields;
1715
use crate::dtype::UnionVariants;
@@ -106,16 +104,6 @@ impl DType {
106104
})
107105
.collect::<VortexResult<Vec<_>>>()?;
108106
let variants = UnionVariants::try_new(names, dtypes, type_ids)?;
109-
110-
let serialized_nullability: Nullability = u.nullable.into();
111-
let derived_nullability = variants.derived_nullability();
112-
vortex_ensure!(
113-
serialized_nullability == derived_nullability,
114-
"Serialized Union nullability does not match its variants: serialized={:?}, derived={:?}",
115-
serialized_nullability,
116-
derived_nullability
117-
);
118-
119107
Ok(Self::Union(variants))
120108
}
121109
DtypeType::Variant(v) => Ok(Self::Variant(v.nullable.into())),
@@ -192,7 +180,6 @@ impl TryFrom<&DType> for pb::DType {
192180
.map(|d| Self::try_from(&d))
193181
.collect::<VortexResult<Vec<_>>>()?,
194182
type_ids: uv.type_ids().iter().map(|t| *t as i32).collect(),
195-
nullable: uv.derived_nullability().into(),
196183
}),
197184
DType::Variant(null) => DtypeType::Variant(pb::Variant {
198185
nullable: (*null).into(),
@@ -435,7 +422,7 @@ mod tests {
435422
}
436423

437424
#[test]
438-
fn test_union_round_trip_proto_with_nullability() {
425+
fn test_union_round_trip_proto_with_nullable_variant() {
439426
let variants = UnionVariants::new(
440427
["null_variant", "str"].into(),
441428
vec![DType::Null, DType::Utf8(Nullability::NonNullable)],
@@ -473,31 +460,6 @@ mod tests {
473460
assert!(!uv.is_consecutive());
474461
}
475462

476-
#[test]
477-
fn test_union_proto_rejects_mismatched_serialized_nullability() {
478-
let proto = pb::DType {
479-
dtype_type: Some(DtypeType::Union(pb::Union {
480-
names: vec!["a".to_string(), "b".to_string()],
481-
dtypes: vec![
482-
pb::DType::try_from(&DType::Primitive(PType::I32, Nullability::NonNullable))
483-
.unwrap(),
484-
pb::DType::try_from(&DType::Utf8(Nullability::NonNullable)).unwrap(),
485-
],
486-
type_ids: vec![0, 1],
487-
nullable: true,
488-
})),
489-
};
490-
491-
let result = DType::from_proto(&proto, &SESSION);
492-
assert!(result.is_err());
493-
assert!(
494-
result
495-
.unwrap_err()
496-
.to_string()
497-
.contains("Serialized Union nullability does not match its variants")
498-
);
499-
}
500-
501463
#[test]
502464
fn test_union_proto_rejects_out_of_range_type_id() {
503465
let proto = pb::DType {
@@ -509,7 +471,6 @@ mod tests {
509471
],
510472
// 200 does not fit in i8.
511473
type_ids: vec![200],
512-
nullable: false,
513474
})),
514475
};
515476

‎vortex-array/src/dtype/serde/serde.rs‎

Lines changed: 4 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -116,12 +116,7 @@ impl Serialize for DType {
116116
state.serialize_field(n)?;
117117
state.end()
118118
}
119-
DType::Union(uv) => {
120-
let mut state = serializer.serialize_tuple_variant("DType", 11, "Union", 2)?;
121-
state.serialize_field(&uv)?;
122-
state.serialize_field(&uv.derived_nullability())?;
123-
state.end()
124-
}
119+
DType::Union(uv) => serializer.serialize_newtype_variant("DType", 11, "Union", uv),
125120
DType::Variant(n) => serializer.serialize_newtype_variant("DType", 10, "Variant", n),
126121
DType::Extension(ext) => {
127122
serializer.serialize_newtype_variant("DType", 9, "Extension", ext)
@@ -237,9 +232,9 @@ impl<'de> DeserializeSeed<'de> for DTypeSerde<'_, DType> {
237232
"Struct" => access.newtype_variant_seed(StructFieldsSeed {
238233
session: self.session,
239234
}),
240-
"Union" => access.newtype_variant_seed(UnionVariantsSeed {
241-
session: self.session,
242-
}),
235+
"Union" => access
236+
.newtype_variant_seed(DTypeSerde::<UnionVariants>::new(self.session))
237+
.map(DType::Union),
243238
"Variant" => {
244239
let n = access.newtype_variant()?;
245240
Ok(DType::Variant(n))
@@ -410,58 +405,6 @@ impl<'de> DeserializeSeed<'de> for StructFieldsSeed<'_> {
410405
}
411406
}
412407

413-
struct UnionVariantsSeed<'a> {
414-
session: &'a VortexSession,
415-
}
416-
417-
impl<'de> DeserializeSeed<'de> for UnionVariantsSeed<'_> {
418-
type Value = DType;
419-
420-
fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
421-
where
422-
D: Deserializer<'de>,
423-
{
424-
struct UnionVisitor<'a> {
425-
session: &'a VortexSession,
426-
}
427-
428-
impl<'de> Visitor<'de> for UnionVisitor<'_> {
429-
type Value = DType;
430-
431-
fn expecting(&self, f: &mut Formatter) -> fmt::Result {
432-
f.write_str("Union tuple (variants, nullability)")
433-
}
434-
435-
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
436-
where
437-
A: SeqAccess<'de>,
438-
{
439-
let variants = seq
440-
.next_element_seed(DTypeSerde::<UnionVariants>::new(self.session))?
441-
.ok_or_else(|| de::Error::invalid_length(0, &self))?;
442-
let serialized_nullability: Nullability = seq
443-
.next_element()?
444-
.ok_or_else(|| de::Error::invalid_length(1, &self))?;
445-
let derived_nullability = variants.derived_nullability();
446-
if serialized_nullability != derived_nullability {
447-
return Err(de::Error::custom(format!(
448-
"Serialized Union nullability does not match its variants: serialized={:?}, derived={:?}",
449-
serialized_nullability, derived_nullability
450-
)));
451-
}
452-
Ok(DType::Union(variants))
453-
}
454-
}
455-
456-
deserializer.deserialize_tuple(
457-
2,
458-
UnionVisitor {
459-
session: self.session,
460-
},
461-
)
462-
}
463-
}
464-
465408
impl<'de> DeserializeSeed<'de> for DTypeSerde<'_, UnionVariants> {
466409
type Value = UnionVariants;
467410

‎vortex-flatbuffers/flatbuffers/vortex-dtype/dtype.fbs‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,13 +67,10 @@ table Variant {
6767
nullable: bool;
6868
}
6969

70-
// Explicit ids keep `nullable` at id 0, the id it had before the variant metadata fields were
71-
// added, so schemas written in between remain readable.
7270
table Union {
73-
names: [string] (id: 1);
74-
dtypes: [DType] (id: 2);
75-
type_ids: [byte] (id: 3); // length must equal dtypes.len()
76-
nullable: bool (id: 0);
71+
names: [string];
72+
dtypes: [DType];
73+
type_ids: [byte]; // length must equal dtypes.len()
7774
}
7875

7976
union Type {

0 commit comments

Comments
 (0)