Skip to content

Commit 1de4f39

Browse files
committed
Add UnionVariants type and embed it in DType::Union
Replaces `DType::Union(Nullability)` with `DType::Union(UnionVariants, Nullability)`. `UnionVariants` carries a `FieldNames` list, per-variant `FieldDType`s, and an `i8` type-id vector (supporting non-consecutive tags like `[0, 5, 7]` so children can be removed without renumbering). The flatbuffer and protobuf `Union` schemas are extended with the `names`/`dtypes`/`type_ids` fields, and serde, flatbuffers, and protobuf paths are filled in with round-trip tests and nullability-constraint checks. The flatbuffer `Union` table uses explicit field ids so that `nullable` keeps the id it had before this change, keeping the schema conformant with develop. Existing `DType::Union(..)` match arms that reject or skip unions are left in place; concrete implementations land alongside the array work. Signed-off-by: Connor Tsui <[email protected]>
1 parent a649d75 commit 1de4f39

17 files changed

Lines changed: 1311 additions & 38 deletions

File tree

‎vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,7 @@ impl AggregateFnVTable for IsSorted {
247247
| DType::List(..)
248248
| DType::FixedSizeList(..)
249249
| DType::Struct(..)
250-
| DType::Union(_)
250+
| DType::Union(..)
251251
| DType::Variant(..)
252252
| DType::Extension(_) => None,
253253
DType::Bool(_)
@@ -264,7 +264,7 @@ impl AggregateFnVTable for IsSorted {
264264
| DType::List(..)
265265
| DType::FixedSizeList(..)
266266
| DType::Struct(..)
267-
| DType::Union(_)
267+
| DType::Union(..)
268268
| DType::Variant(..)
269269
| DType::Extension(_) => None,
270270
DType::Bool(_)

‎vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,9 @@ fn supports_uncompressed_size_in_bytes(dtype: &DType) -> bool {
289289
DType::Struct(fields, _) => fields
290290
.fields()
291291
.all(|field| supports_uncompressed_size_in_bytes(&field)),
292-
DType::Union(_) => todo!("TODO(connor)[Union]: unimplemented"),
292+
DType::Union(variants, _) => variants
293+
.variants()
294+
.all(|variant| supports_uncompressed_size_in_bytes(&variant)),
293295
DType::Variant(_) => false,
294296
DType::Extension(ext_dtype) => {
295297
supports_uncompressed_size_in_bytes(ext_dtype.storage_dtype())

‎vortex-array/src/dtype/dtype_impl.rs‎

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ use crate::dtype::FieldDType;
1515
use crate::dtype::FieldName;
1616
use crate::dtype::PType;
1717
use crate::dtype::StructFields;
18+
use crate::dtype::UnionVariants;
1819
use crate::dtype::decimal::DecimalDType;
1920
use crate::dtype::decimal::DecimalType;
2021
use crate::dtype::extension::ExtDTypeRef;
@@ -63,7 +64,7 @@ impl DType {
6364
| List(_, null)
6465
| FixedSizeList(_, _, null)
6566
| Struct(_, null)
66-
| Union(null)
67+
| Union(_, null)
6768
| Variant(null) => matches!(null, Nullability::Nullable),
6869
Extension(ext_dtype) => ext_dtype.storage_dtype().is_nullable(),
6970
}
@@ -91,7 +92,7 @@ impl DType {
9192
List(edt, _) => List(Arc::clone(edt), nullability),
9293
FixedSizeList(edt, size, _) => FixedSizeList(Arc::clone(edt), *size, nullability),
9394
Struct(sf, _) => Struct(sf.clone(), nullability),
94-
Union(_) => Union(nullability),
95+
Union(vs, _) => Union(vs.clone(), nullability),
9596
Variant(_) => Variant(nullability),
9697
Extension(ext) => Extension(ext.with_nullability(nullability)),
9798
}
@@ -123,7 +124,15 @@ impl DType {
123124
.zip_eq(rhs_dtype.fields())
124125
.all(|(l, r)| l.eq_ignore_nullability(&r)))
125126
}
126-
(Union(_), Union(_)) => true,
127+
(Union(lhs, _), Union(rhs, _)) => {
128+
// Equal `names` implies equal length by FieldNames equality.
129+
lhs.names() == rhs.names()
130+
&& lhs.type_ids() == rhs.type_ids()
131+
&& lhs
132+
.variants()
133+
.zip_eq(rhs.variants())
134+
.all(|(l, r)| l.eq_ignore_nullability(&r))
135+
}
127136
(Variant(_), Variant(_)) => true,
128137
(Extension(lhs_extdtype), Extension(rhs_extdtype)) => {
129138
lhs_extdtype.eq_ignore_nullability(rhs_extdtype)
@@ -425,6 +434,24 @@ impl DType {
425434
}
426435
}
427436

437+
/// Get the [`UnionVariants`] if `self` is a [`DType::Union`], otherwise `None`.
438+
pub fn as_union_variants_opt(&self) -> Option<&UnionVariants> {
439+
if let Union(uv, _) = self {
440+
Some(uv)
441+
} else {
442+
None
443+
}
444+
}
445+
446+
/// Owned version of [Self::as_union_variants_opt].
447+
pub fn into_union_variants_opt(self) -> Option<UnionVariants> {
448+
if let Union(uv, _) = self {
449+
Some(uv)
450+
} else {
451+
None
452+
}
453+
}
454+
428455
/// Downcast a `DType` to an `ExtDType`
429456
pub fn as_extension(&self) -> &ExtDTypeRef {
430457
let Extension(ext) = self else {
@@ -476,7 +503,7 @@ impl Display for DType {
476503
.map(|(field_null, dt)| format!("{field_null}={dt}"))
477504
.join(", "),
478505
),
479-
Union(null) => write!(f, "union(){null}"),
506+
Union(uv, null) => write!(f, "union({uv}){null}"),
480507
Variant(null) => write!(f, "variant{null}"),
481508
Extension(ext) => write!(f, "{}", ext),
482509
}

‎vortex-array/src/dtype/mod.rs‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ mod ptype;
3131
pub mod serde;
3232
pub mod session;
3333
mod struct_;
34+
mod union;
3435

3536
use std::sync::Arc;
3637

@@ -110,11 +111,11 @@ pub enum DType {
110111

111112
/// A logical union (sum) type.
112113
///
113-
/// `Union` is reserved for values that are drawn from one of several possible child domains.
114-
/// The exact child-type metadata and canonical storage are not yet part of the stable public
115-
/// Rust API, so most callers should prefer [`DType::Variant`] for dynamically typed values or
116-
/// [`DType::Struct`] for fixed schemas.
117-
Union(Nullability),
114+
/// A `Union` is composed of one or more **variants**, each with a name and a `DType`. A per-row
115+
/// `i8` tag selects which variant is "live" at that row.
116+
///
117+
/// See [`UnionVariants`] for the type-tag conventions and accessors.
118+
Union(UnionVariants, Nullability),
118119

119120
/// Dynamically typed values stored as Vortex scalars.
120121
///
@@ -146,7 +147,8 @@ impl PartialEq for DType {
146147
}
147148
// StructFields handles its own Arc::ptr_eq in its PartialEq impl.
148149
(Self::Struct(a, na), Self::Struct(b, nb)) => na == nb && a == b,
149-
(Self::Union(a), Self::Union(b)) => a == b,
150+
// UnionVariants handles its own Arc::ptr_eq in its PartialEq impl.
151+
(Self::Union(a, na), Self::Union(b, nb)) => na == nb && a == b,
150152
(Self::Variant(a), Self::Variant(b)) => a == b,
151153
(Self::Extension(a), Self::Extension(b)) => a == b,
152154
// Every variant is listed in the first position so that adding a new
@@ -178,6 +180,7 @@ pub use half;
178180
pub use nullability::*;
179181
pub use ptype::*;
180182
pub use struct_::*;
183+
pub use union::*;
181184

182185
use crate::dtype::extension::ExtDTypeRef;
183186

0 commit comments

Comments
 (0)