Skip to content

Commit f68d60d

Browse files
authored
Prevent non empty unions and clarify null docs (#8882)
## Rationale for this change Tracking issue: #8769 ## What changes are included in this PR? As discussed offline, we have decided to disallow empty unions because we do not think there is a good usecase for it. And as you can see in the diffs here, that meant we could not choose a good default for the "zero value" of scalars. This PR prevents empty unions, as well as clarifies some of the null docs around the union scalar accessors. ## What APIs are changed? Are there any user-facing changes? Some name changes to make things more obvious Signed-off-by: Connor Tsui <[email protected]>
1 parent 1259557 commit f68d60d

11 files changed

Lines changed: 215 additions & 88 deletions

File tree

‎vortex-array/src/dtype/dtype_impl.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -445,7 +445,10 @@ impl DType {
445445
}
446446
}
447447

448-
/// Get the [`UnionVariants`] if `self` is a [`DType::Union`], otherwise `None`.
448+
/// Get the [`UnionVariants`] if `self` is a [`DType::Union`], otherwise [`None`].
449+
///
450+
/// This [`Option`] only represents whether the dtype is a union; Union nullability does not
451+
/// affect the result.
449452
pub fn as_union_variants_opt(&self) -> Option<&UnionVariants> {
450453
if let Union(uv, _) = self {
451454
Some(uv)
@@ -454,7 +457,7 @@ impl DType {
454457
}
455458
}
456459

457-
/// Owned version of [Self::as_union_variants_opt].
460+
/// Owned version of [`Self::as_union_variants_opt`].
458461
pub fn into_union_variants_opt(self) -> Option<UnionVariants> {
459462
if let Union(uv, _) = self {
460463
Some(uv)

‎vortex-array/src/dtype/union.rs‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,9 @@ use crate::dtype::FieldNames;
1919
/// Type information for a union array.
2020
///
2121
/// A `UnionVariants` describes the possible alternative types for each row of a union, along with a
22-
/// per-variant `u8` type tag. We use the term **variants** (rather than "fields") because a union
23-
/// is a sum type: each row chooses exactly one alternative.
22+
/// per-variant `u8` type tag. A union must contain at least one variant. We use the term
23+
/// **variants** (rather than "fields") because a union is a sum type: each row chooses exactly one
24+
/// alternative.
2425
///
2526
/// By default, tag `i` selects the child at offset `i` (`type_ids = [0, 1, ..., N-1]`). Vortex uses
2627
/// unsigned tags and supports up to 256 variants.
@@ -164,22 +165,7 @@ impl UnionVariants {
164165
}
165166
}
166167

167-
impl Default for UnionVariants {
168-
fn default() -> Self {
169-
Self::empty()
170-
}
171-
}
172-
173168
impl UnionVariants {
174-
/// The variants of the empty union.
175-
pub fn empty() -> Self {
176-
Self(Arc::new(UnionVariantsInner::from_fields(
177-
FieldNames::default(),
178-
Arc::from([]),
179-
Arc::from([]),
180-
)))
181-
}
182-
183169
/// Validate that `names`, `dtypes`, and `type_ids` are mutually consistent.
184170
fn validate_shape(names: &FieldNames, n_dtypes: usize, type_ids: &[u8]) -> VortexResult<()> {
185171
vortex_ensure_eq!(
@@ -196,6 +182,10 @@ impl UnionVariants {
196182
names.len(),
197183
type_ids.len()
198184
);
185+
vortex_ensure!(
186+
!names.is_empty(),
187+
"union must have at least one variant (for now)"
188+
);
199189

200190
vortex_ensure!(
201191
type_ids.iter().all_unique(),
@@ -221,8 +211,9 @@ impl UnionVariants {
221211
///
222212
/// # Errors
223213
///
224-
/// Returns an error if names, dtypes, or type IDs do not all have the same length, or if there
225-
/// are any duplicate names or type IDs, or if there are more than 256 variants.
214+
/// Returns an error if the union has no variants, if names, dtypes, or type IDs do not all have
215+
/// the same length, if there are any duplicate names or type IDs, or if there are more than 256
216+
/// variants.
226217
pub fn try_new(names: FieldNames, dtypes: Vec<DType>, type_ids: Vec<u8>) -> VortexResult<Self> {
227218
Self::validate_shape(&names, dtypes.len(), &type_ids)?;
228219

@@ -238,8 +229,8 @@ impl UnionVariants {
238229
///
239230
/// # Errors
240231
///
241-
/// `names` and `dtypes` must have the same length, and `names.len()` cannot be more than
242-
/// `u8::MAX as usize + 1` (256).
232+
/// The union must have at least one variant, `names` and `dtypes` must have the same length, and
233+
/// `names.len()` cannot be more than `u8::MAX as usize + 1` (256).
243234
pub fn new(names: FieldNames, dtypes: Vec<DType>) -> VortexResult<Self> {
244235
const MAX_VARIANTS: usize = u8::MAX as usize + 1;
245236
vortex_ensure!(
@@ -265,8 +256,9 @@ impl UnionVariants {
265256
///
266257
/// # Errors
267258
///
268-
/// Returns an error if names, dtypes, or type IDs do not all have the same length, or if there
269-
/// are any duplicate names or type IDs, or if there are more than 256 variants.
259+
/// Returns an error if the union has no variants, if names, dtypes, or type IDs do not all have
260+
/// the same length, if there are any duplicate names or type IDs, or if there are more than 256
261+
/// variants.
270262
pub(crate) fn try_from_fields(
271263
names: FieldNames,
272264
dtypes: Vec<FieldDType>,
@@ -330,7 +322,7 @@ impl UnionVariants {
330322
)
331323
}
332324

333-
/// Get the [`DType`] of a variant by offset.
325+
/// Get the [`DType`] of a variant by offset, or [`None`] if `index` is out of bounds.
334326
pub fn variant_by_index(&self, index: usize) -> Option<DType> {
335327
Some(
336328
self.0
@@ -600,11 +592,19 @@ mod tests {
600592
}
601593

602594
#[test]
603-
fn test_empty() {
604-
let v = UnionVariants::empty();
605-
assert!(v.is_empty());
606-
assert_eq!(v.len(), 0);
607-
assert_eq!(v.type_ids(), &[] as &[u8]);
608-
assert!(v.is_consecutive());
595+
fn test_empty_rejected() {
596+
let error = UnionVariants::new(FieldNames::default(), vec![]).unwrap_err();
597+
assert!(
598+
error
599+
.to_string()
600+
.contains("union must have at least one variant")
601+
);
602+
603+
let error = UnionVariants::try_new(FieldNames::default(), vec![], vec![]).unwrap_err();
604+
assert!(
605+
error
606+
.to_string()
607+
.contains("union must have at least one variant")
608+
);
609609
}
610610
}

‎vortex-array/src/scalar/constructor.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,9 @@ impl Scalar {
198198
/// Creates a union scalar from a type ID and child scalar.
199199
///
200200
/// The selected child's dtype is verified and then discarded. Its raw value is stored so an
201-
/// inner null child remains distinct from a null at the outer union level.
201+
/// inner null child remains distinct from a null at the outer union level. Passing a null child
202+
/// creates a non-null union with a selected type ID; use [`Scalar::null`] with a nullable
203+
/// [`DType::Union`] to create an outer null union with no selected type ID.
202204
///
203205
/// # Errors
204206
///

‎vortex-array/src/scalar/downcast.rs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,13 @@ impl Scalar {
168168
.vortex_expect("Failed to convert scalar to union")
169169
}
170170

171-
/// Returns a view of the scalar as a union scalar if it has a union type.
171+
/// Returns a view of the scalar as a union scalar if it has a union dtype.
172+
///
173+
/// [`None`] means the scalar's dtype is not [`DType::Union`](crate::dtype::DType); it does not
174+
/// describe the scalar's nullness.
175+
///
176+
/// An outer null union still returns `Some(UnionScalar)` and can be inspected with
177+
/// [`UnionScalar::is_null`](crate::scalar::UnionScalar::is_null).
172178
pub fn as_union_opt(&self) -> Option<UnionScalar<'_>> {
173179
if !self.dtype().is_union() {
174180
return None;

‎vortex-array/src/scalar/proto.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ impl From<&ScalarValue> for pb::ScalarValue {
116116
ScalarValue::Union(v) => pb::ScalarValue {
117117
kind: Some(Kind::UnionValue(Box::new(PbUnionValue {
118118
type_id: u32::from(v.type_id()),
119-
value: Some(Box::new(ScalarValue::to_proto(v.value()))),
119+
value: Some(Box::new(ScalarValue::to_proto(v.child_value()))),
120120
}))),
121121
},
122122
ScalarValue::Variant(v) => pb::ScalarValue {

‎vortex-array/src/scalar/scalar_impl.rs‎

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -101,10 +101,16 @@ impl Scalar {
101101
/// See [`Scalar::zero_value`] for more details about "zero" values.
102102
///
103103
/// For non-nullable nested types, this function recursively creates valid default children.
104+
/// For a union specifically:
105+
///
106+
/// - A nullable union defaults to an **outer null**. It has no selected variant or type ID.
107+
/// - A non-nullable union selects its first variant. That selected child may itself default to
108+
/// an **inner null** when its dtype is nullable; the enclosing union remains non-null and
109+
/// retains the first variant's type ID.
104110
///
105111
/// # Panics
106112
///
107-
/// Panics if `dtype` has no default value, such as a non-nullable union.
113+
/// Panics if `dtype` has no default value.
108114
pub fn default_value(dtype: &DType) -> Self {
109115
Self::try_default_value(dtype)
110116
.unwrap_or_else(|| vortex_panic!("{dtype} has no default value"))
@@ -133,16 +139,19 @@ impl Scalar {
133139
/// element [`DType`]
134140
/// - `Struct`: A struct where each field has a zero value, which is determined by the field
135141
/// [`DType`]
136-
/// - `Union`: Does not have a "zero" value
142+
/// - `Union`: A non-null union selecting the first variant with a non-null child zero value
137143
/// - `Extension`: The zero value of the storage [`DType`]
138-
///
139144
/// # Panics
140145
///
141-
/// Panics if the dtype has no non-null zero value, such as `Null` or a union.
146+
/// Panics if the dtype has no non-null zero value, such as `Null`, or if a nested dtype needed
147+
/// to construct the zero value has no non-null zero value.
148+
///
149+
/// Unlike [`Scalar::default_value`], this never uses either an outer or inner null for a union:
150+
/// even a nullable union's zero value is non-null and contains a non-null selected child.
142151
pub fn zero_value(dtype: &DType) -> Self {
143152
let value = ScalarValue::zero_value(dtype);
144153

145-
// SAFETY: We assume that `zero_value` creates a valid `ScalarValue` for the `DType`.
154+
// SAFETY: `zero_value` creates a valid `ScalarValue` for the `DType`.
146155
unsafe { Self::new_unchecked(dtype.clone(), Some(value)) }
147156
}
148157

@@ -186,8 +195,12 @@ impl Scalar {
186195

187196
/// Returns `true` if the [`Scalar`] has a non-null zero value.
188197
///
189-
/// Returns `None` if the scalar is null, otherwise returns `Some(true)` if the value is zero
190-
/// and `Some(false)` otherwise.
198+
/// Returns `None` if the scalar is null or its zero-ness is undefined. Otherwise, returns
199+
/// `Some(true)` if the value is zero and `Some(false)` if it is not.
200+
///
201+
/// A union that selects a variant other than the first returns `Some(false)`. A union selecting
202+
/// its first variant delegates to that child's [`Scalar::is_zero`], so an inner null child
203+
/// returns `None` and a non-null zero child returns `Some(true)`.
191204
pub fn is_zero(&self) -> Option<bool> {
192205
let value = self.value()?;
193206

@@ -212,8 +225,16 @@ impl Scalar {
212225
.as_struct()
213226
.fields_iter()
214227
.is_some_and(|mut fields| fields.all(|f| f.is_zero() == Some(true))),
215-
// We do not define union to have any valid "zero" value.
216-
DType::Union(..) => false,
228+
// Only the first variant of unions can be zero. Its child determines whether the
229+
// zero-ness is true, false, or undefined.
230+
DType::Union(..) => {
231+
let union = self.as_union();
232+
if union.child_index() != Some(0) {
233+
false
234+
} else {
235+
union.child().and_then(|child| child.is_zero())?
236+
}
237+
}
217238
DType::Variant(_) => self.as_variant().is_zero()?,
218239
DType::Extension(_) => self.as_extension().to_storage_scalar().is_zero()?,
219240
};
@@ -284,7 +305,7 @@ impl Scalar {
284305
.unwrap_or_default(),
285306
DType::Union(..) => self
286307
.as_union()
287-
.value()
308+
.child()
288309
.map_or(0, |value| 1 + value.approx_nbytes()),
289310
DType::Variant(_) => self.as_variant().value().map_or(0, Scalar::approx_nbytes),
290311
DType::Extension(_) => self.as_extension().to_storage_scalar().approx_nbytes(),
@@ -403,7 +424,7 @@ fn partial_cmp_non_null_scalar_values(
403424
let child_index = variants.tag_to_child_index(lhs.type_id())?;
404425
let child_dtype = variants.variant_by_index(child_index)?;
405426

406-
partial_cmp_scalar_values(&child_dtype, lhs.value(), rhs.value())
427+
partial_cmp_scalar_values(&child_dtype, lhs.child_value(), rhs.child_value())
407428
}
408429
// Variant values can have a different dtype in each row, so it doesn't make sense to
409430
// compare them.

‎vortex-array/src/scalar/scalar_value.rs‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ use std::fmt::Formatter;
99
use itertools::Itertools;
1010
use vortex_buffer::BufferString;
1111
use vortex_buffer::ByteBuffer;
12+
use vortex_error::VortexExpect;
1213
use vortex_error::vortex_panic;
1314

1415
use crate::dtype::DType;
@@ -51,7 +52,7 @@ impl ScalarValue {
5152
}
5253

5354
/// Returns the non-null zero value for `dtype`, or [`None`] if no such value exists.
54-
fn try_zero_value(dtype: &DType) -> Option<Self> {
55+
pub(super) fn try_zero_value(dtype: &DType) -> Option<Self> {
5556
Some(match dtype {
5657
DType::Null => return None,
5758
DType::Bool(_) => Self::Bool(false),
@@ -73,7 +74,17 @@ impl ScalarValue {
7374
.collect::<Option<Vec<_>>>()?;
7475
Self::Tuple(field_values)
7576
}
76-
DType::Union(..) => return None,
77+
DType::Union(variants, _) => {
78+
let child_dtype = variants
79+
.variant_by_index(0)
80+
.vortex_expect("union must have at least one variant");
81+
let child_value = Self::try_zero_value(&child_dtype)?;
82+
83+
Self::Union(UnionValue::new(
84+
variants.child_index_to_tag(0),
85+
Some(child_value),
86+
))
87+
}
7788
DType::Variant(_) => Self::Variant(Box::new(Scalar::null(DType::Null))),
7889
DType::Extension(ext_dtype) => {
7990
// Since we have no way to define a "zero" extension value (since we have no idea
@@ -114,7 +125,14 @@ impl ScalarValue {
114125
.collect::<Option<Vec<_>>>()?;
115126
Self::Tuple(field_values)
116127
}
117-
DType::Union(..) => return None,
128+
DType::Union(variants, _) => {
129+
let child_dtype = variants
130+
.variant_by_index(0)
131+
.vortex_expect("union must have at least one variant");
132+
let child_value = Self::try_default_value(&child_dtype)?;
133+
134+
Self::Union(UnionValue::new(variants.child_index_to_tag(0), child_value))
135+
}
118136
DType::Variant(_) => Self::Variant(Box::new(Scalar::null(DType::Null))),
119137
DType::Extension(ext_dtype) => {
120138
// Since we have no way to define a "default" extension value (since we have no idea
@@ -174,7 +192,7 @@ impl Display for ScalarValue {
174192
}
175193
ScalarValue::Union(value) => {
176194
write!(f, "union@{}(", value.type_id())?;
177-
match value.value() {
195+
match value.child_value() {
178196
Some(value) => write!(f, "{value}"),
179197
None => write!(f, "null"),
180198
}?;

‎vortex-array/src/scalar/tests/casting.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -420,15 +420,15 @@ mod tests {
420420
assert_eq!(
421421
unchanged_child
422422
.as_union()
423-
.value()
423+
.child()
424424
.map(|scalar| scalar.dtype().clone()),
425425
Some(DType::Primitive(PType::I32, Nullability::NonNullable))
426426
);
427427
assert_eq!(changed_child.dtype(), &target_dtype);
428428
assert_eq!(
429429
changed_child
430430
.as_union()
431-
.value()
431+
.child()
432432
.map(|scalar| scalar.dtype().clone()),
433433
Some(DType::Utf8(Nullability::Nullable))
434434
);
@@ -482,7 +482,7 @@ mod tests {
482482
let cast = scalar.cast(&target_dtype)?;
483483
let selected_child = cast
484484
.as_union()
485-
.value()
485+
.child()
486486
.vortex_expect("present union must have a selected child");
487487

488488
assert_eq!(cast.dtype(), &target_dtype);
@@ -531,7 +531,7 @@ mod tests {
531531
assert_eq!(cast.dtype(), &target_dtype);
532532
assert_eq!(union.dtype(), &target_union_dtype);
533533
assert_eq!(
534-
union.as_union().value(),
534+
union.as_union().child(),
535535
Some(Scalar::primitive(42_i32, Nullability::Nullable))
536536
);
537537

@@ -605,7 +605,7 @@ mod tests {
605605
assert_eq!(
606606
nullable
607607
.as_union()
608-
.value()
608+
.child()
609609
.map(|scalar| scalar.dtype().clone()),
610610
Some(DType::Primitive(PType::I32, Nullability::NonNullable))
611611
);

0 commit comments

Comments
 (0)