Skip to content

Commit 7240688

Browse files
committed
wip
Signed-off-by: Joe Isaacs <[email protected]>
1 parent e69623f commit 7240688

5 files changed

Lines changed: 316 additions & 21 deletions

File tree

‎vortex-array/src/arrays/struct_/array.rs‎

Lines changed: 122 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use vortex_error::vortex_err;
1919
use crate::Array;
2020
use crate::ArrayRef;
2121
use crate::IntoArray;
22+
use crate::builtins::ArrayBuiltins;
2223
use crate::compute::mask;
2324
use crate::stats::ArrayStats;
2425
use crate::validity::Validity;
@@ -27,8 +28,10 @@ use crate::vtable::ValidityHelper;
2728
/// Metadata for StructArray serialization.
2829
#[derive(Clone, prost::Message)]
2930
pub struct StructMetadata {
30-
/// true = child validity is a superset of struct validity (validity was pushed down)
31-
/// false = default, no guarantee about relationship
31+
/// If true, child validity is a superset of struct validity (validity was pushed down).
32+
/// For nullable children, their validity already includes struct nulls. For non-nullable
33+
/// children, we apply struct validity on field read. If false (default), no guarantee
34+
/// about relationship - must intersect validities on read.
3235
#[prost(bool, tag = "1")]
3336
pub(super) validity_pushed_down: bool,
3437
}
@@ -171,12 +174,74 @@ pub struct StructArrayParts {
171174
}
172175

173176
impl StructArray {
174-
/// Return the struct fields without the validity of the struct applied
177+
/// Note this field may not have the validity of the parent struct applied.
178+
/// Should use `masked_fields` instead, unless you know what you are doing.
175179
pub fn unmasked_fields(&self) -> &Arc<[ArrayRef]> {
176180
&self.fields
177181
}
178182

179-
/// Return the struct field without the validity of the struct applied
183+
pub fn masked_fields(&self) -> VortexResult<Vec<ArrayRef>> {
184+
if !self.dtype.is_nullable() {
185+
// fields need not be masked
186+
return Ok(self.fields.to_vec());
187+
}
188+
189+
if self.has_validity_pushed_down() {
190+
self.fields
191+
.iter()
192+
.cloned()
193+
.map(|f| {
194+
if f.dtype().is_nullable() {
195+
Ok(f.into_array())
196+
} else {
197+
let validity = self.validity().to_array(self.len);
198+
f.mask(validity)
199+
}
200+
})
201+
.collect::<VortexResult<Vec<_>>>()
202+
} else {
203+
// Apply struct validity to all fields
204+
let struct_validity = self.validity().to_array(self.len);
205+
self.fields
206+
.iter()
207+
.map(move |f| f.clone().mask(struct_validity.clone()))
208+
.collect::<VortexResult<Vec<_>>>()
209+
}
210+
}
211+
212+
pub fn field_by_name(&self, name: impl AsRef<str>) -> VortexResult<ArrayRef> {
213+
let name = name.as_ref();
214+
self.field_by_name_opt(name)?.ok_or_else(|| {
215+
vortex_err!(
216+
"Field {name} not found in struct array with names {:?}",
217+
self.names()
218+
)
219+
})
220+
}
221+
222+
pub fn field_by_name_opt(&self, name: impl AsRef<str>) -> VortexResult<Option<ArrayRef>> {
223+
let name = name.as_ref();
224+
self.struct_fields()
225+
.find(name)
226+
.map(|idx| {
227+
let field = self.fields[idx].clone();
228+
// Non-nullable struct: return field as-is (no struct validity to apply)
229+
if !self.dtype.is_nullable() {
230+
return Ok(field);
231+
}
232+
// Non-nullable field: always apply struct validity (even with validity_pushed_down,
233+
// since we can't push validity to non-nullable fields)
234+
if !field.dtype().is_nullable() {
235+
return field.mask(self.validity().to_array(self.len));
236+
}
237+
// Nullable field: return as-is (validity is either in the field or was pushed down)
238+
Ok(field)
239+
})
240+
.transpose()
241+
}
242+
243+
/// Note this field may not have the validity of the parent struct applied.
244+
/// Should use `field_by_name` instead.
180245
pub fn unmasked_field_by_name(&self, name: impl AsRef<str>) -> VortexResult<&ArrayRef> {
181246
let name = name.as_ref();
182247
self.unmasked_field_by_name_opt(name).ok_or_else(|| {
@@ -187,7 +252,8 @@ impl StructArray {
187252
})
188253
}
189254

190-
/// Return the struct field without the validity of the struct applied
255+
/// Note this field may not have the validity of the parent struct applied.
256+
/// Should use `field_by_name_opt` instead.
191257
pub fn unmasked_field_by_name_opt(&self, name: impl AsRef<str>) -> Option<&ArrayRef> {
192258
let name = name.as_ref();
193259
self.struct_fields().find(name).map(|idx| &self.fields[idx])
@@ -495,13 +561,54 @@ impl StructArray {
495561
/// the struct's nulls baked in). This is an optimization that allows readers to
496562
/// skip combining struct+child validity when extracting fields.
497563
pub fn has_validity_pushed_down(&self) -> bool {
564+
#[cfg(debug_assertions)]
565+
if self.validity_pushed_down {
566+
self.check_validity_pushed_down()
567+
.vortex_expect("validity_pushed_down invariant violated");
568+
}
498569
self.validity_pushed_down
499570
}
500571

572+
/// Checks that the validity_pushed_down invariant holds.
573+
///
574+
/// When `validity_pushed_down` is true, for every nullable child field,
575+
/// the child's validity must be a superset of the struct's validity.
576+
/// That is, wherever the struct is invalid (null), the child must also be invalid.
577+
fn check_validity_pushed_down(&self) -> VortexResult<()> {
578+
if !self.validity_pushed_down {
579+
return Ok(());
580+
}
581+
582+
let struct_validity = self.validity_mask()?;
583+
584+
for (idx, field) in self.fields.iter().enumerate() {
585+
// Only check nullable children - non-nullable children cannot have validity pushed down
586+
if !field.dtype().is_nullable() {
587+
continue;
588+
}
589+
590+
let child_validity = field.validity_mask()?;
591+
592+
// Check invariant: struct_invalid => child_invalid
593+
// Equivalently: (!struct_validity) & child_validity should be all-false
594+
// If struct is invalid (false) but child is valid (true), that's a violation
595+
let violation = &(!&struct_validity) & &child_validity;
596+
if !violation.all_false() {
597+
vortex_bail!(
598+
"validity_pushed_down invariant violated for field {}: \
599+
struct has nulls at positions where child is valid",
600+
idx
601+
);
602+
}
603+
}
604+
605+
Ok(())
606+
}
607+
501608
/// Set the validity_pushed_down flag.
502609
///
503-
/// For non-nullable structs, this is a no-op (flag stays false) since there's
504-
/// no validity to push down.
610+
/// For non-nullable structs, this is a no-op (flag stays false) since there's no validity to
611+
/// push down
505612
///
506613
/// For nullable structs, setting this to true indicates that child validity
507614
/// is a superset of struct validity (children include struct's nulls).
@@ -511,6 +618,13 @@ impl StructArray {
511618
return self;
512619
}
513620
self.validity_pushed_down = validity_pushed_down;
621+
622+
#[cfg(debug_assertions)]
623+
if validity_pushed_down {
624+
self.check_validity_pushed_down()
625+
.vortex_expect("validity_pushed_down invariant violated");
626+
}
627+
514628
self
515629
}
516630

@@ -544,7 +658,7 @@ impl StructArray {
544658
// Apply mask only to nullable children - non-nullable children cannot have their
545659
// dtype changed, so we leave them alone
546660
let new_fields: Vec<ArrayRef> = self
547-
.fields()
661+
.unmasked_fields()
548662
.iter()
549663
.map(|field| {
550664
if field.dtype().is_nullable() {

‎vortex-array/src/arrays/struct_/tests.rs‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ use vortex_dtype::FieldNames;
88
use vortex_dtype::Nullability;
99
use vortex_dtype::PType;
1010
use vortex_error::VortexResult;
11+
use vortex_mask::Mask;
1112

1213
use crate::Array;
1314
use crate::IntoArray;
@@ -150,3 +151,193 @@ fn test_uncompressed_size_in_bytes() -> VortexResult<()> {
150151
assert_eq!(uncompressed_size, Some(4000));
151152
Ok(())
152153
}
154+
155+
#[test]
156+
fn test_masked_fields_without_validity_pushed_down() -> VortexResult<()> {
157+
// Create a nullable struct with struct-level nulls at positions 1 and 3
158+
// Fields are non-nullable, so struct validity should be applied to all fields
159+
let field_a = PrimitiveArray::new(buffer![10i32, 20, 30, 40], Validity::NonNullable);
160+
let field_b = PrimitiveArray::new(buffer![100i32, 200, 300, 400], Validity::NonNullable);
161+
162+
let struct_validity = Validity::Array(
163+
BoolArray::from_iter([true, false, true, false]).into_array(), // positions 1, 3 are null
164+
);
165+
let expected_mask = Mask::from_iter([true, false, true, false]);
166+
167+
let struct_array = StructArray::try_new(
168+
FieldNames::from(["a", "b"]),
169+
vec![field_a.into_array(), field_b.into_array()],
170+
4,
171+
struct_validity,
172+
)?;
173+
174+
// Verify validity is NOT pushed down by default
175+
assert!(!struct_array.has_validity_pushed_down());
176+
177+
// === Test masked_fields ===
178+
let masked = struct_array.masked_fields()?;
179+
assert_eq!(masked.len(), 2);
180+
181+
// Both fields should have struct validity applied
182+
assert!(masked[0].dtype().is_nullable());
183+
assert_eq!(masked[0].validity_mask()?, expected_mask);
184+
assert!(masked[1].dtype().is_nullable());
185+
assert_eq!(masked[1].validity_mask()?, expected_mask);
186+
187+
// === Test field_by_name ===
188+
let field_a_by_name = struct_array.field_by_name("a")?;
189+
assert!(field_a_by_name.dtype().is_nullable());
190+
assert_eq!(field_a_by_name.validity_mask()?, expected_mask);
191+
192+
let field_b_by_name = struct_array.field_by_name("b")?;
193+
assert!(field_b_by_name.dtype().is_nullable());
194+
assert_eq!(field_b_by_name.validity_mask()?, expected_mask);
195+
196+
Ok(())
197+
}
198+
199+
#[test]
200+
fn test_masked_fields_with_validity_pushed_down() -> VortexResult<()> {
201+
// Create nullable fields where validity has already been pushed down
202+
// (child nulls include struct nulls at positions 1 and 3)
203+
let field_a = PrimitiveArray::new(
204+
buffer![10i32, 20, 30, 40],
205+
Validity::Array(BoolArray::from_iter([true, false, true, false]).into_array()),
206+
);
207+
let field_b = PrimitiveArray::new(
208+
buffer![100i32, 200, 300, 400],
209+
Validity::Array(BoolArray::from_iter([true, false, true, false]).into_array()),
210+
);
211+
212+
let struct_validity =
213+
Validity::Array(BoolArray::from_iter([true, false, true, false]).into_array());
214+
let expected_mask = Mask::from_iter([true, false, true, false]);
215+
216+
let struct_array = StructArray::try_new(
217+
FieldNames::from(["a", "b"]),
218+
vec![field_a.into_array(), field_b.into_array()],
219+
4,
220+
struct_validity,
221+
)?
222+
.with_validity_pushed_down(true);
223+
224+
// Verify validity IS pushed down
225+
assert!(struct_array.has_validity_pushed_down());
226+
227+
// === Test masked_fields ===
228+
let masked = struct_array.masked_fields()?;
229+
assert_eq!(masked.len(), 2);
230+
231+
// Nullable fields returned as-is (validity already pushed)
232+
assert!(masked[0].dtype().is_nullable());
233+
assert_eq!(masked[0].validity_mask()?, expected_mask);
234+
assert!(masked[1].dtype().is_nullable());
235+
assert_eq!(masked[1].validity_mask()?, expected_mask);
236+
237+
// === Test field_by_name ===
238+
let field_a_by_name = struct_array.field_by_name("a")?;
239+
assert!(field_a_by_name.dtype().is_nullable());
240+
assert_eq!(field_a_by_name.validity_mask()?, expected_mask);
241+
242+
let field_b_by_name = struct_array.field_by_name("b")?;
243+
assert!(field_b_by_name.dtype().is_nullable());
244+
assert_eq!(field_b_by_name.validity_mask()?, expected_mask);
245+
246+
Ok(())
247+
}
248+
249+
#[test]
250+
fn test_masked_fields_mixed_nullability_with_pushed_down() -> VortexResult<()> {
251+
// Create a struct with mixed nullable and non-nullable fields
252+
// Nullable field has validity pushed down, non-nullable field needs masking
253+
let nullable_field = PrimitiveArray::new(
254+
buffer![10i32, 20, 30, 40],
255+
Validity::Array(BoolArray::from_iter([true, false, true, false]).into_array()),
256+
);
257+
let non_nullable_field =
258+
PrimitiveArray::new(buffer![100i32, 200, 300, 400], Validity::NonNullable);
259+
260+
let struct_validity =
261+
Validity::Array(BoolArray::from_iter([true, false, true, false]).into_array());
262+
let expected_mask = Mask::from_iter([true, false, true, false]);
263+
264+
let struct_array = StructArray::try_new(
265+
FieldNames::from(["nullable", "non_nullable"]),
266+
vec![nullable_field.into_array(), non_nullable_field.into_array()],
267+
4,
268+
struct_validity,
269+
)?
270+
.with_validity_pushed_down(true);
271+
272+
assert!(struct_array.has_validity_pushed_down());
273+
274+
// === Test masked_fields ===
275+
let masked = struct_array.masked_fields()?;
276+
277+
// Nullable field: returned as-is (validity already pushed down)
278+
assert!(masked[0].dtype().is_nullable());
279+
assert_eq!(masked[0].validity_mask()?, expected_mask);
280+
281+
// Non-nullable field: struct validity applied (becomes nullable)
282+
assert!(masked[1].dtype().is_nullable());
283+
assert_eq!(masked[1].validity_mask()?, expected_mask);
284+
285+
// === Test field_by_name ===
286+
let nullable_by_name = struct_array.field_by_name("nullable")?;
287+
assert!(nullable_by_name.dtype().is_nullable());
288+
assert_eq!(nullable_by_name.validity_mask()?, expected_mask);
289+
290+
let non_nullable_by_name = struct_array.field_by_name("non_nullable")?;
291+
assert!(non_nullable_by_name.dtype().is_nullable());
292+
assert_eq!(non_nullable_by_name.validity_mask()?, expected_mask);
293+
294+
Ok(())
295+
}
296+
297+
#[test]
298+
fn test_masked_fields_non_nullable_struct() -> VortexResult<()> {
299+
// Non-nullable struct: fields should be returned unchanged
300+
let field_a = PrimitiveArray::new(buffer![10i32, 20, 30, 40], Validity::NonNullable);
301+
let field_b = PrimitiveArray::new(
302+
buffer![100i32, 200, 300, 400],
303+
Validity::Array(BoolArray::from_iter([true, true, false, true]).into_array()),
304+
);
305+
let expected_mask_all_valid = Mask::new_true(4);
306+
let expected_mask_b = Mask::from_iter([true, true, false, true]);
307+
308+
let struct_array = StructArray::try_new(
309+
FieldNames::from(["a", "b"]),
310+
vec![field_a.into_array(), field_b.into_array()],
311+
4,
312+
Validity::NonNullable,
313+
)?;
314+
315+
// Non-nullable struct cannot have validity pushed down
316+
assert!(!struct_array.has_validity_pushed_down());
317+
318+
// with_validity_pushed_down should be a no-op for non-nullable structs
319+
let struct_array = struct_array.with_validity_pushed_down(true);
320+
assert!(!struct_array.has_validity_pushed_down());
321+
322+
// === Test masked_fields ===
323+
let masked = struct_array.masked_fields()?;
324+
325+
// Non-nullable field: stays non-nullable (no struct validity to apply)
326+
assert!(!masked[0].dtype().is_nullable());
327+
assert_eq!(masked[0].validity_mask()?, expected_mask_all_valid);
328+
329+
// Nullable field: keeps its own validity
330+
assert!(masked[1].dtype().is_nullable());
331+
assert_eq!(masked[1].validity_mask()?, expected_mask_b);
332+
333+
// === Test field_by_name ===
334+
let field_a_by_name = struct_array.field_by_name("a")?;
335+
assert!(!field_a_by_name.dtype().is_nullable());
336+
assert_eq!(field_a_by_name.validity_mask()?, expected_mask_all_valid);
337+
338+
let field_b_by_name = struct_array.field_by_name("b")?;
339+
assert!(field_b_by_name.dtype().is_nullable());
340+
assert_eq!(field_b_by_name.validity_mask()?, expected_mask_b);
341+
342+
Ok(())
343+
}

0 commit comments

Comments
 (0)