Skip to content

Commit a10d4f5

Browse files
authored
Merge dd4420b into 044a741
2 parents 044a741 + dd4420b commit a10d4f5

7 files changed

Lines changed: 334 additions & 28 deletions

File tree

‎vortex-array/src/builders/fixed_size_list.rs‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ use crate::arrays::fixed_size_list::FixedSizeListArraySlotsExt;
1919
use crate::builders::ArrayBuilder;
2020
use crate::builders::ChildBuilder;
2121
use crate::builders::DEFAULT_BUILDER_CAPACITY;
22-
use crate::builders::LazyBitBufferBuilder;
22+
use crate::builders::ValidityBuilder;
2323
use crate::canonical::Canonical;
2424
use crate::dtype::DType;
2525
use crate::dtype::Nullability;
@@ -39,7 +39,7 @@ pub struct FixedSizeListBuilder {
3939
/// The null map builder of the [`FixedSizeListArray`].
4040
///
4141
/// We also use this type to store the length of the final output array.
42-
nulls: LazyBitBufferBuilder,
42+
nulls: ValidityBuilder,
4343
}
4444

4545
impl FixedSizeListBuilder {
@@ -64,7 +64,7 @@ impl FixedSizeListBuilder {
6464

6565
let elements_builder = ChildBuilder::with_capacity(&element_dtype, elements_capacity);
6666
let fsl_dtype = DType::FixedSizeList(element_dtype, list_size, nullability);
67-
let nulls = LazyBitBufferBuilder::new(capacity);
67+
let nulls = ValidityBuilder::new(capacity);
6868

6969
Self {
7070
dtype: fsl_dtype,
@@ -116,8 +116,7 @@ impl FixedSizeListBuilder {
116116
}
117117

118118
self.elements_builder.append_array(array.elements(), ctx)?;
119-
self.nulls
120-
.append_validity_mask(&array.validity()?.execute_mask(array.len(), ctx)?);
119+
self.nulls.append_validity(array.validity()?, array.len());
121120
Ok(())
122121
}
123122

@@ -262,7 +261,7 @@ impl ArrayBuilder for FixedSizeListBuilder {
262261
}
263262

264263
unsafe fn set_validity_unchecked(&mut self, validity: Mask) {
265-
self.nulls = LazyBitBufferBuilder::from_validity_mask(validity);
264+
self.nulls.set_validity(validity);
266265
}
267266

268267
fn finish(&mut self) -> ArrayRef {

‎vortex-array/src/builders/list.rs‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@ use crate::arrays::listview::ListViewRebuildMode;
2828
use crate::builders::ArrayBuilder;
2929
use crate::builders::ChildBuilder;
3030
use crate::builders::DEFAULT_BUILDER_CAPACITY;
31-
use crate::builders::LazyBitBufferBuilder;
3231
use crate::builders::PrimitiveBuilder;
32+
use crate::builders::ValidityBuilder;
3333
use crate::dtype::DType;
3434
use crate::dtype::IntegerPType;
3535
use crate::dtype::Nullability;
@@ -52,7 +52,7 @@ pub struct ListBuilder<O: OffsetBuilderPType> {
5252
offsets_builder: PrimitiveBuilder<O>,
5353

5454
/// The null map builder of the [`ListArray`].
55-
nulls: LazyBitBufferBuilder,
55+
nulls: ValidityBuilder,
5656
}
5757

5858
impl<O: OffsetBuilderPType> ListBuilder<O> {
@@ -90,7 +90,7 @@ impl<O: OffsetBuilderPType> ListBuilder<O> {
9090
Self {
9191
elements_builder,
9292
offsets_builder,
93-
nulls: LazyBitBufferBuilder::new(capacity),
93+
nulls: ValidityBuilder::new(capacity),
9494
dtype: DType::List(value_dtype, nullability),
9595
}
9696
}
@@ -191,8 +191,7 @@ impl<O: OffsetBuilderPType> ListBuilder<O> {
191191
return Ok(());
192192
}
193193

194-
self.nulls
195-
.append_validity_mask(&array.validity()?.execute_mask(array.len(), ctx)?);
194+
self.nulls.append_validity(array.validity()?, array.len());
196195

197196
let num_lists = array.len();
198197
let offsets = array.offsets().clone().execute::<PrimitiveArray>(ctx)?;
@@ -242,8 +241,7 @@ impl<O: OffsetBuilderPType> ListBuilder<O> {
242241
return Ok(());
243242
}
244243

245-
self.nulls
246-
.append_validity_mask(&array.validity()?.execute_mask(array.len(), ctx)?);
244+
self.nulls.append_validity(array.validity()?, array.len());
247245

248246
// Flatten the views into the only layout `ListArray` offsets can express. This is a cheap
249247
// clone when they already are laid out that way, and the flattened result keeps the
@@ -380,7 +378,7 @@ impl<O: OffsetBuilderPType> ArrayBuilder for ListBuilder<O> {
380378
}
381379

382380
unsafe fn set_validity_unchecked(&mut self, validity: Mask) {
383-
self.nulls = LazyBitBufferBuilder::from_validity_mask(validity);
381+
self.nulls.set_validity(validity);
384382
}
385383

386384
fn finish(&mut self) -> ArrayRef {

‎vortex-array/src/builders/listview.rs‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ use crate::builders::ChildBuilder;
3535
use crate::builders::DEFAULT_BUILDER_CAPACITY;
3636
use crate::builders::PrimitiveBuilder;
3737
use crate::builders::UninitRange;
38-
use crate::builders::lazy_null_builder::LazyBitBufferBuilder;
38+
use crate::builders::ValidityBuilder;
3939
use crate::builtins::ArrayBuiltins;
4040
use crate::dtype::DType;
4141
use crate::dtype::IntegerPType;
@@ -68,7 +68,7 @@ pub struct ListViewBuilder<O: OffsetBuilderPType, S: OffsetBuilderPType> {
6868
sizes_builder: PrimitiveBuilder<S>,
6969

7070
/// The null map builder of the [`ListViewArray`].
71-
nulls: LazyBitBufferBuilder,
71+
nulls: ValidityBuilder,
7272

7373
/// Whether the appends so far leave the result zero-copyable to a [`ListArray`].
7474
///
@@ -112,7 +112,7 @@ impl<O: OffsetBuilderPType, S: OffsetBuilderPType> ListViewBuilder<O, S> {
112112
let sizes_builder =
113113
PrimitiveBuilder::<S>::with_capacity(Nullability::NonNullable, capacity);
114114

115-
let nulls = LazyBitBufferBuilder::new(capacity);
115+
let nulls = ValidityBuilder::new(capacity);
116116

117117
Self {
118118
dtype: DType::List(element_dtype, nullability),
@@ -259,8 +259,7 @@ impl<O: OffsetBuilderPType, S: OffsetBuilderPType> ListViewBuilder<O, S> {
259259
return Ok(());
260260
}
261261

262-
self.nulls
263-
.append_validity_mask(&array.validity()?.execute_mask(array.len(), ctx)?);
262+
self.nulls.append_validity(array.validity()?, array.len());
264263

265264
let offsets = array.offsets().clone().execute::<PrimitiveArray>(ctx)?;
266265
match_each_integer_ptype!(offsets.ptype(), |OffsetType| {
@@ -304,8 +303,7 @@ impl<O: OffsetBuilderPType, S: OffsetBuilderPType> ListViewBuilder<O, S> {
304303
// it lands flush against the elements already in the builder. Any other layout does not.
305304
self.zero_copy_to_list &= listview.is_zero_copy_to_list();
306305

307-
self.nulls
308-
.append_validity_mask(&array.validity()?.execute_mask(array.len(), ctx)?);
306+
self.nulls.append_validity(array.validity()?, array.len());
309307

310308
// Bulk append the trimmed elements; the offsets are rebased onto them below.
311309
let old_elements_len = self.elements_builder.len();
@@ -419,7 +417,7 @@ impl<O: OffsetBuilderPType, S: OffsetBuilderPType> ArrayBuilder for ListViewBuil
419417
}
420418

421419
unsafe fn set_validity_unchecked(&mut self, validity: Mask) {
422-
self.nulls = LazyBitBufferBuilder::from_validity_mask(validity);
420+
self.nulls.set_validity(validity);
423421
}
424422

425423
fn finish(&mut self) -> ArrayRef {

‎vortex-array/src/builders/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ mod listview;
6565
mod null;
6666
mod primitive;
6767
mod struct_;
68+
mod validity;
6869
mod varbinview;
6970

7071
pub use bool::*;
@@ -77,6 +78,7 @@ pub use listview::*;
7778
pub use null::*;
7879
pub use primitive::*;
7980
pub use struct_::*;
81+
pub(crate) use validity::ValidityBuilder;
8082
pub use varbinview::*;
8183

8284
pub use crate::arrays::varbin::builder::DynVarBinBuilder;

‎vortex-array/src/builders/struct_.rs‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ use crate::arrays::struct_::StructArrayExt;
1919
use crate::builders::ArrayBuilder;
2020
use crate::builders::ChildBuilder;
2121
use crate::builders::DEFAULT_BUILDER_CAPACITY;
22-
use crate::builders::LazyBitBufferBuilder;
22+
use crate::builders::ValidityBuilder;
2323
use crate::canonical::Canonical;
2424
use crate::dtype::DType;
2525
use crate::dtype::Nullability;
@@ -31,7 +31,7 @@ use crate::scalar::StructScalar;
3131
pub struct StructBuilder {
3232
dtype: DType,
3333
builders: Vec<ChildBuilder>,
34-
nulls: LazyBitBufferBuilder,
34+
nulls: ValidityBuilder,
3535
}
3636

3737
impl StructBuilder {
@@ -53,7 +53,7 @@ impl StructBuilder {
5353

5454
Self {
5555
builders,
56-
nulls: LazyBitBufferBuilder::new(capacity),
56+
nulls: ValidityBuilder::new(capacity),
5757
dtype: DType::Struct(struct_dtype, nullability),
5858
}
5959
}
@@ -134,8 +134,7 @@ impl StructBuilder {
134134
builder.append_array(field, ctx)?;
135135
}
136136

137-
self.nulls
138-
.append_validity_mask(&array.validity()?.execute_mask(array.len(), ctx)?);
137+
self.nulls.append_validity(array.validity()?, array.len());
139138
Ok(())
140139
}
141140
}
@@ -192,7 +191,7 @@ impl ArrayBuilder for StructBuilder {
192191
}
193192

194193
unsafe fn set_validity_unchecked(&mut self, validity: Mask) {
195-
self.nulls = LazyBitBufferBuilder::from_validity_mask(validity);
194+
self.nulls.set_validity(validity);
196195
}
197196

198197
fn finish(&mut self) -> ArrayRef {

‎vortex-array/src/builders/tests.rs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1080,6 +1080,37 @@ fn test_struct_builder_interleaves_arrays_and_scalars() -> VortexResult<()> {
10801080
/// length long enough to tell chunks apart.
10811081
const CHUNK_LEN: usize = 64;
10821082

1083+
/// A nested builder's own validity is accumulated the same way its children are: an appended
1084+
/// array's validity is kept as it arrived rather than executed into a mask and copied bit by bit.
1085+
#[test]
1086+
fn test_appended_validity_is_not_materialized() -> VortexResult<()> {
1087+
let mut ctx = array_session().create_execution_ctx();
1088+
1089+
let array = StructArray::try_from_iter_with_validity(
1090+
[("a", iota(CHUNK_LEN))],
1091+
Validity::from_iter((0..CHUNK_LEN).map(|i| i % 3 != 0)),
1092+
)?
1093+
.into_array();
1094+
1095+
let mut builder = builder_with_capacity(array.dtype(), 0);
1096+
array.append_to_builder(builder.as_mut(), &mut ctx)?;
1097+
array.append_to_builder(builder.as_mut(), &mut ctx)?;
1098+
let built = builder.finish();
1099+
1100+
let Validity::Array(validity) = built.validity()? else {
1101+
panic!("expected array-backed validity");
1102+
};
1103+
assert!(
1104+
validity.is::<Chunked>(),
1105+
"the two appended validities should have been concatenated, not copied into one buffer",
1106+
);
1107+
1108+
let expected = ChunkedArray::try_new(vec![array.clone(), array], built.dtype().clone())?;
1109+
assert_arrays_eq!(&built, &expected, &mut ctx);
1110+
1111+
Ok(())
1112+
}
1113+
10831114
/// A non-canonical array of [`CHUNK_LEN`] `i32` values.
10841115
fn constant_i32() -> ArrayRef {
10851116
ConstantArray::new(0i32, CHUNK_LEN).into_array()

0 commit comments

Comments
 (0)