Skip to content

Commit 83dbd41

Browse files
committed
fixes
Signed-off-by: Robert Kruszewski <[email protected]>
1 parent 64c3464 commit 83dbd41

3 files changed

Lines changed: 28 additions & 29 deletions

File tree

‎vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,7 @@ mod tests {
361361
use crate::arrays::VarBinViewArray;
362362
use crate::arrays::VariantArray;
363363
use crate::arrays::listview::ListViewRebuildMode;
364+
use crate::builders::builder_with_capacity;
364365
use crate::dtype::DType;
365366
use crate::dtype::DecimalDType;
366367
use crate::dtype::FieldNames;
@@ -376,10 +377,21 @@ mod tests {
376377
use crate::scalar::ScalarValue;
377378
use crate::validity::Validity;
378379

380+
/// The size the array occupies once rebuilt through the canonical builders, which is the
381+
/// layout [`UncompressedSizeInBytes`] is defined against: the builders normalize physical
382+
/// widths that the input array is free to choose differently, picking the smallest decimal
383+
/// value type for a precision and `u64` list-view offsets and sizes.
384+
///
385+
/// Builders no longer canonicalize their children, so the finished array is only canonical at
386+
/// the top level - recursively canonicalize it before measuring.
379387
fn materialized_uncompressed_size_in_bytes(array: &ArrayRef) -> u64 {
380388
let mut ctx = array_session().create_execution_ctx();
389+
let mut builder = builder_with_capacity(array.dtype(), array.len());
381390
array
382-
.clone()
391+
.append_to_builder(builder.as_mut(), &mut ctx)
392+
.vortex_expect("appended");
393+
builder
394+
.finish()
383395
.execute::<RecursiveCanonical>(&mut ctx)
384396
.vortex_expect("recursively canonicalized")
385397
.0

‎vortex-array/src/arrays/varbin/builder.rs‎

Lines changed: 13 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -456,17 +456,6 @@ impl<O: OffsetBuilderPType> VarBinBuilder<O> {
456456
}
457457
}
458458

459-
fn replace_validity(&mut self, validity: Mask) {
460-
self.validity = match validity {
461-
Mask::AllTrue(len) => BitBufferMut::new_set(len),
462-
Mask::AllFalse(len) => BitBufferMut::new_unset(len),
463-
values @ Mask::Values(_) => values
464-
.into_bit_buffer()
465-
.try_into_mut()
466-
.unwrap_or_else(|buffer| BitBufferMut::copy_from(&buffer)),
467-
};
468-
}
469-
470459
/// Appends `count` end offsets derived from `end_offsets`, shifted past the current data end.
471460
///
472461
/// `end_offsets` must be monotonically non-decreasing and end at exactly `num_bytes`. Offsets
@@ -934,21 +923,19 @@ mod tests {
934923
#[case(true)]
935924
fn test_array_builder_methods(#[case] large_offsets: bool) -> VortexResult<()> {
936925
let mut ctx = array_session().create_execution_ctx();
937-
for validity in [
938-
Mask::new_true(3),
939-
Mask::new_false(3),
940-
Mask::from_iter([true, false, true]),
941-
] {
942-
let result = with_offsets(large_offsets, DType::Utf8(Nullable), |builder| {
943-
builder.reserve_exact(3);
944-
builder.append_zero();
945-
builder.append_scalar(&Scalar::utf8("hello", Nullable))?;
946-
builder.append_null();
947-
assert_eq!(builder.len(), 3);
948-
Ok(())
949-
})?;
950-
assert_eq!(result.validity()?.execute_mask(3, &mut ctx)?, validity);
951-
}
926+
let result = with_offsets(large_offsets, DType::Utf8(Nullable), |builder| {
927+
builder.reserve_exact(3);
928+
builder.append_zero();
929+
builder.append_scalar(&Scalar::utf8("hello", Nullable))?;
930+
builder.append_null();
931+
assert_eq!(builder.len(), 3);
932+
Ok(())
933+
})?;
934+
935+
assert_eq!(
936+
result.validity()?.execute_mask(3, &mut ctx)?,
937+
Mask::from_iter([true, true, false])
938+
);
952939
Ok(())
953940
}
954941

‎vortex-array/src/builders/validity.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ use crate::validity::Validity;
2121
/// concatenates them at the end, exactly as [`Validity::concat`] does.
2222
///
2323
/// Materializing them instead would mean executing every appended array's validity into a
24-
/// [`Mask`] and copying its bits, which for a builder assembling many chunks is the dominant cost
25-
/// of tracking validity at all.
24+
/// [`Mask`](vortex_mask::Mask) and copying its bits, which for a builder assembling many chunks is
25+
/// the dominant cost of tracking validity at all.
2626
pub(crate) struct ValidityBuilder {
2727
/// Completed runs, in logical order, with the number of values each covers. Never contains an
2828
/// empty run.

0 commit comments

Comments
 (0)