Skip to content

Commit 0bc58a0

Browse files
committed
Address review: deletes past the file end, runs across 2^16, ranges() docs
1 parent 1820b84 commit 0bc58a0

2 files changed

Lines changed: 27 additions & 5 deletions

File tree

‎crates/iceberg/src/arrow/reader/positional_deletes.rs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -299,8 +299,16 @@ mod tests {
299299

300300
for deletes in mixed_delete_vectors() {
301301
let deleted: Vec<u64> = deletes.iter().collect();
302-
let num_rows = deleted.last().map_or(0, |&last| last + 1) + rand() % 100 + 1;
303302
for _ in 0..8 {
303+
// Half the time the file ends at or before a deleted position, so the deletes
304+
// past its end must be ignored.
305+
let num_rows = match deleted.len() {
306+
n if n > 0 && rand() % 2 == 0 => {
307+
deleted[(rand() % n as u64) as usize].saturating_add(rand() % 2)
308+
}
309+
_ => deleted.last().map_or(0, |&last| last + 1) + rand() % 100,
310+
}
311+
.max(1);
304312
// Row group boundaries, half of them next to a deleted position so that they
305313
// fall inside runs.
306314
let mut cuts: Vec<u64> = (0..rand() % 8)

‎crates/iceberg/src/delete_vector.rs‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,9 @@ impl DeleteVector {
9494

9595
/// Returns the deleted positions as ascending, non-overlapping half-open ranges.
9696
///
97-
/// Adjacent ranges may not be merged (a run can be split at a container or 2^32 boundary),
98-
/// so callers shouldn't assume a range is maximal.
97+
/// Adjacent ranges may not be merged (a run is split at each 2^32 boundary), so callers
98+
/// shouldn't assume a range is maximal. A range holding only `u64::MAX` is dropped, because a
99+
/// half-open `u64` range can't hold it and no Parquet row can be there.
99100
pub(crate) fn ranges(&self) -> DeleteVectorRanges<'_> {
100101
DeleteVectorRanges {
101102
bitmaps: self.inner.bitmaps(),
@@ -337,8 +338,7 @@ pub(crate) struct DeleteVectorRanges<'a> {
337338
}
338339

339340
impl DeleteVectorRanges<'_> {
340-
/// Returns `None` only for a range holding nothing but `u64::MAX`, which a half-open `u64`
341-
/// range can't hold. That position is past any row a Parquet file can have.
341+
/// Returns `None` only for a range holding nothing but `u64::MAX`, see [`DeleteVector::ranges`].
342342
fn full_range(&self, start: u32, end_inclusive: u32) -> Option<(u64, u64)> {
343343
let base = u64::from(self.high_bits) << 32;
344344
let start = base | u64::from(start);
@@ -657,6 +657,20 @@ mod tests {
657657
assert_eq!(ranges.next(), None);
658658
}
659659

660+
#[test]
661+
fn test_ranges_join_runs_across_2_16() {
662+
// `roaring`'s `next_range` joins a run across containers, so these come back whole.
663+
// Starting near the boundary also covers the positions walked before `next_range`.
664+
for start in [(1 << 16) - 100, (1 << 16) - 2] {
665+
let mut dv = DeleteVector::default();
666+
dv.inner.insert_range(start..(3 << 16) + 5);
667+
assert_eq!(dv.ranges().collect::<Vec<_>>(), vec![(
668+
start,
669+
(3 << 16) + 5
670+
)]);
671+
}
672+
}
673+
660674
#[test]
661675
fn test_ranges_end_at_u64_max() {
662676
let dv = DeleteVector::new(RoaringTreemap::from_iter([5, u64::MAX - 2, u64::MAX - 1]));

0 commit comments

Comments
 (0)