Skip to content

Commit f30f3ae

Browse files
committed
fix(layout): rebase sliced list chunks before writing the list layout
The list writer stores each chunk's whole elements and shifts its offsets by the elements written before it, which assumes the offsets span exactly [0, elements.len()]. A sliced ListArray keeps its parent's elements, so its lists read back as the wrong elements. Reset each ListArray chunk's offsets first. ListView chunks, and so the default file writer, which canonicalizes before the list writer, were already trimmed. Signed-off-by: gitRasheed <[email protected]>
1 parent a6c3b7e commit f30f3ae

3 files changed

Lines changed: 241 additions & 1 deletion

File tree

‎vortex-file/src/tests.rs‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use rstest::rstest;
1919
use tempfile::tempdir;
2020
use vortex_array::ArrayRef;
2121
use vortex_array::EmptyMetadata;
22+
use vortex_array::ExecutionCtx;
2223
use vortex_array::IntoArray;
2324
use vortex_array::VortexSessionExecute;
2425
use vortex_array::aggregate_fn::AggregateFnRef;
@@ -3575,3 +3576,106 @@ async fn supplied_lazy_dtype_is_charged_without_parsing_it() -> VortexResult<()>
35753576
);
35763577
Ok(())
35773578
}
3579+
#[derive(Clone, Copy, Debug)]
3580+
enum ListWriteStrategy {
3581+
Default,
3582+
SmallRowBlocks,
3583+
ListLayoutOnly,
3584+
}
3585+
3586+
/// Lists `[0, 1, 2]`, `[]`, `null` (holding `[3, 4, 5]`), `[6, 7]`, `[8, 9]`.
3587+
fn sliceable_list(ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
3588+
Ok(ListArray::try_new(
3589+
PrimitiveArray::from_iter(0..10i32).into_array(),
3590+
buffer![0u32, 3, 3, 6, 8, 10].into_array(),
3591+
Validity::Array(BoolArray::from_iter([true, true, false, true, true]).into_array()),
3592+
ctx,
3593+
)?
3594+
.into_array())
3595+
}
3596+
3597+
fn sliceable_nested_list(ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
3598+
let inner = ListArray::try_new(
3599+
PrimitiveArray::from_iter(0..12i32).into_array(),
3600+
buffer![0u32, 2, 2, 5, 7, 9, 12, 12, 12, 12].into_array(),
3601+
Validity::NonNullable,
3602+
ctx,
3603+
)?
3604+
.into_array();
3605+
Ok(ListArray::try_new(
3606+
inner,
3607+
buffer![0u32, 2, 3, 3, 6, 9].into_array(),
3608+
Validity::Array(BoolArray::from_iter([true, false, true, true, true]).into_array()),
3609+
ctx,
3610+
)?
3611+
.into_array())
3612+
}
3613+
3614+
fn sliceable_list_of_struct(ctx: &mut ExecutionCtx) -> VortexResult<ArrayRef> {
3615+
let elements = StructArray::from_fields(&[
3616+
("a", PrimitiveArray::from_iter(0..10i32).into_array()),
3617+
("b", PrimitiveArray::from_iter(10..20i64).into_array()),
3618+
])?
3619+
.into_array();
3620+
Ok(ListArray::try_new(
3621+
elements,
3622+
buffer![0u32, 3, 3, 6, 8, 10].into_array(),
3623+
Validity::NonNullable,
3624+
ctx,
3625+
)?
3626+
.into_array())
3627+
}
3628+
3629+
/// Writes a table whose list column arrives as slices of one list, which share its whole
3630+
/// `elements`. With small row blocks the writer's own repartitioning slices the lists too.
3631+
#[rstest]
3632+
#[tokio::test]
3633+
#[cfg_attr(miri, ignore)]
3634+
async fn sliced_list_column_roundtrip(
3635+
#[values(sliceable_list, sliceable_nested_list, sliceable_list_of_struct)]
3636+
make_list: fn(&mut ExecutionCtx) -> VortexResult<ArrayRef>,
3637+
#[values(
3638+
ListWriteStrategy::Default,
3639+
ListWriteStrategy::SmallRowBlocks,
3640+
ListWriteStrategy::ListLayoutOnly
3641+
)]
3642+
strategy: ListWriteStrategy,
3643+
) -> VortexResult<()> {
3644+
let mut ctx = SESSION.create_execution_ctx();
3645+
let list = make_list(&mut ctx)?;
3646+
let chunks = [0..1, 1..3, 3..list.len()]
3647+
.into_iter()
3648+
.map(|range| Ok(StructArray::from_fields(&[("lists", list.slice(range)?)])?.into_array()))
3649+
.collect::<VortexResult<Vec<_>>>()?;
3650+
let expected = StructArray::from_fields(&[("lists", list)])?.into_array();
3651+
let chunked = ChunkedArray::try_new(chunks, expected.dtype().clone())?.into_array();
3652+
3653+
let options = SESSION.write_options();
3654+
let options = match strategy {
3655+
ListWriteStrategy::Default => options,
3656+
ListWriteStrategy::SmallRowBlocks => options.with_strategy(
3657+
crate::strategy::WriteStrategyBuilder::from_session(&SESSION)
3658+
.with_row_block_size(2)
3659+
.with_data_block_target_bytes(None)
3660+
.build(),
3661+
),
3662+
ListWriteStrategy::ListLayoutOnly => {
3663+
let chunked: Arc<dyn LayoutStrategy> =
3664+
Arc::new(ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()));
3665+
options.with_strategy(Arc::new(
3666+
TableStrategy::new(Arc::clone(&chunked), chunked).with_list_layout(),
3667+
))
3668+
}
3669+
};
3670+
let mut buf = ByteBufferMut::empty();
3671+
options.write(&mut buf, chunked.to_array_stream()).await?;
3672+
let result = SESSION
3673+
.open_options()
3674+
.open_buffer(buf)?
3675+
.scan()?
3676+
.into_array_stream()?
3677+
.read_all()
3678+
.await?;
3679+
assert_arrays_eq!(result, expected, &mut ctx);
3680+
Ok(())
3681+
}

‎vortex-layout/src/layouts/list/reader.rs‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -813,7 +813,9 @@ mod tests {
813813
use vortex_array::RecursiveCanonical;
814814
use vortex_array::VortexSessionExecute;
815815
use vortex_array::arrays::BoolArray;
816+
use vortex_array::arrays::ChunkedArray;
816817
use vortex_array::arrays::ListArray;
818+
use vortex_array::arrays::ListViewArray;
817819
use vortex_array::arrays::PrimitiveArray;
818820
use vortex_array::arrays::StructArray;
819821
use vortex_array::assert_arrays_eq;
@@ -1979,4 +1981,135 @@ mod tests {
19791981
.into_array();
19801982
Ok(StructArray::from_fields(&[("l", l)])?.into_array())
19811983
}
1984+
1985+
/// Lists `[0, 1, 2]`, `[]`, `[3, 4, 5]`, `[6, 7]`, `[8, 9]`. When nullable, the third is null
1986+
/// and keeps its elements.
1987+
fn create_ten_element_list_array(nullable: bool) -> ArrayRef {
1988+
let mut ctx = SESSION.create_execution_ctx();
1989+
let validity = if nullable {
1990+
Validity::Array(BoolArray::from_iter([true, true, false, true, true]).into_array())
1991+
} else {
1992+
Validity::NonNullable
1993+
};
1994+
ListArray::try_new(
1995+
PrimitiveArray::from_iter(0..10i32).into_array(),
1996+
buffer![0u32, 3, 3, 6, 8, 10].into_array(),
1997+
validity,
1998+
&mut ctx,
1999+
)
2000+
.expect("array is valid")
2001+
.into_array()
2002+
}
2003+
2004+
/// Slices of one list share its whole `elements`, so every chunk but the first starts at a
2005+
/// nonzero offset and every chunk but the last leaves trailing elements unreferenced.
2006+
fn sliced_chunks(array: &ArrayRef, cuts: &[usize]) -> VortexResult<Vec<ArrayRef>> {
2007+
let mut bounds = vec![0];
2008+
bounds.extend_from_slice(cuts);
2009+
bounds.push(array.len());
2010+
bounds
2011+
.windows(2)
2012+
.map(|range| array.slice(range[0]..range[1]))
2013+
.collect()
2014+
}
2015+
2016+
fn chunked_flat() -> Arc<dyn LayoutStrategy> {
2017+
Arc::new(ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()))
2018+
}
2019+
2020+
/// A list strategy that keeps every chunk, so a multi-chunk stream reaches the list writer.
2021+
fn chunked_list_strategy() -> ListLayoutStrategy {
2022+
ListLayoutStrategy::default()
2023+
.with_elements(chunked_flat())
2024+
.with_offsets(chunked_flat())
2025+
.with_validity(chunked_flat())
2026+
}
2027+
2028+
async fn assert_round_trips<S: LayoutStrategy>(
2029+
strategy: &S,
2030+
chunks: Vec<ArrayRef>,
2031+
expected: ArrayRef,
2032+
) -> VortexResult<()> {
2033+
let chunked = ChunkedArray::try_new(chunks, expected.dtype().clone())?.into_array();
2034+
let (segments, layout, session) = write_layout(strategy, chunked).await?;
2035+
let reader =
2036+
layout.new_reader("".into(), segments, &session, &LayoutReaderContext::new())?;
2037+
2038+
let len = expected.len();
2039+
let expr = root().bind(reader.dtype())?;
2040+
let result = reader
2041+
.projection_evaluation(&(0..len as u64), &expr, MaskFuture::new_true(len))?
2042+
.await?;
2043+
let mut exec_ctx = session.create_execution_ctx();
2044+
assert_arrays_eq!(result, expected, &mut exec_ctx);
2045+
Ok(())
2046+
}
2047+
2048+
#[rstest]
2049+
#[case::one_cut(&[2], false, false)]
2050+
#[case::two_cuts(&[1, 3], false, false)]
2051+
#[case::null_and_empty_at_cuts(&[1, 3], true, false)]
2052+
#[case::list_view_chunks(&[1, 3], true, true)]
2053+
#[tokio::test]
2054+
async fn sliced_list_chunks_round_trip(
2055+
#[case] cuts: &[usize],
2056+
#[case] nullable: bool,
2057+
#[case] as_list_view: bool,
2058+
) -> VortexResult<()> {
2059+
let list = create_ten_element_list_array(nullable);
2060+
let source = if as_list_view {
2061+
let mut ctx = SESSION.create_execution_ctx();
2062+
list.clone().execute::<ListViewArray>(&mut ctx)?.into_array()
2063+
} else {
2064+
list.clone()
2065+
};
2066+
let chunks = sliced_chunks(&source, cuts)?;
2067+
assert_round_trips(&chunked_list_strategy(), chunks, list).await
2068+
}
2069+
2070+
#[tokio::test]
2071+
async fn sliced_nested_list_chunks_round_trip() -> VortexResult<()> {
2072+
let mut ctx = SESSION.create_execution_ctx();
2073+
let inner = ListArray::try_new(
2074+
PrimitiveArray::from_iter(0..12i32).into_array(),
2075+
buffer![0u32, 2, 2, 5, 7, 9, 12].into_array(),
2076+
Validity::NonNullable,
2077+
&mut ctx,
2078+
)?
2079+
.into_array();
2080+
let outer = ListArray::try_new(
2081+
inner,
2082+
buffer![0u32, 2, 3, 5, 6].into_array(),
2083+
Validity::Array(BoolArray::from_iter([true, false, true, true]).into_array()),
2084+
&mut ctx,
2085+
)?
2086+
.into_array();
2087+
2088+
let strategy = chunked_list_strategy().with_elements(Arc::new(chunked_list_strategy()));
2089+
assert_round_trips(&strategy, sliced_chunks(&outer, &[1, 2])?, outer).await
2090+
}
2091+
2092+
#[tokio::test]
2093+
async fn sliced_list_of_struct_chunks_round_trip() -> VortexResult<()> {
2094+
let mut ctx = SESSION.create_execution_ctx();
2095+
let elements = StructArray::from_fields(
2096+
[
2097+
("a", PrimitiveArray::from_iter(0..10i32).into_array()),
2098+
("b", PrimitiveArray::from_iter(10..20i64).into_array()),
2099+
]
2100+
.as_slice(),
2101+
)?
2102+
.into_array();
2103+
let list = ListArray::try_new(
2104+
elements,
2105+
buffer![0u32, 3, 3, 6, 8, 10].into_array(),
2106+
Validity::NonNullable,
2107+
&mut ctx,
2108+
)?
2109+
.into_array();
2110+
2111+
let strategy = chunked_list_strategy()
2112+
.with_elements(Arc::new(TableStrategy::new(chunked_flat(), chunked_flat())));
2113+
assert_round_trips(&strategy, sliced_chunks(&list, &[1, 3])?, list).await
2114+
}
19822115
}

‎vortex-layout/src/layouts/list/writer.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ use vortex_array::arrays::ConstantArray;
1818
use vortex_array::arrays::List;
1919
use vortex_array::arrays::ListView;
2020
use vortex_array::arrays::PrimitiveArray;
21+
use vortex_array::arrays::list::ListArrayExt;
2122
use vortex_array::arrays::list::ListDataParts;
2223
use vortex_array::arrays::listview::list_from_list_view;
2324
use vortex_array::builtins::ArrayBuiltins;
@@ -289,7 +290,9 @@ fn canonicalize_to_list_parts(
289290
) -> VortexResult<ListDataParts> {
290291
let canonical = array.execute_until::<AnyList>(exec_ctx)?;
291292
if let Some(list) = canonical.as_opt::<List>() {
292-
Ok(list.into_owned().into_data_parts())
293+
// A sliced list keeps its parent's whole `elements`, but `global_offsets` assumes each
294+
// chunk's offsets span exactly `[0, elements.len()]`.
295+
Ok(list.reset_offsets(false, exec_ctx)?.into_data_parts())
293296
} else if let Some(view) = canonical.as_opt::<ListView>() {
294297
Ok(list_from_list_view(view.into_owned(), exec_ctx)?.into_data_parts())
295298
} else {

0 commit comments

Comments
 (0)