Skip to content

Commit d842053

Browse files
committed
fix(layout): rebase sliced list chunks before writing the list layout
The list writer stores each chunk's whole elements and shifts its offsets by the elements written before it, which assumes the offsets span exactly [0, elements.len()]. A sliced ListArray keeps its parent's elements, so its lists read back as the wrong elements. Reset each ListArray chunk's offsets first. ListView chunks, and so the default file writer, which canonicalizes before the list writer, were already trimmed. Signed-off-by: gitRasheed <[email protected]>
1 parent 65c992d commit d842053

2 files changed

Lines changed: 104 additions & 1 deletion

File tree

‎vortex-layout/src/layouts/list/reader.rs‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -621,6 +621,7 @@ mod tests {
621621
use vortex_array::ArrayContext;
622622
use vortex_array::VortexSessionExecute;
623623
use vortex_array::arrays::BoolArray;
624+
use vortex_array::arrays::ChunkedArray;
624625
use vortex_array::arrays::ListArray;
625626
use vortex_array::arrays::PrimitiveArray;
626627
use vortex_array::assert_arrays_eq;
@@ -1217,4 +1218,103 @@ mod tests {
12171218
assert_arrays_eq!(result, expected, &mut exec_ctx);
12181219
Ok(())
12191220
}
1221+
1222+
/// Lists `[0, 1, 2]`, `[]`, `[3, 4, 5]`, `[6, 7]`, `[8, 9]`. When nullable, the third is null
1223+
/// and keeps its elements.
1224+
fn create_ten_element_list_array(nullable: bool) -> ArrayRef {
1225+
let mut ctx = SESSION.create_execution_ctx();
1226+
let validity = if nullable {
1227+
Validity::Array(BoolArray::from_iter([true, true, false, true, true]).into_array())
1228+
} else {
1229+
Validity::NonNullable
1230+
};
1231+
ListArray::try_new(
1232+
PrimitiveArray::from_iter(0..10i32).into_array(),
1233+
buffer![0u32, 3, 3, 6, 8, 10].into_array(),
1234+
validity,
1235+
&mut ctx,
1236+
)
1237+
.expect("array is valid")
1238+
.into_array()
1239+
}
1240+
1241+
/// Slices of one list share its whole `elements`, so every chunk but the first starts at a
1242+
/// nonzero offset and every chunk but the last leaves trailing elements unreferenced.
1243+
fn sliced_chunks(array: &ArrayRef, cuts: &[usize]) -> VortexResult<Vec<ArrayRef>> {
1244+
let mut bounds = vec![0];
1245+
bounds.extend_from_slice(cuts);
1246+
bounds.push(array.len());
1247+
bounds
1248+
.windows(2)
1249+
.map(|range| array.slice(range[0]..range[1]))
1250+
.collect()
1251+
}
1252+
1253+
fn chunked_flat() -> Arc<dyn LayoutStrategy> {
1254+
Arc::new(ChunkedLayoutStrategy::new(FlatLayoutStrategy::default()))
1255+
}
1256+
1257+
/// A list strategy that keeps every chunk, so a multi-chunk stream reaches the list writer.
1258+
fn chunked_list_strategy() -> ListLayoutStrategy {
1259+
ListLayoutStrategy::default()
1260+
.with_elements(chunked_flat())
1261+
.with_offsets(chunked_flat())
1262+
.with_validity(chunked_flat())
1263+
}
1264+
1265+
async fn assert_round_trips<S: LayoutStrategy>(
1266+
strategy: &S,
1267+
chunks: Vec<ArrayRef>,
1268+
expected: ArrayRef,
1269+
) -> VortexResult<()> {
1270+
let chunked = ChunkedArray::try_new(chunks, expected.dtype().clone())?.into_array();
1271+
let (segments, layout, session) = write_layout(strategy, chunked).await?;
1272+
let reader =
1273+
layout.new_reader("".into(), segments, &session, &LayoutReaderContext::new())?;
1274+
1275+
let len = expected.len();
1276+
let expr = root().bind(reader.dtype())?;
1277+
let result = reader
1278+
.projection_evaluation(&(0..len as u64), &expr, MaskFuture::new_true(len))?
1279+
.await?;
1280+
let mut exec_ctx = session.create_execution_ctx();
1281+
assert_arrays_eq!(result, expected, &mut exec_ctx);
1282+
Ok(())
1283+
}
1284+
1285+
#[rstest]
1286+
#[case::one_cut(&[2], false)]
1287+
#[case::two_cuts(&[1, 3], false)]
1288+
#[case::null_and_empty_at_cuts(&[1, 3], true)]
1289+
#[tokio::test]
1290+
async fn sliced_list_chunks_round_trip(
1291+
#[case] cuts: &[usize],
1292+
#[case] nullable: bool,
1293+
) -> VortexResult<()> {
1294+
let list = create_ten_element_list_array(nullable);
1295+
let chunks = sliced_chunks(&list, cuts)?;
1296+
assert_round_trips(&chunked_list_strategy(), chunks, list).await
1297+
}
1298+
1299+
#[tokio::test]
1300+
async fn sliced_nested_list_chunks_round_trip() -> VortexResult<()> {
1301+
let mut ctx = SESSION.create_execution_ctx();
1302+
let inner = ListArray::try_new(
1303+
PrimitiveArray::from_iter(0..12i32).into_array(),
1304+
buffer![0u32, 2, 2, 5, 7, 9, 12].into_array(),
1305+
Validity::NonNullable,
1306+
&mut ctx,
1307+
)?
1308+
.into_array();
1309+
let outer = ListArray::try_new(
1310+
inner,
1311+
buffer![0u32, 2, 3, 5, 6].into_array(),
1312+
Validity::Array(BoolArray::from_iter([true, false, true, true]).into_array()),
1313+
&mut ctx,
1314+
)?
1315+
.into_array();
1316+
1317+
let strategy = chunked_list_strategy().with_elements(Arc::new(chunked_list_strategy()));
1318+
assert_round_trips(&strategy, sliced_chunks(&outer, &[1, 2])?, outer).await
1319+
}
12201320
}

‎vortex-layout/src/layouts/list/writer.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ use vortex_array::arrays::ConstantArray;
1818
use vortex_array::arrays::List;
1919
use vortex_array::arrays::ListView;
2020
use vortex_array::arrays::PrimitiveArray;
21+
use vortex_array::arrays::list::ListArrayExt;
2122
use vortex_array::arrays::list::ListDataParts;
2223
use vortex_array::arrays::listview::list_from_list_view;
2324
use vortex_array::builtins::ArrayBuiltins;
@@ -289,7 +290,9 @@ fn canonicalize_to_list_parts(
289290
) -> VortexResult<ListDataParts> {
290291
let canonical = array.execute_until::<AnyList>(exec_ctx)?;
291292
if let Some(list) = canonical.as_opt::<List>() {
292-
Ok(list.into_owned().into_data_parts())
293+
// A sliced list keeps its parent's whole `elements`, but `global_offsets` assumes each
294+
// chunk's offsets span exactly `[0, elements.len()]`.
295+
Ok(list.reset_offsets(false, exec_ctx)?.into_data_parts())
293296
} else if let Some(view) = canonical.as_opt::<ListView>() {
294297
Ok(list_from_list_view(view.into_owned(), exec_ctx)?.into_data_parts())
295298
} else {

0 commit comments

Comments
 (0)