Skip to content

Commit 20dd173

Browse files
authored
Fix: FixedSizeList DuckDB exporter bug (#4610)
Tracking Issue: #4372 Changes checked out from: #4605 Fixes a bug in the DuckDB exporter where if we do not flatten / materialize the inner child of the fixed-size list before exporting the vector to DuckDB, DuckDB will segfault after running a function like `scan`. Also fixes a bug where I didn't correctly append garbage values when the validity is all null (I took that from the list implementation, which doesn't need to append garbage values since null can be an empty list there). I don't actually know if this is an issue to be honest, but it seems wrong? I was not able to figure out how to write regression tests for both of these. The early return might not actually be an issue, but the flattening at the end definitely is. However, I was not able to create a minimally reproducible test for that, and it only fails in query 7 and 8 in `statpopgen.sql`. ```sql -- 7. Read just one variant (this is the sixth one). SELECT * FROM statpopgen WHERE "CHROM" == 'chr21' AND "POS" == 5030278; -- 8. Read a 700 base-pair window of variants. SELECT * FROM statpopgen WHERE "CHROM" == 'chr21' AND "POS" >= 5030300 AND "POS" <= 5031000; ``` If anyone has an idea of how to recreate this easily that would be appreciated! Finally, I simplified the tests I wrote from before since a lot were redundant. --------- Signed-off-by: Connor Tsui <[email protected]>
1 parent 4765a21 commit 20dd173

3 files changed

Lines changed: 178 additions & 14 deletions

File tree

‎bench-vortex/statpopgen.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ SELECT "CHROM", "POS", "REF", "ALT", CAST(LIST_SUM(GT) AS DOUBLE) / (2 * LIST_SU
1818
SELECT "CHROM", "POS", "REF", "ALT", LIST_SUM(LIST_TRANSFORM(GT, lambda GT: GT IS NOT NULL)) AS n_called FROM statpopgen;
1919
-- 6.
2020
--
21-
-- Collect the necessary statistics for a Hardy-Weinberg Equalibrium test. The
21+
-- Collect the necessary statistics for a Hardy-Weinberg Equilibrium test. The
2222
-- actual test involves the Levene-Haldane distribution which is somewhat subtle
2323
-- to implement in SQL.
2424
SELECT "CHROM", "POS", "REF", "ALT",

‎vortex-duckdb/src/e2e_test/vortex_scan_test.rs‎

Lines changed: 162 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@ use jiff::{Span, Timestamp, Zoned, tz};
1414
use num_traits::AsPrimitive;
1515
use tempfile::NamedTempFile;
1616
use vortex::IntoArray;
17-
use vortex::arrays::{BoolArray, ConstantArray, PrimitiveArray, StructArray, VarBinArray};
17+
use vortex::arrays::{
18+
BoolArray, ConstantArray, FixedSizeListArray, ListArray, PrimitiveArray, StructArray,
19+
VarBinArray, VarBinViewArray,
20+
};
1821
use vortex::buffer::buffer;
1922
use vortex::file::VortexWriteOptions;
2023
use vortex::scalar::Scalar;
@@ -386,3 +389,161 @@ fn test_write_timestamps() {
386389
Zoned::from_str("2025-05-03 16:19:14.338895-07[-07]").unwrap()
387390
);
388391
}
392+
393+
#[test]
394+
fn test_vortex_scan_nested_fixed_size_list_utf8() {
395+
// Regression test for a segfault that occurs inside query 7 and 8 of the `statpopgen` benchmark
396+
// when running with `FixedSizeList` instead of `List`.
397+
398+
// Test FixedSizeList of Utf8 to ensure proper materialization.
399+
400+
let runtime = tokio::runtime::Runtime::new().unwrap();
401+
let file = runtime.block_on(async {
402+
// Create a large number of strings to stress test.
403+
let strings: Vec<&str> = (0..24)
404+
.map(|i| match i % 6 {
405+
0 => "first",
406+
1 => "second",
407+
2 => "third",
408+
3 => "fourth",
409+
4 => "fifth",
410+
_ => "sixth",
411+
})
412+
.collect();
413+
414+
let strings_array = VarBinViewArray::from_iter_str(strings);
415+
416+
// Create inner fixed-size lists.
417+
let inner_fsl = FixedSizeListArray::new(
418+
strings_array.into_array(),
419+
4, // 4 strings per inner list
420+
Validity::AllValid,
421+
6, // 6 inner lists
422+
);
423+
424+
// Create outer fixed-size list of lists.
425+
let outer_fsl = FixedSizeListArray::new(
426+
inner_fsl.into_array(),
427+
3, // 3 inner lists per outer list
428+
Validity::AllValid,
429+
2, // 2 outer lists
430+
);
431+
432+
write_single_column_vortex_file("nested_string_lists", outer_fsl).await
433+
});
434+
435+
let conn = database_connection();
436+
let file_path = file.path().to_string_lossy();
437+
438+
// Query the nested structure.
439+
let result = conn
440+
.query(&format!(
441+
"SELECT nested_string_lists FROM vortex_scan('{file_path}')"
442+
))
443+
.unwrap();
444+
445+
let mut row_count = 0;
446+
for chunk in result {
447+
row_count += chunk.len();
448+
// Accessing the nested structure should not cause a segfault.
449+
let _vec = chunk.get_vector(0);
450+
}
451+
assert_eq!(row_count, 2, "Should have retrieved 2 outer lists");
452+
}
453+
454+
#[test]
455+
fn test_vortex_scan_ultra_deep_nesting() {
456+
// Test ultra-deep nesting: Multiple levels of FSL and List combinations with UTF8.
457+
// FSL[List[FSL[List[FSL[UTF8]]]]]
458+
459+
let runtime = tokio::runtime::Runtime::new().unwrap();
460+
let file = runtime.block_on(async {
461+
// Level 1: Create base UTF8 strings - need a lot for deep nesting.
462+
let strings = VarBinViewArray::from_iter_str(
463+
(0..360)
464+
.map(|i| match i % 10 {
465+
0 => "zero",
466+
1 => "one",
467+
2 => "two",
468+
3 => "three",
469+
4 => "four",
470+
5 => "five",
471+
6 => "six",
472+
7 => "seven",
473+
8 => "eight",
474+
_ => "nine",
475+
})
476+
.collect::<Vec<_>>(),
477+
);
478+
479+
// Level 2: Inner-most FixedSizeList of strings.
480+
let level2_fsl = FixedSizeListArray::new(
481+
strings.into_array(),
482+
5, // 5 strings per list
483+
Validity::AllValid,
484+
72, // 72 lists at this level
485+
);
486+
487+
// Level 3: Variable-length lists of level 2 FSLs.
488+
let level3_offsets = buffer![0i32, 3, 6, 8, 12, 15, 18, 20, 24, 27, 30, 32, 36];
489+
let level3_list = ListArray::try_new(
490+
level2_fsl.into_array(),
491+
level3_offsets.into_array(),
492+
Validity::AllValid,
493+
)
494+
.unwrap();
495+
496+
// Level 4: FixedSizeList of level 3 lists.
497+
let level4_fsl = FixedSizeListArray::new(
498+
level3_list.into_array(),
499+
3, // 3 variable lists per FSL
500+
Validity::AllValid,
501+
4, // 4 FSLs at this level
502+
);
503+
504+
// Level 5: Variable-length lists of level 4 FSLs.
505+
let level5_offsets = buffer![0i32, 2, 4];
506+
let level5_list = ListArray::try_new(
507+
level4_fsl.into_array(),
508+
level5_offsets.into_array(),
509+
Validity::AllValid,
510+
)
511+
.unwrap();
512+
513+
// Level 6: Outermost FixedSizeList.
514+
let outermost_fsl = FixedSizeListArray::new(
515+
level5_list.into_array(),
516+
2, // 2 lists per outermost FSL
517+
Validity::AllValid,
518+
1, // 1 outermost FSL
519+
);
520+
521+
write_single_column_vortex_file("ultra_deep", outermost_fsl).await
522+
});
523+
524+
let conn = database_connection();
525+
let file_path = file.path().to_string_lossy();
526+
527+
// Query the ultra-deep nested structure.
528+
let result = conn
529+
.query(&format!("SELECT COUNT(*) FROM vortex_scan('{file_path}')"))
530+
.unwrap();
531+
let chunk = result.into_iter().next().unwrap();
532+
let vec = chunk.get_vector(0);
533+
let count = vec.as_slice_with_len::<i64>(chunk.len().as_())[0];
534+
assert_eq!(count, 1, "Should have 1 outermost list");
535+
536+
// Try to access the data - this is the critical test for segfaults.
537+
let result = conn
538+
.query(&format!(
539+
"SELECT ultra_deep FROM vortex_scan('{file_path}')"
540+
))
541+
.unwrap();
542+
543+
let mut row_count = 0;
544+
for chunk in result {
545+
row_count += chunk.len();
546+
let _vec = chunk.get_vector(0);
547+
}
548+
assert_eq!(row_count, 1, "Should have retrieved 1 row");
549+
}

‎vortex-duckdb/src/exporter/fixed_size_list.rs‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -54,25 +54,28 @@ impl ColumnExporter for FixedSizeListExporter {
5454
self.validity.len()
5555
);
5656

57-
// Set validity if necessary.
58-
// SAFETY: We've asserted that offset + len <= self.validity.len(), which ensures
59-
// we won't read past the validity mask bounds.
57+
let list_size = self.list_size as usize;
58+
59+
// If all values are null, then we don't need to worry about exporting values (similar to
60+
// the primitive exporter).
61+
// SAFETY: We've asserted that offset + len <= self.validity.len(), which ensures we won't
62+
// read past the validity mask bounds.
6063
if unsafe { vector.set_validity(&self.validity, offset, len) } {
61-
// All values are null, so no point copying the data.
6264
return Ok(());
6365
}
6466

65-
// Get the child vector for array elements.
67+
// Get the child vector for array elements and export the elements directly.
6668
let mut elements_vector = vector.array_vector_get_child();
69+
self.elements_exporter
70+
.export(offset * list_size, len * list_size, &mut elements_vector)?;
6771

68-
// Export elements directly.
69-
// For fixed-size lists: elements start at offset * list_size
70-
// and we export len * list_size elements.
71-
let element_offset = offset * self.list_size as usize;
72-
let element_count = len * self.list_size as usize;
72+
// TODO(connor): We must flatten the child vector to ensure any child dictionary views
73+
// (namely UTF-8 string views in dictionaries) are materialized.
74+
// See https://github.com/vortex-data/vortex/pull/4610#issuecomment-3286676825 for a
75+
// detailed explanation on why we need this for now.
76+
elements_vector.flatten((len * list_size) as u64);
7377

74-
self.elements_exporter
75-
.export(element_offset, element_count, &mut elements_vector)
78+
Ok(())
7679
}
7780
}
7881

0 commit comments

Comments
 (0)