Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
b803a42
feat: enhance runtime casting for FixedSizeList and improve planner/r…
kosiew Jun 16, 2026
0e82dcb
feat: add downcast_array macro for improved type safety in casting
kosiew Jun 16, 2026
62c5adf
fix: close FixedSizeList size-mismatch loophole in nested_struct.rs
kosiew Jun 16, 2026
b561ef9
refactor: rename create_fsl_test_fields to create_fixed_size_list_tes…
kosiew Jun 16, 2026
3286a0c
feat(tests): add regression and parity tests for FixedSizeList behavior
kosiew Jun 16, 2026
38bdcf7
chore: refactor code for improved clarity and reduced redundancy
kosiew Jun 16, 2026
d8c1b17
feat: refactor nested_struct to use Arrow methods and remove downcast…
kosiew Jun 18, 2026
70237b3
feat: enhance data type validation with improved fallback structure
kosiew Jun 18, 2026
040691b
feat(nested_struct): add targeted fallback comments for FixedSizeList…
kosiew Jun 18, 2026
ced5b41
feat(nested_struct): refine compatibility checks and restructure cast…
kosiew Jun 29, 2026
dd00ef9
Merge branch 'main' into schema-evolution-02-20835
alamb Jul 22, 2026
d943826
Merge branch 'main' into schema-evolution-02-20835
kosiew Jul 24, 2026
96ab349
feat(datafusion): simplify nested struct handling by removing compati…
kosiew Jul 24, 2026
a59ede7
feat: add FixedSizeList Parquet E2E coverage
kosiew Jul 24, 2026
6600725
feat: replace panicking FixedSizeListArray::new with fallible try_new
kosiew Jul 24, 2026
8b0c478
docs: add mention of equal-width FixedSizeList
kosiew Jul 24, 2026
39ce782
Merge branch 'schema-evolution-02-20835' of github.com:kosiew/datafus…
kosiew Jul 24, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat: add FixedSizeList Parquet E2E coverage
- Implemented old/new schema evolution via ListingTable
- Added support for SELECT *
- Enabled nested-field projection
- Implemented rejection for missing non-nullable fields
- Added rejection for incompatible fields
- Corrected existing negative fixture to truly omit chain
  • Loading branch information
kosiew committed Jul 24, 2026
commit a59ede741a46dc6b8c54d9d07a05bd70beeeace4
124 changes: 101 additions & 23 deletions datafusion/core/tests/parquet/expr_adapter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
use std::sync::Arc;

use arrow::array::{
Array, ArrayRef, BooleanArray, Int32Array, Int64Array, LargeListArray, ListArray,
RecordBatch, StringArray, StructArray, record_batch,
Array, ArrayRef, BooleanArray, FixedSizeListArray, Int32Array, Int64Array,
LargeListArray, ListArray, RecordBatch, StringArray, StructArray, record_batch,
};
use arrow::buffer::OffsetBuffer;
use arrow::compute::concat_batches;
Expand Down Expand Up @@ -60,13 +60,19 @@ async fn write_parquet(batch: RecordBatch, store: Arc<dyn ObjectStore>, path: &s
enum NestedListKind {
List,
LargeList,
FixedSizeList,
}

const FIXED_SIZE_LIST_LEN: usize = 2;

impl NestedListKind {
fn field_data_type(self, item_field: Arc<Field>) -> DataType {
match self {
Self::List => DataType::List(item_field),
Self::LargeList => DataType::LargeList(item_field),
Self::FixedSizeList => {
DataType::FixedSizeList(item_field, FIXED_SIZE_LIST_LEN as i32)
}
}
}

Expand All @@ -89,13 +95,27 @@ impl NestedListKind {
values,
None,
)),
Self::FixedSizeList => {
assert_eq!(
lengths.as_slice(),
&[FIXED_SIZE_LIST_LEN],
"FixedSizeList fixtures must contain exactly {FIXED_SIZE_LIST_LEN} elements per row"
);
Arc::new(FixedSizeListArray::new(
item_field,
FIXED_SIZE_LIST_LEN as i32,
values,
None,
))
}
}
}

fn name(self) -> &'static str {
match self {
Self::List => "list",
Self::LargeList => "large_list",
Self::FixedSizeList => "fixed_size_list",
}
}
}
Expand Down Expand Up @@ -277,7 +297,8 @@ fn nested_list_table_schema(
}

// Helper to extract message values from a nested list column.
// Returns the values at indices 0 and 1 from either a ListArray or LargeListArray.
// Returns the values at indices 0 and 1 from either a ListArray, LargeListArray,
// or FixedSizeListArray.
fn extract_nested_list_values(
kind: NestedListKind,
column: &ArrayRef,
Expand All @@ -297,7 +318,50 @@ fn extract_nested_list_values(
.expect("messages should be a LargeListArray");
(list.value(0), list.value(1))
}
NestedListKind::FixedSizeList => {
let list = column
.as_any()
.downcast_ref::<FixedSizeListArray>()
.expect("messages should be a FixedSizeListArray");
(list.value(0), list.value(1))
}
}
}

fn evolved_messages(kind: NestedListKind) -> Vec<NestedMessageRow<'static>> {
let mut messages = vec![NestedMessageRow {
id: 30,
name: "gamma",
chain: Some("eth"),
ignored: Some(99),
}];
if matches!(kind, NestedListKind::FixedSizeList) {
messages.push(NestedMessageRow {
id: 40,
name: "delta",
chain: Some("doge"),
ignored: Some(100),
});
}
messages
}

fn error_messages(kind: NestedListKind) -> Vec<NestedMessageRow<'static>> {
let mut messages = vec![NestedMessageRow {
id: 10,
name: "alpha",
chain: Some("eth"),
ignored: None,
}];
if matches!(kind, NestedListKind::FixedSizeList) {
messages.push(NestedMessageRow {
id: 20,
name: "beta",
chain: Some("doge"),
ignored: None,
});
}
messages
}

// Helper to set up a nested list test fixture.
Expand Down Expand Up @@ -352,15 +416,11 @@ async fn assert_nested_list_struct_schema_evolution(kind: NestedListKind) -> Res
);

// new.parquet shape: messages item struct adds nullable `chain` and extra `ignored`.
let new_messages = evolved_messages(kind);
let new_batch = nested_messages_batch(
kind,
2,
&[NestedMessageRow {
id: 30,
name: "gamma",
chain: Some("eth"),
ignored: Some(99),
}],
&new_messages,
&message_fields(DataType::Utf8, true, true, true),
);

Expand Down Expand Up @@ -429,7 +489,12 @@ async fn assert_nested_list_struct_schema_evolution(kind: NestedListKind) -> Res
.as_any()
.downcast_ref::<StringArray>()
.unwrap();
assert_eq!(new_chain.iter().collect::<Vec<_>>(), vec![Some("eth")]);
let expected_new_chain = if matches!(kind, NestedListKind::FixedSizeList) {
vec![Some("eth"), Some("doge")]
} else {
vec![Some("eth")]
};
assert_eq!(new_chain.iter().collect::<Vec<_>>(), expected_new_chain);

let projected = ctx
.sql(
Expand Down Expand Up @@ -863,12 +928,12 @@ async fn test_struct_schema_evolution_projection_and_filter() -> Result<()> {
Ok(())
}

/// Macro to generate paired test functions for List and LargeList variants.
/// Expands to two `#[tokio::test]` functions with the specified names.
macro_rules! test_struct_schema_evolution_pair {
/// Macro to generate schema evolution tests for list-like variants.
macro_rules! test_struct_schema_evolution_variants {
(
list: $list_test:ident,
large_list: $large_list_test:ident,
fixed_size_list: $fixed_size_list_test:ident,
fn: $assertion_fn:path $(, args: $($arg:expr),+)?
) => {
#[tokio::test]
Expand All @@ -880,10 +945,16 @@ macro_rules! test_struct_schema_evolution_pair {
async fn $large_list_test() {
$assertion_fn(NestedListKind::LargeList $(, $($arg),+)?).await;
}

#[tokio::test]
async fn $fixed_size_list_test() {
$assertion_fn(NestedListKind::FixedSizeList $(, $($arg),+)?).await;
}
};
(
list: $list_test:ident,
large_list: $large_list_test:ident,
fixed_size_list: $fixed_size_list_test:ident,
fn_result: $assertion_fn:path
) => {
#[tokio::test]
Expand All @@ -895,31 +966,34 @@ macro_rules! test_struct_schema_evolution_pair {
async fn $large_list_test() -> Result<()> {
$assertion_fn(NestedListKind::LargeList).await
}

#[tokio::test]
async fn $fixed_size_list_test() -> Result<()> {
$assertion_fn(NestedListKind::FixedSizeList).await
}
};
}

test_struct_schema_evolution_pair!(
test_struct_schema_evolution_variants!(
list: test_list_struct_schema_evolution_end_to_end,
large_list: test_large_list_struct_schema_evolution_end_to_end,
fixed_size_list: test_fixed_size_list_struct_schema_evolution_end_to_end,
fn_result: assert_nested_list_struct_schema_evolution
);

async fn assert_nested_list_struct_schema_evolution_errors(
kind: NestedListKind,
source_includes_chain: bool,
chain_type: DataType,
chain_nullable: bool,
expected_error: &str,
) {
let messages = error_messages(kind);
let batch = nested_messages_batch(
kind,
1,
&[NestedMessageRow {
id: 10,
name: "alpha",
chain: Some("eth"),
ignored: None,
}],
&message_fields(DataType::Utf8, true, true, false),
&messages,
&message_fields(DataType::Utf8, true, source_includes_chain, false),
);

let table_schema =
Expand Down Expand Up @@ -949,6 +1023,7 @@ async fn assert_nested_list_struct_schema_evolution_errors(
async fn assert_non_nullable_missing_chain_field_fails(kind: NestedListKind) {
assert_nested_list_struct_schema_evolution_errors(
kind,
false,
DataType::Utf8,
false,
"non-nullable",
Expand All @@ -959,6 +1034,7 @@ async fn assert_non_nullable_missing_chain_field_fails(kind: NestedListKind) {
async fn assert_incompatible_chain_field_fails(kind: NestedListKind) {
assert_nested_list_struct_schema_evolution_errors(
kind,
true,
incompatible_chain_type(),
true,
"Cannot cast struct field 'chain'",
Expand All @@ -970,15 +1046,17 @@ fn incompatible_chain_type() -> DataType {
DataType::Struct(vec![Arc::new(Field::new("value", DataType::Utf8, true))].into())
}

test_struct_schema_evolution_pair!(
test_struct_schema_evolution_variants!(
list: test_list_struct_schema_evolution_non_nullable_missing_field_fails,
large_list: test_large_list_struct_schema_evolution_non_nullable_missing_field_fails,
fixed_size_list: test_fixed_size_list_struct_schema_evolution_non_nullable_missing_field_fails,
fn: assert_non_nullable_missing_chain_field_fails
);

test_struct_schema_evolution_pair!(
test_struct_schema_evolution_variants!(
list: test_list_struct_schema_evolution_incompatible_field_fails,
large_list: test_large_list_struct_schema_evolution_incompatible_field_fails,
fixed_size_list: test_fixed_size_list_struct_schema_evolution_incompatible_field_fails,
fn: assert_incompatible_chain_field_fails
);

Expand Down