Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
b803a42
feat: enhance runtime casting for FixedSizeList and improve planner/r…
kosiew Jun 16, 2026
0e82dcb
feat: add downcast_array macro for improved type safety in casting
kosiew Jun 16, 2026
62c5adf
fix: close FixedSizeList size-mismatch loophole in nested_struct.rs
kosiew Jun 16, 2026
b561ef9
refactor: rename create_fsl_test_fields to create_fixed_size_list_tes…
kosiew Jun 16, 2026
3286a0c
feat(tests): add regression and parity tests for FixedSizeList behavior
kosiew Jun 16, 2026
38bdcf7
chore: refactor code for improved clarity and reduced redundancy
kosiew Jun 16, 2026
d8c1b17
feat: refactor nested_struct to use Arrow methods and remove downcast…
kosiew Jun 18, 2026
70237b3
feat: enhance data type validation with improved fallback structure
kosiew Jun 18, 2026
040691b
feat(nested_struct): add targeted fallback comments for FixedSizeList…
kosiew Jun 18, 2026
ced5b41
feat(nested_struct): refine compatibility checks and restructure cast…
kosiew Jun 29, 2026
dd00ef9
Merge branch 'main' into schema-evolution-02-20835
alamb Jul 22, 2026
d943826
Merge branch 'main' into schema-evolution-02-20835
kosiew Jul 24, 2026
96ab349
feat(datafusion): simplify nested struct handling by removing compati…
kosiew Jul 24, 2026
a59ede7
feat: add FixedSizeList Parquet E2E coverage
kosiew Jul 24, 2026
6600725
feat: replace panicking FixedSizeListArray::new with fallible try_new
kosiew Jul 24, 2026
8b0c478
docs: add mention of equal-width FixedSizeList
kosiew Jul 24, 2026
39ce782
Merge branch 'schema-evolution-02-20835' of github.com:kosiew/datafus…
kosiew Jul 24, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat(nested_struct): refine compatibility checks and restructure cast…
…ing logic

- Removed fixed-size-list compatibility check from the normal cast path.
- Moved compatibility guard into the null-parent retry path only.
- Adjusted cast_struct_column all-null fast path to follow struct compatibility validation, ensuring all-null structs reject missing non-nullable fields.
  • Loading branch information
kosiew committed Jun 29, 2026
commit ced5b4143423ac532cfbefddf9a3eff7d6821c7a
37 changes: 22 additions & 15 deletions datafusion/common/src/nested_struct.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,7 @@ fn cast_struct_column(
target_fields: &[Arc<Field>],
cast_options: &CastOptions,
) -> Result<ArrayRef> {
if source_col.data_type() == &DataType::Null
|| (!source_col.is_empty() && source_col.null_count() == source_col.len())
{
if source_col.data_type() == &DataType::Null {
return Ok(new_null_array(
&Struct(target_fields.to_vec().into()),
source_col.len(),
Expand All @@ -71,6 +69,14 @@ fn cast_struct_column(
if let Some(source_struct) = source_col.as_any().downcast_ref::<StructArray>() {
let source_fields = source_struct.fields();
validate_struct_compatibility(source_fields, target_fields)?;

if !source_col.is_empty() && source_col.null_count() == source_col.len() {
return Ok(new_null_array(
&Struct(target_fields.to_vec().into()),
source_col.len(),
));
}

let mut fields: Vec<Arc<Field>> = Vec::with_capacity(target_fields.len());
let mut arrays: Vec<ArrayRef> = Vec::with_capacity(target_fields.len());
let num_rows = source_col.len();
Expand Down Expand Up @@ -269,16 +275,6 @@ fn cast_fixed_size_list_column(
let source_values = source_list.values();
let target_type = target_inner_field.data_type();

// Guard schema compatibility before masking null-parent child values below.
// The retry path is only for value-level cast failures in child slots hidden
// by a null parent list, and must not make runtime accept schemas that
// planning rejects.
validate_data_type_compatibility(
target_inner_field.name(),
source_values.data_type(),
target_type,
)?;

let cast_values = match cast_column(source_values, target_type, cast_options) {
Ok(cast_values) => cast_values,
Err(error) => match cast_fixed_size_list_values_with_parent_nulls(
Expand All @@ -287,6 +283,7 @@ fn cast_fixed_size_list_column(
cast_options,
source_list.nulls(),
target_list_size,
target_inner_field.name(),
) {
Some(masked_cast) => masked_cast?,
None => return Err(error),
Expand All @@ -307,13 +304,23 @@ fn cast_fixed_size_list_values_with_parent_nulls(
cast_options: &CastOptions,
parent_nulls: Option<&NullBuffer>,
list_size: i32,
field_name: &str,
) -> Option<Result<ArrayRef>> {
let parent_nulls = parent_nulls.filter(|nulls| nulls.null_count() > 0)?;

// FixedSizeList stores child slots for null parent lists. Those child
// values are semantically hidden, but recursive casts still inspect them.
// Only in that case, mask hidden child slots to null and retry; if there
// are no parent nulls, propagate the original cast error unchanged.
// Before masking and retrying, guard schema compatibility so the retry only
// handles value-level failures in hidden child slots and cannot make runtime
// accept schemas that planning rejects.
if let Err(error) = validate_data_type_compatibility(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i still dont agree with having this validate call on the cast path

we dont do anything similar for other list types even though they could technically contain invalid data (that cant be casted) in a non-empty null slot

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

field_name,
source_values.data_type(),
target_type,
) {
return Some(Err(error));
}

let hidden_child_nulls = parent_nulls.expand(list_size as usize);
let masked_values = mask_array_values(source_values, &hidden_child_nulls);
Some(masked_values.and_then(|values| cast_column(&values, target_type, cast_options)))
Expand Down
Loading