Created
September 14, 2026 14:10
-
-
Save neilconway/b0c3cb48a16d1fd06ac9818e6cad8fb1 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| diff --git a/datafusion/functions/src/core/getfield.rs b/datafusion/functions/src/core/getfield.rs | |
| index c7003a372..6cb5cdaa3 100644 | |
| --- a/datafusion/functions/src/core/getfield.rs | |
| +++ b/datafusion/functions/src/core/getfield.rs | |
| @@ -18,11 +18,11 @@ | |
| use std::sync::{Arc, OnceLock}; | |
| use arrow::array::{ | |
| - Array, ArrayRef, Capacities, MutableArrayData, Scalar, cast::AsArray, make_array, | |
| - make_comparator, | |
| + Array, ArrayRef, BooleanArray, Capacities, MutableArrayData, Scalar, cast::AsArray, | |
| + layout, make_array, make_comparator, | |
| }; | |
| use arrow::buffer::NullBuffer; | |
| -use arrow::compute::SortOptions; | |
| +use arrow::compute::{SortOptions, nullif}; | |
| use arrow::datatypes::{DataType, Field, FieldRef}; | |
| use datafusion_common::cast::{as_map_array, as_struct_array}; | |
| @@ -199,27 +199,28 @@ fn apply_parent_nulls(col: &ArrayRef, parent_nulls: &NullBuffer) -> Result<Array | |
| return Ok(Arc::clone(col)); | |
| } | |
| - let data = col.to_data(); | |
| - match col.data_type() { | |
| - DataType::Union(_, _) => { | |
| - // Unions represent nulls in their children. Rebuild the array so null | |
| - // parents become null union values in both sparse and dense layouts. | |
| - let mut mutable = MutableArrayData::new(vec![&data], true, data.len()); | |
| - let mut end = 0; | |
| - for (start, valid_end) in parent_nulls.valid_slices() { | |
| - mutable.try_extend_nulls(start - end)?; | |
| - mutable.try_extend(0, start, valid_end)?; | |
| - end = valid_end; | |
| - } | |
| - mutable.try_extend_nulls(data.len() - end)?; | |
| + if layout(col.data_type()).can_contain_null_mask { | |
| + // `nullif` marks a row null where the mask is true and keeps the | |
| + // field's own nulls. Only the validity bitmap is rebuilt; the value | |
| + // buffers and child arrays are shared with `col`. | |
| + let null_parents = BooleanArray::new(!parent_nulls.inner(), None); | |
| + return Ok(nullif(col.as_ref(), &null_parents)?); | |
| + } | |
| - Ok(make_array(mutable.freeze())) | |
| - } | |
| - _ => { | |
| - let nulls = NullBuffer::union(col.nulls(), Some(parent_nulls)); | |
| - Ok(make_array(data.into_builder().nulls(nulls).build()?)) | |
| - } | |
| + // Unions and run-end encoded arrays have no validity bitmap of their own | |
| + // and represent nulls in their children. Rebuild the array so null parents | |
| + // become null values in those children. | |
| + let data = col.to_data(); | |
| + let mut mutable = MutableArrayData::new(vec![&data], true, data.len()); | |
| + let mut end = 0; | |
| + for (start, valid_end) in parent_nulls.valid_slices() { | |
| + mutable.try_extend_nulls(start - end)?; | |
| + mutable.try_extend(0, start, valid_end)?; | |
| + end = valid_end; | |
| } | |
| + mutable.try_extend_nulls(data.len() - end)?; | |
| + | |
| + Ok(make_array(mutable.freeze())) | |
| } | |
| /// Extract a single field from a struct or map array | |
| @@ -934,6 +935,54 @@ mod tests { | |
| Ok(()) | |
| } | |
| + #[test] | |
| + fn test_get_field_run_encoded_parent_nulls() -> Result<()> { | |
| + use arrow::array::RunArray; | |
| + | |
| + // Logical values: [10, 10, 20, NULL, NULL] | |
| + let run_ends = Int32Array::from(vec![2, 3, 5]); | |
| + let values = Int32Array::from(vec![Some(10), Some(20), None]); | |
| + let child = RunArray::<Int32Type>::try_new(&run_ends, &values)?; | |
| + let run_type = child.data_type().clone(); | |
| + let parent = StructArray::new( | |
| + vec![Field::new("r", run_type.clone(), true)].into(), | |
| + vec![Arc::new(child)], | |
| + Some(NullBuffer::from(vec![true, false, true, true, true])), | |
| + ); | |
| + | |
| + for (input, expected) in [ | |
| + ( | |
| + Arc::new(parent.clone()) as ArrayRef, | |
| + vec![Some(10), None, Some(20), None, None], | |
| + ), | |
| + ( | |
| + Arc::new(parent.slice(1, 4)) as ArrayRef, | |
| + vec![None, Some(20), None, None], | |
| + ), | |
| + ] { | |
| + let result = extract_single_field( | |
| + ColumnarValue::Array(input), | |
| + ScalarValue::Utf8(Some("r".to_string())), | |
| + )? | |
| + .into_array(expected.len())?; | |
| + assert_eq!(result.data_type(), &run_type); | |
| + result.to_data().validate_full()?; | |
| + | |
| + let run_array = result.as_run::<Int32Type>(); | |
| + let run_values = run_array.values().as_primitive::<Int32Type>(); | |
| + let logical: Vec<Option<i32>> = (0..run_array.len()) | |
| + .map(|i| { | |
| + let physical = run_array.get_physical_index(i); | |
| + run_values | |
| + .is_valid(physical) | |
| + .then(|| run_values.value(physical)) | |
| + }) | |
| + .collect(); | |
| + assert_eq!(logical, expected); | |
| + } | |
| + Ok(()) | |
| + } | |
| + | |
| #[test] | |
| fn test_get_field_dict_encoded_struct() -> Result<()> { | |
| use arrow::array::{DictionaryArray, StringArray, UInt32Array}; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment