Skip to content

Instantly share code, notes, and snippets.

@neilconway
Created September 14, 2026 14:10
Show Gist options
  • Select an option

  • Save neilconway/b0c3cb48a16d1fd06ac9818e6cad8fb1 to your computer and use it in GitHub Desktop.

Select an option

Save neilconway/b0c3cb48a16d1fd06ac9818e6cad8fb1 to your computer and use it in GitHub Desktop.
diff --git a/datafusion/functions/src/core/getfield.rs b/datafusion/functions/src/core/getfield.rs
index c7003a372..6cb5cdaa3 100644
--- a/datafusion/functions/src/core/getfield.rs
+++ b/datafusion/functions/src/core/getfield.rs
@@ -18,11 +18,11 @@
use std::sync::{Arc, OnceLock};
use arrow::array::{
- Array, ArrayRef, Capacities, MutableArrayData, Scalar, cast::AsArray, make_array,
- make_comparator,
+ Array, ArrayRef, BooleanArray, Capacities, MutableArrayData, Scalar, cast::AsArray,
+ layout, make_array, make_comparator,
};
use arrow::buffer::NullBuffer;
-use arrow::compute::SortOptions;
+use arrow::compute::{SortOptions, nullif};
use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::cast::{as_map_array, as_struct_array};
@@ -199,27 +199,28 @@ fn apply_parent_nulls(col: &ArrayRef, parent_nulls: &NullBuffer) -> Result<Array
return Ok(Arc::clone(col));
}
- let data = col.to_data();
- match col.data_type() {
- DataType::Union(_, _) => {
- // Unions represent nulls in their children. Rebuild the array so null
- // parents become null union values in both sparse and dense layouts.
- let mut mutable = MutableArrayData::new(vec![&data], true, data.len());
- let mut end = 0;
- for (start, valid_end) in parent_nulls.valid_slices() {
- mutable.try_extend_nulls(start - end)?;
- mutable.try_extend(0, start, valid_end)?;
- end = valid_end;
- }
- mutable.try_extend_nulls(data.len() - end)?;
+ if layout(col.data_type()).can_contain_null_mask {
+ // `nullif` marks a row null where the mask is true and keeps the
+ // field's own nulls. Only the validity bitmap is rebuilt; the value
+ // buffers and child arrays are shared with `col`.
+ let null_parents = BooleanArray::new(!parent_nulls.inner(), None);
+ return Ok(nullif(col.as_ref(), &null_parents)?);
+ }
- Ok(make_array(mutable.freeze()))
- }
- _ => {
- let nulls = NullBuffer::union(col.nulls(), Some(parent_nulls));
- Ok(make_array(data.into_builder().nulls(nulls).build()?))
- }
+ // Unions and run-end encoded arrays have no validity bitmap of their own
+ // and represent nulls in their children. Rebuild the array so null parents
+ // become null values in those children.
+ let data = col.to_data();
+ let mut mutable = MutableArrayData::new(vec![&data], true, data.len());
+ let mut end = 0;
+ for (start, valid_end) in parent_nulls.valid_slices() {
+ mutable.try_extend_nulls(start - end)?;
+ mutable.try_extend(0, start, valid_end)?;
+ end = valid_end;
}
+ mutable.try_extend_nulls(data.len() - end)?;
+
+ Ok(make_array(mutable.freeze()))
}
/// Extract a single field from a struct or map array
@@ -934,6 +935,54 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_get_field_run_encoded_parent_nulls() -> Result<()> {
+ use arrow::array::RunArray;
+
+ // Logical values: [10, 10, 20, NULL, NULL]
+ let run_ends = Int32Array::from(vec![2, 3, 5]);
+ let values = Int32Array::from(vec![Some(10), Some(20), None]);
+ let child = RunArray::<Int32Type>::try_new(&run_ends, &values)?;
+ let run_type = child.data_type().clone();
+ let parent = StructArray::new(
+ vec![Field::new("r", run_type.clone(), true)].into(),
+ vec![Arc::new(child)],
+ Some(NullBuffer::from(vec![true, false, true, true, true])),
+ );
+
+ for (input, expected) in [
+ (
+ Arc::new(parent.clone()) as ArrayRef,
+ vec![Some(10), None, Some(20), None, None],
+ ),
+ (
+ Arc::new(parent.slice(1, 4)) as ArrayRef,
+ vec![None, Some(20), None, None],
+ ),
+ ] {
+ let result = extract_single_field(
+ ColumnarValue::Array(input),
+ ScalarValue::Utf8(Some("r".to_string())),
+ )?
+ .into_array(expected.len())?;
+ assert_eq!(result.data_type(), &run_type);
+ result.to_data().validate_full()?;
+
+ let run_array = result.as_run::<Int32Type>();
+ let run_values = run_array.values().as_primitive::<Int32Type>();
+ let logical: Vec<Option<i32>> = (0..run_array.len())
+ .map(|i| {
+ let physical = run_array.get_physical_index(i);
+ run_values
+ .is_valid(physical)
+ .then(|| run_values.value(physical))
+ })
+ .collect();
+ assert_eq!(logical, expected);
+ }
+ Ok(())
+ }
+
#[test]
fn test_get_field_dict_encoded_struct() -> Result<()> {
use arrow::array::{DictionaryArray, StringArray, UInt32Array};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment