Skip to content

Commit 90dc9a3

Browse files
committed
fix: guard nested float normalization at every recursion level
Once an enclosing struct contained a float, normalize_nested_floats rebuilt every child regardless of whether that child held any float data. An empty struct child then reached StructArray::new, which cannot infer a length from zero columns and panics. Iceberg exposes _partition as struct<> on an unpartitioned table, so arrays_overlap over a struct element carrying that column failed with a native exception where Spark returns a result. Apply the has_float_leaf guard inside normalize_nested_floats rather than only at the call sites, so float-free subtrees are returned unchanged. This fixes the panic and keeps the rebuild proportional to the float data.
1 parent e98a8f9 commit 90dc9a3

2 files changed

Lines changed: 116 additions & 3 deletions

File tree

‎native/spark-expr/src/array_funcs/arrays_overlap.rs‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -546,10 +546,11 @@ fn needs_comparator(dt: &DataType) -> bool {
546546
mod tests {
547547
use super::*;
548548
use arrow::array::{
549-
Float64Builder, Int32Array, Int32Builder, ListArray, ListBuilder, StructBuilder,
549+
Float64Array, Float64Builder, Int32Array, Int32Builder, ListArray, ListBuilder,
550+
StructArray, StructBuilder,
550551
};
551552
use arrow::buffer::{NullBuffer, OffsetBuffer};
552-
use arrow::datatypes::Field;
553+
use arrow::datatypes::{Field, Fields};
553554

554555
fn make_list_array(
555556
values: &Int32Array,
@@ -1095,6 +1096,42 @@ mod tests {
10951096
list_builder.finish()
10961097
}
10971098

1099+
/// Regression test for an empty struct alongside a float field. Iceberg exposes
1100+
/// `_partition` as `struct<>` on an unpartitioned table, so
1101+
/// `arrays_overlap(array(named_struct('x', x, 'p', _partition)), ...)` reaches the
1102+
/// nested path with a zero column struct child. Rebuilding that child used to panic.
1103+
#[test]
1104+
fn test_struct_with_empty_struct_field_overlap() -> Result<()> {
1105+
fn make_list(values: Vec<f64>) -> ListArray {
1106+
let len = values.len();
1107+
let x: ArrayRef = Arc::new(Float64Array::from(values));
1108+
let partition: ArrayRef = Arc::new(StructArray::new_empty_fields(len, None));
1109+
let fields: Fields = vec![
1110+
Arc::new(Field::new("x", DataType::Float64, true)),
1111+
Arc::new(Field::new("p", partition.data_type().clone(), true)),
1112+
]
1113+
.into();
1114+
let element: ArrayRef =
1115+
Arc::new(StructArray::new(fields.clone(), vec![x, partition], None));
1116+
ListArray::new(
1117+
Arc::new(Field::new("item", DataType::Struct(fields), true)),
1118+
OffsetBuffer::new((0..=len as i32).collect::<Vec<_>>().into()),
1119+
element,
1120+
None,
1121+
)
1122+
}
1123+
1124+
// One row per element, mirroring `SELECT ... FROM t` over values 1.0 and 2.0.
1125+
let left = make_list(vec![1.0, 2.0]);
1126+
let right = make_list(vec![1.0, 2.0]);
1127+
1128+
let result = arrays_overlap_list::<i32>(&left, &right)?;
1129+
let result = result.as_any().downcast_ref::<BooleanArray>().unwrap();
1130+
assert!(result.value(0));
1131+
assert!(result.value(1));
1132+
Ok(())
1133+
}
1134+
10981135
#[test]
10991136
fn test_struct_float_field_signed_zero_overlap() -> Result<()> {
11001137
// [{-0.0}] vs [{0.0}] => true, matching Spark

‎native/spark-expr/src/array_funcs/nested_float_normalize.rs‎

Lines changed: 77 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,17 @@ pub(super) fn has_float_leaf(dt: &DataType) -> bool {
3636

3737
/// Recursively rebuilds nested arrays with `-0.0` normalized to `0.0` and NaN canonicalized
3838
/// in any Float32/Float64 leaves.
39+
///
40+
/// Subtrees without a float leaf are returned as is. The guard is applied at every level, not
41+
/// just by the caller, so that a float-free sibling of a float field is never rebuilt. That
42+
/// keeps the rebuild proportional to the float data, and it also leaves empty structs alone:
43+
/// `StructArray::new` cannot infer a length from zero columns and would panic. An empty struct
44+
/// reaches this code through Iceberg's `_partition` metadata column on an unpartitioned table.
3945
pub(super) fn normalize_nested_floats(array: &ArrayRef) -> ArrayRef {
46+
if !has_float_leaf(array.data_type()) {
47+
return Arc::clone(array);
48+
}
49+
4050
match array.data_type() {
4151
DataType::Float32 => {
4252
let normalized: Float32Array =
@@ -96,8 +106,9 @@ pub(super) fn normalize_nested_floats(array: &ArrayRef) -> ArrayRef {
96106
mod tests {
97107
use super::*;
98108
use arrow::array::Float64Builder;
109+
use arrow::array::Int32Array;
99110
use arrow::array::ListBuilder;
100-
use arrow::datatypes::Field;
111+
use arrow::datatypes::{Field, Fields};
101112

102113
#[test]
103114
fn test_has_float_leaf() {
@@ -160,6 +171,71 @@ mod tests {
160171
assert_eq!(inner.value(1).to_bits(), f64::NAN.to_bits());
161172
}
162173

174+
/// An empty struct sibling of a float field must survive normalization. Iceberg exposes
175+
/// `_partition` as `struct<>` on an unpartitioned table, and rebuilding it would panic
176+
/// because `StructArray::new` cannot infer a length from zero columns.
177+
#[test]
178+
fn test_normalize_struct_with_empty_struct_sibling() {
179+
let x = Float64Array::from(vec![Some(-0.0), Some(1.0)]);
180+
let partition = StructArray::new_empty_fields(2, None);
181+
let fields = vec![
182+
Arc::new(Field::new("x", DataType::Float64, true)),
183+
Arc::new(Field::new("p", partition.data_type().clone(), true)),
184+
];
185+
let arr: ArrayRef = Arc::new(StructArray::new(
186+
fields.into(),
187+
vec![Arc::new(x), Arc::new(partition)],
188+
None,
189+
));
190+
191+
let normalized = normalize_nested_floats(&arr);
192+
let normalized = normalized.as_struct();
193+
194+
let col_x = normalized.column(0).as_primitive::<Float64Type>();
195+
assert_eq!(col_x.value(0).to_bits(), 0.0f64.to_bits());
196+
assert_eq!(col_x.value(1), 1.0);
197+
198+
let col_p = normalized.column(1).as_struct();
199+
assert_eq!(col_p.num_columns(), 0);
200+
assert_eq!(col_p.len(), 2);
201+
}
202+
203+
/// A float-free subtree is returned as is rather than rebuilt. The child here is a nested
204+
/// struct, which the recursive arms would otherwise rebuild into a fresh array.
205+
#[test]
206+
fn test_normalize_leaves_float_free_subtree_untouched() {
207+
let inner_fields: Fields = vec![Arc::new(Field::new("i", DataType::Int32, true))].into();
208+
let inner: ArrayRef = Arc::new(StructArray::new(
209+
inner_fields.clone(),
210+
vec![Arc::new(Int32Array::from(vec![Some(1), Some(2)]))],
211+
None,
212+
));
213+
let floats = Float64Array::from(vec![Some(-0.0), Some(1.0)]);
214+
let fields: Fields = vec![
215+
Arc::new(Field::new("s", DataType::Struct(inner_fields), true)),
216+
Arc::new(Field::new("f", DataType::Float64, true)),
217+
]
218+
.into();
219+
let arr: ArrayRef = Arc::new(StructArray::new(
220+
fields,
221+
vec![Arc::clone(&inner), Arc::new(floats)],
222+
None,
223+
));
224+
225+
let normalized = normalize_nested_floats(&arr);
226+
let normalized = normalized.as_struct();
227+
228+
assert!(Arc::ptr_eq(normalized.column(0), &inner));
229+
assert_eq!(
230+
normalized
231+
.column(1)
232+
.as_primitive::<Float64Type>()
233+
.value(0)
234+
.to_bits(),
235+
0.0f64.to_bits()
236+
);
237+
}
238+
163239
#[test]
164240
fn test_normalize_struct_floats() {
165241
let a = Float64Array::from(vec![Some(-0.0), Some(1.0)]);

0 commit comments

Comments
 (0)