Skip to content

Commit 9f377c4

Browse files
authored
fix: correct list field inner type in array functions (apache#24345)
## Which issue does this PR close? - Closes apache#24341 ## Rationale for this change There is a regression from 54.1.0 to 55.0.0rc2 where the spark function `slice` can fail as demonstrated in the updated unit test, and as described in the connected issue. ## What changes are included in this PR? In the nested functions, get the inner field directly from the input for array slicing operations. Also, since it is possible the inner list could be non-nullable emit an empty slice instead of a null child element for the outer lists's nulls. ## Are these changes tested? Added 6 new tests in the SLT suite. ## Are there any user-facing changes? None
1 parent 574fe67 commit 9f377c4

4 files changed

Lines changed: 97 additions & 9 deletions

File tree

‎datafusion/functions-nested/src/extract.rs‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,8 @@ use arrow::array::{
2323
};
2424
use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
2525
use arrow::datatypes::DataType;
26-
use arrow::datatypes::{
27-
DataType::{FixedSizeList, LargeList, LargeListView, List, ListView, Null},
28-
Field,
26+
use arrow::datatypes::DataType::{
27+
FixedSizeList, LargeList, LargeListView, List, ListView, Null,
2928
};
3029
use datafusion_common::cast::as_large_list_array;
3130
use datafusion_common::cast::as_list_array;
@@ -622,9 +621,23 @@ where
622621
let values = array.values();
623622
let original_data = values.to_data();
624623
let capacity = Capacities::Array(original_data.len());
624+
// Carry the input's list field through to the output so that the returned
625+
// type matches the one promised by `return_type` / `return_field_from_args`,
626+
// including the field name, nullability and metadata.
627+
let field = match array.data_type() {
628+
List(field) | LargeList(field) => Arc::clone(field),
629+
other => {
630+
return internal_err!(
631+
"general_array_slice got unexpected data type: {other}"
632+
);
633+
}
634+
};
625635

636+
// `use_nulls` is false because we never call `try_extend_nulls`: null rows are
637+
// emitted as empty slices. Arrow still allocates a validity buffer on its own
638+
// if the child array has nulls.
626639
let mut mutable =
627-
MutableArrayData::with_capacities(vec![&original_data], true, capacity);
640+
MutableArrayData::with_capacities(vec![&original_data], false, capacity);
628641

629642
// We have the slice syntax compatible with DuckDB v0.8.1.
630643
// The rule `adjusted_from_index` and `adjusted_to_index` follows the rule of array_slice in duckdb.
@@ -638,9 +651,11 @@ where
638651
let end = offset_window[1];
639652
let len = end - start;
640653

654+
// The row is null, so its contents are never observed. Emit an empty
655+
// slice rather than a null child element: the input's list field may be
656+
// non-nullable, in which case a null child would be invalid.
641657
if nulls.as_ref().is_some_and(|n| n.is_null(row_index)) {
642-
mutable.try_extend_nulls(1)?;
643-
offsets.push(offsets[row_index] + O::usize_as(1));
658+
offsets.push(offsets[row_index]);
644659
continue;
645660
}
646661

@@ -682,7 +697,7 @@ where
682697
let data = mutable.freeze();
683698

684699
Ok(Arc::new(GenericListArray::<O>::try_new(
685-
Arc::new(Field::new_list_field(array.value_type(), true)),
700+
field,
686701
OffsetBuffer::<O>::new(offsets.into()),
687702
arrow::array::make_array(data),
688703
nulls,
@@ -704,12 +719,15 @@ where
704719
let field = match array.data_type() {
705720
ListView(field) | LargeListView(field) => Arc::clone(field),
706721
other => {
707-
return internal_err!("array_slice got unexpected data type: {}", other);
722+
return internal_err!(
723+
"general_list_view_array_slice got unexpected data type: {other}"
724+
);
708725
}
709726
};
710727

728+
// See the note on `use_nulls` in `general_array_slice`.
711729
let mut mutable =
712-
MutableArrayData::with_capacities(vec![&original_data], true, capacity);
730+
MutableArrayData::with_capacities(vec![&original_data], false, capacity);
713731

714732
// We must build `offsets` and `sizes` buffers manually as ListView does not enforce
715733
// monotonically increasing offsets.

‎datafusion/sqllogictest/test_files/array/array_pop.slt‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,5 +318,38 @@ select array_pop_front(arrow_cast([1, 2], 'LargeListView(Int64)'));
318318
----
319319
[2]
320320

321+
# maintains inner nullability
322+
query ??TT
323+
select
324+
array_pop_front(column1),
325+
array_pop_back(column1),
326+
arrow_typeof(array_pop_front(column1)),
327+
arrow_typeof(array_pop_back(column1))
328+
from values
329+
(arrow_cast([], 'List(non-null Int32)')),
330+
(arrow_cast(NULL, 'List(non-null Int32)')),
331+
(arrow_cast([1, 3, 5, -5], 'List(non-null Int32)'))
332+
;
333+
----
334+
[] [] List(non-null Int32) List(non-null Int32)
335+
NULL NULL List(non-null Int32) List(non-null Int32)
336+
[3, 5, -5] [1, 3, 5] List(non-null Int32) List(non-null Int32)
337+
338+
query ??TT
339+
select
340+
array_pop_front(column1),
341+
array_pop_back(column1),
342+
arrow_typeof(array_pop_front(column1)),
343+
arrow_typeof(array_pop_back(column1))
344+
from values
345+
(arrow_cast([], 'LargeList(non-null Int32)')),
346+
(arrow_cast(NULL, 'LargeList(non-null Int32)')),
347+
(arrow_cast([1, 3, 5, -5], 'LargeList(non-null Int32)'))
348+
;
349+
----
350+
[] [] LargeList(non-null Int32) LargeList(non-null Int32)
351+
NULL NULL LargeList(non-null Int32) LargeList(non-null Int32)
352+
[3, 5, -5] [1, 3, 5] LargeList(non-null Int32) LargeList(non-null Int32)
353+
321354

322355
include ./cleanup.slt.part

‎datafusion/sqllogictest/test_files/array/array_slice.slt‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,37 @@ NULL
450450
NULL
451451
[1, 3, 5]
452452

453+
# maintains inner nullability
454+
query ?T
455+
select array_slice(column1, 2, 3), arrow_typeof(array_slice(column1, 2, 3))
456+
from values
457+
(arrow_cast([], 'List(non-null Int32)')),
458+
(arrow_cast(NULL, 'List(non-null Int32)')),
459+
(arrow_cast([1, 3, 5, -5], 'List(non-null Int32)'))
460+
;
461+
----
462+
[] List(non-null Int32)
463+
NULL List(non-null Int32)
464+
[3, 5] List(non-null Int32)
465+
466+
query ?T
467+
select array_slice(column1, 2, 3), arrow_typeof(array_slice(column1, 2, 3))
468+
from values
469+
(arrow_cast([], 'LargeList(non-null Int32)')),
470+
(arrow_cast(NULL, 'LargeList(non-null Int32)')),
471+
(arrow_cast([1, 3, 5, -5], 'LargeList(non-null Int32)'))
472+
;
473+
----
474+
[] LargeList(non-null Int32)
475+
NULL LargeList(non-null Int32)
476+
[3, 5] LargeList(non-null Int32)
477+
478+
query ?T
479+
select array_slice(column1, 2, 3, 2), arrow_typeof(array_slice(column1, 2, 3, 2))
480+
from values (arrow_cast([1, 3, 5, -5], 'List(non-null Int32)'));
481+
----
482+
[3] List(non-null Int32)
483+
453484
# Testing with empty arguments should result in an error
454485
query error DataFusion error: Error during planning: 'array_slice' does not support zero arguments
455486
select array_slice();

‎datafusion/sqllogictest/test_files/spark/array/slice.slt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,3 +152,9 @@ query ?
152152
SELECT slice(make_array(1), 3, 4)
153153
----
154154
[]
155+
156+
# the inner field name of the input list is preserved
157+
query ?T
158+
SELECT slice(array(1, 2, 3, 4), 2, 2), arrow_typeof(slice(array(1, 2, 3, 4), 2, 2));
159+
----
160+
[2, 3] List(Int64, field: 'element')

0 commit comments

Comments
 (0)