Skip to content

Commit 00c58db

Browse files
authored
fix: Ensure array_append, array_prepend handle NULLs correctly (apache#25229)
## Which issue does this PR close? - Closes apache#25222. ## Rationale for this change Arrow allows elements of a ListArray to be logically NULL (according to the validity buffer), but to have a non-empty offset range. Such elements should be considered NULL. `array_append` and `array_prepend` only considered the offsets when computing the results of these functions, which resulted in mishandling such NULL values. ## What changes are included in this PR? * Fix bug in `array_append`, `array_prepend` * Update test expectations to remove incorrect expected behavior for FSLs * Add new tests * Clarify docs on `generic_append_and_prepend` NULL handling ## What is the testing strategy for this PR? Existing tests pass, new tests added. ## Are there any user-facing changes? Yes, bugs fixed.
1 parent 6079a1e commit 00c58db

3 files changed

Lines changed: 80 additions & 9 deletions

File tree

‎datafusion/functions-nested/src/concat.rs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -572,7 +572,8 @@ where
572572
///
573573
/// This function takes a ListArray, an ArrayRef, a FieldRef, and a boolean flag
574574
/// indicating whether to append or prepend the elements. It returns a `Result<ArrayRef>`
575-
/// representing the resulting ListArray after the operation.
575+
/// representing the resulting ListArray after the operation. A NULL list is
576+
/// treated as an empty list, so its result holds only the element.
576577
///
577578
/// # Arguments
578579
///
@@ -583,10 +584,13 @@ where
583584
///
584585
/// # Examples
585586
///
587+
/// ```text
586588
/// generic_append_and_prepend(
587589
/// [1, 2, 3], 4, append => [1, 2, 3, 4]
588590
/// 5, [6, 7, 8], prepend => [5, 6, 7, 8]
591+
/// NULL, 4, append => [4]
589592
/// )
593+
/// ```
590594
fn generic_append_and_prepend<O: OffsetSizeTrait>(
591595
list_array: &GenericListArray<O>,
592596
element_array: &ArrayRef,
@@ -612,8 +616,14 @@ where
612616
let element_index = 1;
613617

614618
for (row_index, offset_window) in list_array.offsets().windows(2).enumerate() {
615-
let start = offset_window[0].to_usize().unwrap();
616-
let end = offset_window[1].to_usize().unwrap();
619+
let (start, end) = if list_array.is_null(row_index) {
620+
(0, 0)
621+
} else {
622+
(
623+
offset_window[0].to_usize().unwrap(),
624+
offset_window[1].to_usize().unwrap(),
625+
)
626+
};
617627
if is_append {
618628
mutable.try_extend(values_index, start, end)?;
619629
mutable.try_extend(element_index, row_index, row_index + 1)?;

‎datafusion/sqllogictest/test_files/array/array_append.slt‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ select array_append(column1, column2) from fixed_arrays_values;
192192
[11, 12, 13, 14, 15, 16, 17, 18, NULL, 20, 12]
193193
[21, 22, 23, NULL, 25, 26, 27, 28, 29, 30, 23]
194194
[31, 32, 33, 34, 35, NULL, 37, 38, 39, 40, 34]
195-
[NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 44]
195+
[44]
196196
[41, 42, 43, 44, 45, 46, 47, 48, 49, 50, NULL]
197197
[51, 52, NULL, 54, 55, 56, 57, 58, 59, 60, 55]
198198
[61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 66]
@@ -247,8 +247,8 @@ select array_append(column2, 100.1), array_append(column3, '.') from fixed_size_
247247
[7.7, 8.8, 9.9, 100.1] [d, NULL, l, o, r, .]
248248
[10.1, NULL, 12.2, 100.1] [s, i, t, a, b, .]
249249
[13.3, 14.4, 15.5, 100.1] [a, m, e, t, x, .]
250-
[NULL, NULL, NULL, 100.1] [,, a, b, c, d, .]
251-
[16.6, 17.7, 18.8, 100.1] [NULL, NULL, NULL, NULL, NULL, .]
250+
[100.1] [,, a, b, c, d, .]
251+
[16.6, 17.7, 18.8, 100.1] [.]
252252

253253
# array_append with columns and scalars #2
254254
query ??
@@ -331,5 +331,35 @@ from values (3), (NULL);
331331
[1, 2, 3] List(Int64)
332332
[1, 2, NULL] List(Int64)
333333

334+
# https://github.com/apache/datafusion/issues/25222
335+
query I??
336+
select
337+
column3,
338+
nullif(column1, column2),
339+
array_append(nullif(column1, column2), 9)
340+
from values ([1, 2, 3], [1, 2, 3], 1), ([4, 5, 6], [7], 2)
341+
order by column3;
342+
----
343+
1 NULL [9]
344+
2 [4, 5, 6] [4, 5, 6, 9]
345+
346+
query I?
347+
select
348+
column3,
349+
array_append(arrow_cast(nullif(column1, column2), 'LargeList(Int64)'), 9)
350+
from values ([1, 2, 3], [1, 2, 3], 1), ([4, 5, 6], [7], 2)
351+
order by column3;
352+
----
353+
1 [9]
354+
2 [4, 5, 6, 9]
355+
356+
query I?
357+
select column2, array_append(column1, 9)
358+
from values (arrow_cast([1, 2, 3], 'FixedSizeList(3, Int64)'), 1), (NULL, 2)
359+
order by column2;
360+
----
361+
1 [1, 2, 3, 9]
362+
2 [9]
363+
334364

335365
include ./cleanup.slt.part

‎datafusion/sqllogictest/test_files/array/array_prepend.slt‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ select array_prepend(column2, column1) from fixed_arrays_values;
196196
[12, 11, 12, 13, 14, 15, 16, 17, 18, NULL, 20]
197197
[23, 21, 22, 23, NULL, 25, 26, 27, 28, 29, 30]
198198
[34, 31, 32, 33, 34, 35, NULL, 37, 38, 39, 40]
199-
[44, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL]
199+
[44]
200200
[NULL, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50]
201201
[55, 51, 52, NULL, 54, 55, 56, 57, 58, 59, 60]
202202
[66, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70]
@@ -251,8 +251,8 @@ select array_prepend(100.1, column2), array_prepend('.', column3) from fixed_siz
251251
[100.1, 7.7, 8.8, 9.9] [., d, NULL, l, o, r]
252252
[100.1, 10.1, NULL, 12.2] [., s, i, t, a, b]
253253
[100.1, 13.3, 14.4, 15.5] [., a, m, e, t, x]
254-
[100.1, NULL, NULL, NULL] [., ,, a, b, c, d]
255-
[100.1, 16.6, 17.7, 18.8] [., NULL, NULL, NULL, NULL, NULL]
254+
[100.1] [., ,, a, b, c, d]
255+
[100.1, 16.6, 17.7, 18.8] [.]
256256

257257
# array_prepend with columns and scalars #2 (element is list)
258258
query ??
@@ -336,4 +336,35 @@ from values (0), (NULL);
336336
[NULL, 1, 2] List(Int64)
337337

338338

339+
# https://github.com/apache/datafusion/issues/25222
340+
query I??
341+
select
342+
column3,
343+
nullif(column1, column2),
344+
array_prepend(9, nullif(column1, column2))
345+
from values ([1, 2, 3], [1, 2, 3], 1), ([4, 5, 6], [7], 2)
346+
order by column3;
347+
----
348+
1 NULL [9]
349+
2 [4, 5, 6] [9, 4, 5, 6]
350+
351+
query I?
352+
select
353+
column3,
354+
array_prepend(9, arrow_cast(nullif(column1, column2), 'LargeList(Int64)'))
355+
from values ([1, 2, 3], [1, 2, 3], 1), ([4, 5, 6], [7], 2)
356+
order by column3;
357+
----
358+
1 [9]
359+
2 [9, 4, 5, 6]
360+
361+
query I?
362+
select column2, array_prepend(9, column1)
363+
from values (arrow_cast([1, 2, 3], 'FixedSizeList(3, Int64)'), 1), (NULL, 2)
364+
order by column2;
365+
----
366+
1 [9, 1, 2, 3]
367+
2 [9]
368+
369+
339370
include ./cleanup.slt.part

0 commit comments

Comments
 (0)