Skip to content

Commit 0639fcf

Browse files
neilconwayOmega359
authored andcommitted
fix: Correct array_repeat NULL handling (apache#25230)
## Which issue does this PR close? - Closes apache#25223. ## Rationale for this change Arrow allows elements of a `ListArray` to be logically NULL (according to the validity buffer), but to have non-empty offset ranges. Such elements should be considered NULL. `array_repeat` only considered the offsets and ignored the validity buffer, which resulted in mishandling such NULL values. ## What changes are included in this PR? * Fix `array_repeat` bug * Add unit test * Add SLT ## What is the testing strategy for this PR? Existing tests pass; new tests added. We add both a unit test and an SLT because the SLT depends on `nullif` creating such a NULL value, which might change in the future. ## Are there any user-facing changes? Yes, bug fixed.
1 parent 5c45370 commit 0639fcf

2 files changed

Lines changed: 79 additions & 6 deletions

File tree

‎datafusion/functions-nested/src/repeat.rs‎

Lines changed: 47 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -262,8 +262,16 @@ fn general_list_repeat<O: OffsetSizeTrait>(
262262
continue;
263263
};
264264
let list_is_valid = list_array.is_valid(row_idx);
265-
let start = list_offsets[row_idx].to_usize().unwrap();
266-
let end = list_offsets[row_idx + 1].to_usize().unwrap();
265+
// A NULL list slot may still have a non-empty offset range; treat the
266+
// range as empty to keep the inner offsets and taken values in step.
267+
let (start, end) = if list_is_valid {
268+
(
269+
list_offsets[row_idx].to_usize().unwrap(),
270+
list_offsets[row_idx + 1].to_usize().unwrap(),
271+
)
272+
} else {
273+
(0, 0)
274+
};
267275
let row_len = end - start;
268276

269277
for _ in 0..count {
@@ -272,9 +280,7 @@ fn general_list_repeat<O: OffsetSizeTrait>(
272280
let offset = checked_repeat_offset::<O>(inner_running)?;
273281
inner_offsets.push(offset);
274282
inner_nulls.append(list_is_valid);
275-
if list_is_valid {
276-
take_indices.extend(start as u64..end as u64);
277-
}
283+
take_indices.extend(start as u64..end as u64);
278284
}
279285
}
280286

@@ -388,7 +394,7 @@ mod tests {
388394
use super::{array_repeat_inner, general_list_repeat, general_repeat};
389395
use arrow::array::{Array, ArrayRef, AsArray, Int32Array, Int64Array, ListArray};
390396
use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
391-
use arrow::datatypes::{Field, Int32Type};
397+
use arrow::datatypes::{DataType, Field, Int32Type};
392398
use datafusion_common::Result;
393399
use std::sync::Arc;
394400

@@ -445,6 +451,41 @@ mod tests {
445451
Ok(())
446452
}
447453

454+
#[test]
455+
fn test_array_repeat_nested_null_list_with_nonempty_range() -> Result<()> {
456+
// A NULL list slot might still have a non-empty offset range
457+
// https://github.com/apache/datafusion/issues/25223
458+
let values = Int32Array::from(vec![1, 2, 3, 4, 5, 6]);
459+
let list_array = ListArray::new(
460+
Arc::new(Field::new_list_field(DataType::Int32, true)),
461+
OffsetBuffer::new(ScalarBuffer::from(vec![0, 3, 6])),
462+
Arc::new(values),
463+
Some(NullBuffer::from(vec![false, true])),
464+
);
465+
let counts = Int64Array::from(vec![2, 2]);
466+
467+
let result = general_list_repeat::<i32>(&list_array, &counts)?;
468+
let repeated_values = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
469+
None,
470+
None,
471+
Some(vec![Some(4), Some(5), Some(6)]),
472+
Some(vec![Some(4), Some(5), Some(6)]),
473+
]);
474+
let expected = ListArray::new(
475+
Arc::new(Field::new_list_field(
476+
repeated_values.data_type().clone(),
477+
true,
478+
)),
479+
OffsetBuffer::new(ScalarBuffer::from(vec![0, 2, 4])),
480+
Arc::new(repeated_values),
481+
None,
482+
);
483+
484+
assert_eq!(result.as_list::<i32>(), &expected);
485+
486+
Ok(())
487+
}
488+
448489
#[test]
449490
fn scalar_count_exceeding_max_array_size_returns_error() {
450491
let element: ArrayRef = Arc::new(Int64Array::from(vec![1]));

‎datafusion/sqllogictest/test_files/array/array_repeat.slt‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,5 +238,37 @@ NULL [NULL, NULL, NULL]
238238
statement ok
239239
drop table array_repeat_large_nested_null_table
240240

241+
# https://github.com/apache/datafusion/issues/25223
242+
statement ok
243+
create table array_repeat_null_slot_table
244+
as values
245+
([1, 2, 3], [1, 2, 3], 1),
246+
([4, 5, 6], [7], 2);
247+
248+
query I?
249+
select column3, array_repeat(nullif(column1, column2), 2)
250+
from array_repeat_null_slot_table order by column3;
251+
----
252+
1 [NULL, NULL]
253+
2 [[4, 5, 6], [4, 5, 6]]
254+
255+
query I?
256+
select
257+
column3,
258+
array_repeat(
259+
nullif(
260+
arrow_cast(column1, 'LargeList(Int64)'),
261+
arrow_cast(column2, 'LargeList(Int64)')
262+
),
263+
2
264+
)
265+
from array_repeat_null_slot_table order by column3;
266+
----
267+
1 [NULL, NULL]
268+
2 [[4, 5, 6], [4, 5, 6]]
269+
270+
statement ok
271+
drop table array_repeat_null_slot_table
272+
241273

242274
include ./cleanup.slt.part

0 commit comments

Comments
 (0)