Skip to content

Commit 505af5f

Browse files
neilconwaythorrester
authored andcommitted
fix: map[key] panics for nested map keys (apache#25130)
## Which issue does this PR close? - Closes apache#25082. ## Rationale for this change `get_field` has two map lookup paths: * `process_map_with_nested_key` uses a comparator and works for all key types * `process_map_array` uses the `eq` kernel, and therefore does not support map keys that are nested types (e.g., lists). The previous implementation arranged to call `process_map_with_nested_key` for map-typed keys, but that was too narrow: this path should be taken for list- and struct-typed keys as well. ## What changes are included in this PR? * Route every nested key type through `process_map_with_nested_key` * Add unit test and SLTs ## What is the testing strategy for this PR? Existing tests pass; new test cases added. ## Are there any user-facing changes? No, aside from addressing the bug. (cherry picked from commit 301235e)
1 parent 65e8bf0 commit 505af5f

2 files changed

Lines changed: 85 additions & 33 deletions

File tree

‎datafusion/functions/src/core/getfield.rs‎

Lines changed: 55 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,11 @@
1818
use std::sync::{Arc, OnceLock};
1919

2020
use arrow::array::{
21-
Array, BooleanArray, Capacities, MutableArrayData, Scalar, cast::AsArray, make_array,
21+
Array, Capacities, MutableArrayData, Scalar, cast::AsArray, make_array,
2222
make_comparator,
2323
};
2424
use arrow::compute::SortOptions;
2525
use arrow::datatypes::{DataType, Field, FieldRef};
26-
use arrow_buffer::NullBuffer;
2726

2827
use datafusion_common::cast::{as_map_array, as_struct_array};
2928
use datafusion_common::{
@@ -100,29 +99,19 @@ impl Default for GetFieldFunc {
10099
}
101100
}
102101

103-
/// Process a map array by finding matching keys and extracting corresponding values.
102+
/// Process a map array with a non-nested key type by comparing the single
103+
/// lookup key against every map key with the `eq` kernel, then scanning the
104+
/// result for each row.
104105
///
105-
/// This function handles both simple (scalar) and nested key types by using
106-
/// appropriate comparison strategies.
106+
/// `eq` does not support nested types, so list, struct, and map keys go
107+
/// through [`process_map_with_nested_key`] instead.
107108
fn process_map_array(
108109
array: &dyn Array,
109110
key_array: Arc<dyn Array>,
110111
) -> Result<ColumnarValue> {
111112
let map_array = as_map_array(array)?;
112-
let keys = if key_array.data_type().is_nested() {
113-
let comparator = make_comparator(
114-
map_array.keys().as_ref(),
115-
key_array.as_ref(),
116-
SortOptions::default(),
117-
)?;
118-
let len = map_array.keys().len().min(key_array.len());
119-
let values = (0..len).map(|i| comparator(i, i).is_eq()).collect();
120-
let nulls = NullBuffer::union(map_array.keys().nulls(), key_array.nulls());
121-
BooleanArray::new(values, nulls)
122-
} else {
123-
let be_compared = Scalar::new(key_array);
124-
arrow::compute::kernels::cmp::eq(&be_compared, map_array.keys())?
125-
};
113+
let be_compared = Scalar::new(key_array);
114+
let keys = arrow::compute::kernels::cmp::eq(&be_compared, map_array.keys())?;
126115

127116
let original_data = map_array.entries().column(1).to_data();
128117
let capacity = Capacities::Array(original_data.len());
@@ -219,19 +208,14 @@ fn extract_single_field(base: ColumnarValue, name: ScalarValue) -> Result<Column
219208
dict.with_values(Arc::clone(field_col)),
220209
))
221210
}
222-
(DataType::Map(_, _), ScalarValue::List(arr), _) => {
223-
let key_array: Arc<dyn Array> = arr;
224-
process_map_array(&array, key_array)
225-
}
226-
(DataType::Map(_, _), ScalarValue::Struct(arr), _) => {
227-
process_map_array(&array, arr as Arc<dyn Array>)
228-
}
229-
(DataType::Map(_, _), other, _) => {
230-
let data_type = other.data_type();
231-
if data_type.is_nested() {
232-
process_map_with_nested_key(&array, &other.to_array()?)
211+
(DataType::Map(_, _), key, _) => {
212+
// The lookup key is a single scalar. `eq` does not support nested
213+
// key types, so those are matched with a comparator instead.
214+
let key_array = key.to_array()?;
215+
if key_array.data_type().is_nested() {
216+
process_map_with_nested_key(&array, key_array.as_ref())
233217
} else {
234-
process_map_array(&array, other.to_array()?)
218+
process_map_array(&array, key_array)
235219
}
236220
}
237221
(DataType::Struct(_), _, Some(k)) => {
@@ -710,8 +694,11 @@ impl ScalarUDFImpl for GetFieldFunc {
710694
#[cfg(test)]
711695
mod tests {
712696
use super::*;
713-
use arrow::array::{ArrayRef, Int32Array, StructArray};
714-
use arrow::datatypes::Fields;
697+
use arrow::array::{
698+
ArrayRef, Int32Array, Int32Builder, ListArray, ListBuilder, MapBuilder,
699+
StructArray,
700+
};
701+
use arrow::datatypes::{Fields, Int32Type};
715702

716703
#[test]
717704
fn test_get_field_utf8view_key() -> Result<()> {
@@ -749,6 +736,41 @@ mod tests {
749736
Ok(())
750737
}
751738

739+
#[test]
740+
fn test_get_field_map_list_key() -> Result<()> {
741+
// One map row with two list keys. The lookup key matches the second
742+
// entry, so the match is not at the first entry of the row.
743+
let mut builder = MapBuilder::new(
744+
None,
745+
ListBuilder::new(Int32Builder::new()),
746+
Int32Builder::new(),
747+
);
748+
builder.keys().append_value([Some(1), Some(2)]);
749+
builder.values().append_value(1);
750+
builder.keys().append_value([Some(3), Some(4)]);
751+
builder.values().append_value(2);
752+
builder.append(true)?;
753+
let base = ColumnarValue::Array(Arc::new(builder.finish()));
754+
755+
let list_key = |values: Vec<i32>| {
756+
ScalarValue::List(Arc::new(
757+
ListArray::from_iter_primitive::<Int32Type, _, _>([Some(
758+
values.into_iter().map(Some),
759+
)]),
760+
))
761+
};
762+
763+
let result = extract_single_field(base.clone(), list_key(vec![3, 4]))?;
764+
let expected = Int32Array::from(vec![Some(2)]);
765+
assert_eq!(result.into_array(1)?.as_ref(), &expected as &dyn Array);
766+
767+
let result = extract_single_field(base, list_key(vec![9, 9]))?;
768+
let expected = Int32Array::from(vec![None]);
769+
assert_eq!(result.into_array(1)?.as_ref(), &expected as &dyn Array);
770+
771+
Ok(())
772+
}
773+
752774
#[test]
753775
fn test_get_field_dict_encoded_struct() -> Result<()> {
754776
use arrow::array::{DictionaryArray, StringArray, UInt32Array};

‎datafusion/sqllogictest/test_files/map.slt‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -597,6 +597,36 @@ NULL
597597
# TODO(https://github.com/apache/datafusion/pull/18394): Test accessing map with empty map as key
598598
# TODO(https://github.com/apache/datafusion/pull/18394): Test accessing map with null map as key
599599

600+
# accessing map with list key: match on a later entry, absent key, match on the first entry
601+
query III
602+
SELECT MAP {[1,2]:1, [3,4]:2}[[3,4]], MAP {[1,2]:1, [3,4]:2}[[9,9]], MAP {[1,2]:1, [3,4]:2}[[1,2]];
603+
----
604+
2 NULL 1
605+
606+
# accessing map with struct key
607+
query II
608+
SELECT MAP {named_struct('a',1):1, named_struct('a',2):2}[named_struct('a',2)],
609+
MAP {named_struct('a',1):1, named_struct('a',2):2}[named_struct('a',3)];
610+
----
611+
2 NULL
612+
613+
# accessing map with list key from a column, so the lookup runs on a batch rather than at planning time
614+
statement ok
615+
CREATE TABLE map_list_keys AS
616+
SELECT MAP {[1,2]:1, [3,4]:2} AS m
617+
UNION ALL SELECT MAP {[3,4]:5, [1,2]:6}
618+
UNION ALL SELECT NULL;
619+
620+
query II rowsort
621+
SELECT m[[3,4]], m[[9,9]] FROM map_list_keys;
622+
----
623+
2 NULL
624+
5 NULL
625+
NULL NULL
626+
627+
statement ok
628+
DROP TABLE map_list_keys;
629+
600630
# accessing map with non-string key
601631
query I
602632
SELECT MAKE_MAP(1, null, 2, 33, 3, null)[2];

0 commit comments

Comments
 (0)