|
18 | 18 | use std::sync::{Arc, OnceLock}; |
19 | 19 |
|
20 | 20 | use arrow::array::{ |
21 | | - Array, BooleanArray, Capacities, MutableArrayData, Scalar, cast::AsArray, make_array, |
| 21 | + Array, Capacities, MutableArrayData, Scalar, cast::AsArray, make_array, |
22 | 22 | make_comparator, |
23 | 23 | }; |
24 | 24 | use arrow::compute::SortOptions; |
25 | 25 | use arrow::datatypes::{DataType, Field, FieldRef}; |
26 | | -use arrow_buffer::NullBuffer; |
27 | 26 |
|
28 | 27 | use datafusion_common::cast::{as_map_array, as_struct_array}; |
29 | 28 | use datafusion_common::{ |
@@ -100,29 +99,19 @@ impl Default for GetFieldFunc { |
100 | 99 | } |
101 | 100 | } |
102 | 101 |
|
103 | | -/// Process a map array by finding matching keys and extracting corresponding values. |
| 102 | +/// Process a map array with a non-nested key type by comparing the single |
| 103 | +/// lookup key against every map key with the `eq` kernel, then scanning the |
| 104 | +/// result for each row. |
104 | 105 | /// |
105 | | -/// This function handles both simple (scalar) and nested key types by using |
106 | | -/// appropriate comparison strategies. |
| 106 | +/// `eq` does not support nested types, so list, struct, and map keys go |
| 107 | +/// through [`process_map_with_nested_key`] instead. |
107 | 108 | fn process_map_array( |
108 | 109 | array: &dyn Array, |
109 | 110 | key_array: Arc<dyn Array>, |
110 | 111 | ) -> Result<ColumnarValue> { |
111 | 112 | let map_array = as_map_array(array)?; |
112 | | - let keys = if key_array.data_type().is_nested() { |
113 | | - let comparator = make_comparator( |
114 | | - map_array.keys().as_ref(), |
115 | | - key_array.as_ref(), |
116 | | - SortOptions::default(), |
117 | | - )?; |
118 | | - let len = map_array.keys().len().min(key_array.len()); |
119 | | - let values = (0..len).map(|i| comparator(i, i).is_eq()).collect(); |
120 | | - let nulls = NullBuffer::union(map_array.keys().nulls(), key_array.nulls()); |
121 | | - BooleanArray::new(values, nulls) |
122 | | - } else { |
123 | | - let be_compared = Scalar::new(key_array); |
124 | | - arrow::compute::kernels::cmp::eq(&be_compared, map_array.keys())? |
125 | | - }; |
| 113 | + let be_compared = Scalar::new(key_array); |
| 114 | + let keys = arrow::compute::kernels::cmp::eq(&be_compared, map_array.keys())?; |
126 | 115 |
|
127 | 116 | let original_data = map_array.entries().column(1).to_data(); |
128 | 117 | let capacity = Capacities::Array(original_data.len()); |
@@ -219,19 +208,14 @@ fn extract_single_field(base: ColumnarValue, name: ScalarValue) -> Result<Column |
219 | 208 | dict.with_values(Arc::clone(field_col)), |
220 | 209 | )) |
221 | 210 | } |
222 | | - (DataType::Map(_, _), ScalarValue::List(arr), _) => { |
223 | | - let key_array: Arc<dyn Array> = arr; |
224 | | - process_map_array(&array, key_array) |
225 | | - } |
226 | | - (DataType::Map(_, _), ScalarValue::Struct(arr), _) => { |
227 | | - process_map_array(&array, arr as Arc<dyn Array>) |
228 | | - } |
229 | | - (DataType::Map(_, _), other, _) => { |
230 | | - let data_type = other.data_type(); |
231 | | - if data_type.is_nested() { |
232 | | - process_map_with_nested_key(&array, &other.to_array()?) |
| 211 | + (DataType::Map(_, _), key, _) => { |
| 212 | + // The lookup key is a single scalar. `eq` does not support nested |
| 213 | + // key types, so those are matched with a comparator instead. |
| 214 | + let key_array = key.to_array()?; |
| 215 | + if key_array.data_type().is_nested() { |
| 216 | + process_map_with_nested_key(&array, key_array.as_ref()) |
233 | 217 | } else { |
234 | | - process_map_array(&array, other.to_array()?) |
| 218 | + process_map_array(&array, key_array) |
235 | 219 | } |
236 | 220 | } |
237 | 221 | (DataType::Struct(_), _, Some(k)) => { |
@@ -672,8 +656,11 @@ impl ScalarUDFImpl for GetFieldFunc { |
672 | 656 | #[cfg(test)] |
673 | 657 | mod tests { |
674 | 658 | use super::*; |
675 | | - use arrow::array::{ArrayRef, Int32Array, StructArray}; |
676 | | - use arrow::datatypes::Fields; |
| 659 | + use arrow::array::{ |
| 660 | + ArrayRef, Int32Array, Int32Builder, ListArray, ListBuilder, MapBuilder, |
| 661 | + StructArray, |
| 662 | + }; |
| 663 | + use arrow::datatypes::{Fields, Int32Type}; |
677 | 664 |
|
678 | 665 | #[test] |
679 | 666 | fn test_get_field_utf8view_key() -> Result<()> { |
@@ -711,6 +698,41 @@ mod tests { |
711 | 698 | Ok(()) |
712 | 699 | } |
713 | 700 |
|
| 701 | + #[test] |
| 702 | + fn test_get_field_map_list_key() -> Result<()> { |
| 703 | + // One map row with two list keys. The lookup key matches the second |
| 704 | + // entry, so the match is not at the first entry of the row. |
| 705 | + let mut builder = MapBuilder::new( |
| 706 | + None, |
| 707 | + ListBuilder::new(Int32Builder::new()), |
| 708 | + Int32Builder::new(), |
| 709 | + ); |
| 710 | + builder.keys().append_value([Some(1), Some(2)]); |
| 711 | + builder.values().append_value(1); |
| 712 | + builder.keys().append_value([Some(3), Some(4)]); |
| 713 | + builder.values().append_value(2); |
| 714 | + builder.append(true)?; |
| 715 | + let base = ColumnarValue::Array(Arc::new(builder.finish())); |
| 716 | + |
| 717 | + let list_key = |values: Vec<i32>| { |
| 718 | + ScalarValue::List(Arc::new( |
| 719 | + ListArray::from_iter_primitive::<Int32Type, _, _>([Some( |
| 720 | + values.into_iter().map(Some), |
| 721 | + )]), |
| 722 | + )) |
| 723 | + }; |
| 724 | + |
| 725 | + let result = extract_single_field(base.clone(), list_key(vec![3, 4]))?; |
| 726 | + let expected = Int32Array::from(vec![Some(2)]); |
| 727 | + assert_eq!(result.into_array(1)?.as_ref(), &expected as &dyn Array); |
| 728 | + |
| 729 | + let result = extract_single_field(base, list_key(vec![9, 9]))?; |
| 730 | + let expected = Int32Array::from(vec![None]); |
| 731 | + assert_eq!(result.into_array(1)?.as_ref(), &expected as &dyn Array); |
| 732 | + |
| 733 | + Ok(()) |
| 734 | + } |
| 735 | + |
714 | 736 | #[test] |
715 | 737 | fn test_get_field_dict_encoded_struct() -> Result<()> { |
716 | 738 | use arrow::array::{DictionaryArray, StringArray, UInt32Array}; |
|
0 commit comments