|
29 | 29 | //! A long input value cannot match an inline list value because its length is |
30 | 30 | //! part of the view. |
31 | 31 |
|
32 | | -use std::marker::PhantomData; |
33 | 32 | use std::sync::Arc; |
34 | 33 |
|
35 | 34 | use arrow::array::{ |
36 | | - Array, ArrayRef, AsArray, BooleanArray, GenericByteViewArray, MAX_INLINE_VIEW_LEN, |
37 | | - PrimitiveArray, |
38 | | -}; |
39 | | -use arrow::buffer::ScalarBuffer; |
40 | | -use arrow::datatypes::{ |
41 | | - BinaryViewType, ByteViewType, DataType, Decimal128Type, StringViewType, |
| 35 | + Array, ArrayRef, AsArray, BooleanArray, MAX_INLINE_VIEW_LEN, PrimitiveArray, |
42 | 36 | }; |
| 37 | +use arrow::buffer::{NullBuffer, ScalarBuffer}; |
| 38 | +use arrow::datatypes::{DataType, Decimal128Type}; |
43 | 39 | use arrow::util::bit_iterator::BitIndexIterator; |
44 | 40 | use datafusion_common::{Result, exec_datafusion_err, internal_datafusion_err}; |
45 | 41 |
|
46 | 42 | use super::primitive_filter::instantiate_primitive_filter; |
47 | 43 | use super::static_filter::{StaticFilter, StaticFilterRef}; |
48 | 44 |
|
49 | | -fn view_len(view: u128) -> u32 { |
50 | | - view as u32 |
| 45 | +fn is_inline(view: u128) -> bool { |
| 46 | + (view as u32) <= MAX_INLINE_VIEW_LEN |
51 | 47 | } |
52 | 48 |
|
53 | | -fn downcast_byte_view<T: ByteViewType>( |
54 | | - array: &dyn Array, |
55 | | -) -> Result<&GenericByteViewArray<T>> { |
56 | | - array.as_byte_view_opt::<T>().ok_or_else(|| { |
57 | | - exec_datafusion_err!( |
58 | | - "Expected concrete {} array, got {}", |
59 | | - T::DATA_TYPE, |
60 | | - array.data_type() |
61 | | - ) |
62 | | - }) |
63 | | -} |
64 | | - |
65 | | -fn all_inline<T: ByteViewType>(array: &GenericByteViewArray<T>) -> bool { |
66 | | - let is_inline = |idx: usize| view_len(array.views()[idx]) <= MAX_INLINE_VIEW_LEN; |
67 | | - match array.nulls() { |
| 49 | +fn all_inline(views: &ScalarBuffer<u128>, nulls: Option<&NullBuffer>) -> bool { |
| 50 | + match nulls { |
68 | 51 | Some(nulls) => { |
69 | 52 | BitIndexIterator::new(nulls.validity(), nulls.offset(), nulls.len()) |
70 | | - .all(is_inline) |
| 53 | + .all(|idx| is_inline(views[idx])) |
71 | 54 | } |
72 | | - None => (0..array.len()).all(is_inline), |
| 55 | + None => views.iter().copied().all(is_inline), |
73 | 56 | } |
74 | 57 | } |
75 | 58 |
|
76 | | -fn as_decimal128<T: ByteViewType>( |
77 | | - array: &GenericByteViewArray<T>, |
| 59 | +fn as_decimal128( |
| 60 | + views: &ScalarBuffer<u128>, |
| 61 | + nulls: Option<&NullBuffer>, |
78 | 62 | ) -> PrimitiveArray<Decimal128Type> { |
79 | | - let views = array.views(); |
80 | 63 | // `views.inner()` is already sliced to the array's offset. |
81 | 64 | let values = ScalarBuffer::<i128>::new(views.inner().clone(), 0, views.len()); |
82 | | - PrimitiveArray::<Decimal128Type>::new(values, array.nulls().cloned()) |
| 65 | + PrimitiveArray::<Decimal128Type>::new(values, nulls.cloned()) |
83 | 66 | } |
84 | 67 |
|
85 | 68 | /// Adapts the selected primitive filter to the original byte-view type. |
86 | 69 | /// |
87 | 70 | /// Arrow requires unused inline bytes to be zero, so equal values have equal |
88 | 71 | /// `u128` views. |
89 | | -struct ByteViewFilter<T: ByteViewType> { |
| 72 | +struct ByteViewFilter { |
| 73 | + data_type: DataType, |
90 | 74 | inner: StaticFilterRef, |
91 | | - _marker: PhantomData<T>, |
92 | 75 | } |
93 | 76 |
|
94 | | -impl<T: ByteViewType> StaticFilter for ByteViewFilter<T> { |
| 77 | +impl StaticFilter for ByteViewFilter { |
95 | 78 | fn null_count(&self) -> usize { |
96 | 79 | self.inner.null_count() |
97 | 80 | } |
98 | 81 |
|
99 | 82 | fn contains(&self, v: &dyn Array, negated: bool) -> Result<BooleanArray> { |
100 | | - let array = downcast_byte_view::<T>(v)?; |
| 83 | + let (views, nulls) = match &self.data_type { |
| 84 | + DataType::Utf8View => v.as_string_view_opt().map(|a| (a.views(), a.nulls())), |
| 85 | + DataType::BinaryView => { |
| 86 | + v.as_binary_view_opt().map(|a| (a.views(), a.nulls())) |
| 87 | + } |
| 88 | + _ => unreachable!(), |
| 89 | + } |
| 90 | + .ok_or_else(|| { |
| 91 | + exec_datafusion_err!( |
| 92 | + "Expected concrete {} array, got {}", |
| 93 | + self.data_type, |
| 94 | + v.data_type() |
| 95 | + ) |
| 96 | + })?; |
101 | 97 | // List values are all inline (len <= 12). Long input views cannot match |
102 | 98 | // because their encoded length (> 12) is part of the 128-bit key, so |
103 | 99 | // input lengths do not need to be checked. |
104 | | - self.inner.contains(&as_decimal128(array), negated) |
| 100 | + let decimal = as_decimal128(views, nulls); |
| 101 | + self.inner.contains(&decimal, negated) |
105 | 102 | } |
106 | 103 | } |
107 | 104 |
|
108 | | -fn instantiate_typed_filter<T: ByteViewType>( |
| 105 | +/// Returns a filter when every non-null byte view is inline. |
| 106 | +pub(super) fn instantiate_byte_view_filter( |
109 | 107 | in_array: &ArrayRef, |
110 | 108 | ) -> Result<Option<StaticFilterRef>> { |
111 | | - let array = downcast_byte_view::<T>(in_array.as_ref())?; |
112 | | - if !all_inline(array) { |
| 109 | + let views = match in_array.data_type() { |
| 110 | + DataType::Utf8View => in_array.as_string_view().views(), |
| 111 | + DataType::BinaryView => in_array.as_binary_view().views(), |
| 112 | + _ => return Ok(None), |
| 113 | + }; |
| 114 | + |
| 115 | + if !all_inline(views, in_array.nulls()) { |
113 | 116 | return Ok(None); |
114 | 117 | } |
115 | 118 |
|
116 | | - let primitive: ArrayRef = Arc::new(as_decimal128(array)); |
| 119 | + let primitive: ArrayRef = Arc::new(as_decimal128(views, in_array.nulls())); |
117 | 120 | let inner = instantiate_primitive_filter(&primitive)?.ok_or_else(|| { |
118 | 121 | internal_datafusion_err!( |
119 | 122 | "Byte view filter: no primitive filter for {}", |
120 | 123 | primitive.data_type() |
121 | 124 | ) |
122 | 125 | })?; |
123 | | - Ok(Some(Arc::new(ByteViewFilter::<T> { |
| 126 | + Ok(Some(Arc::new(ByteViewFilter { |
| 127 | + data_type: in_array.data_type().clone(), |
124 | 128 | inner, |
125 | | - _marker: PhantomData, |
126 | 129 | }))) |
127 | 130 | } |
128 | 131 |
|
129 | | -/// Returns a filter when every non-null byte view is inline. |
130 | | -pub(super) fn instantiate_byte_view_filter( |
131 | | - in_array: &ArrayRef, |
132 | | -) -> Result<Option<StaticFilterRef>> { |
133 | | - match in_array.data_type() { |
134 | | - DataType::Utf8View => instantiate_typed_filter::<StringViewType>(in_array), |
135 | | - DataType::BinaryView => instantiate_typed_filter::<BinaryViewType>(in_array), |
136 | | - _ => Ok(None), |
137 | | - } |
138 | | -} |
139 | | - |
140 | 132 | #[cfg(test)] |
141 | 133 | mod tests { |
142 | 134 | use super::*; |
@@ -221,6 +213,15 @@ mod tests { |
221 | 213 | let filter = instantiate_byte_view_filter(&binary)?.unwrap(); |
222 | 214 | let needles = BinaryViewArray::from(vec![b"a".as_slice(), b"z".as_slice()]); |
223 | 215 | assert_contains(&*filter, &needles, vec![Some(true), Some(false)])?; |
| 216 | + |
| 217 | + let mismatch_err = filter |
| 218 | + .contains(&StringViewArray::from(vec!["a"]), false) |
| 219 | + .unwrap_err(); |
| 220 | + assert!( |
| 221 | + mismatch_err |
| 222 | + .to_string() |
| 223 | + .contains("Expected concrete BinaryView array, got Utf8View") |
| 224 | + ); |
224 | 225 | Ok(()) |
225 | 226 | } |
226 | 227 | } |
0 commit comments