|
18 | 18 | //! Defines physical expressions that can evaluated at runtime during query execution |
19 | 19 |
|
20 | 20 | use crate::hyperloglog::{HLL_HASH_STATE, HyperLogLog}; |
21 | | -use arrow::array::{Array, BinaryArray, StringViewArray}; |
| 21 | +use arrow::array::{Array, BinaryArray, ByteView, StringViewArray}; |
22 | 22 | use arrow::array::{ |
23 | 23 | GenericBinaryArray, GenericStringArray, OffsetSizeTrait, PrimitiveArray, |
24 | 24 | }; |
@@ -213,16 +213,43 @@ impl Accumulator for StringViewHLLAccumulator { |
213 | 213 | fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> { |
214 | 214 | let array: &StringViewArray = downcast_value!(values[0], StringViewArray); |
215 | 215 |
|
216 | | - // When all strings are stored inline in the StringView (≤ 12 bytes), |
217 | | - // hash the raw u128 view directly instead of materializing a &str. |
| 216 | + // When strings are stored inline in the StringView (≤ 12 bytes), hash |
| 217 | + // the raw u128 view directly. Whether a string is inline is determined |
| 218 | + // solely by its length, so the same value always takes the same code |
| 219 | + // path and gets the same hash. |
218 | 220 | if array.data_buffers().is_empty() { |
| 221 | + // All strings are inline — skip per-element length check |
219 | 222 | for (i, &view) in array.views().iter().enumerate() { |
220 | 223 | if !array.is_null(i) { |
221 | 224 | self.hll.add_hashed(HLL_HASH_STATE.hash_one(view)); |
222 | 225 | } |
223 | 226 | } |
224 | 227 | } else { |
225 | | - self.hll.extend(array.iter().flatten()); |
| 228 | + // At least some strings stored out-of-line |
| 229 | + let buffers = array.data_buffers(); |
| 230 | + for (i, &view) in array.views().iter().enumerate() { |
| 231 | + if array.is_null(i) { |
| 232 | + continue; |
| 233 | + } |
| 234 | + let view_len = view as u32; |
| 235 | + if view_len <= 12 { |
| 236 | + self.hll.add_hashed(HLL_HASH_STATE.hash_one(view)); |
| 237 | + } else { |
| 238 | + // For out-of-line strings, it is faster to hash the bytes |
| 239 | + // in the appropriate data buffer directly, rather than |
| 240 | + // constructing a &str to call self.hll.add() |
| 241 | + let bv = ByteView::from(view); |
| 242 | + let offset = bv.offset as usize; |
| 243 | + // SAFETY: view came from the array so buffer_index |
| 244 | + // and offset..offset+len are in bounds. |
| 245 | + let bytes = unsafe { |
| 246 | + buffers |
| 247 | + .get_unchecked(bv.buffer_index as usize) |
| 248 | + .get_unchecked(offset..offset + view_len as usize) |
| 249 | + }; |
| 250 | + self.hll.add_hashed(HLL_HASH_STATE.hash_one(bytes)); |
| 251 | + } |
| 252 | + } |
226 | 253 | } |
227 | 254 |
|
228 | 255 | Ok(()) |
|
0 commit comments