Skip to content

Commit ef1f173

Browse files
committed
Per-string short-string opt, access data buffer directly
1 parent cccd362 commit ef1f173

1 file changed

Lines changed: 31 additions & 4 deletions

File tree

‎datafusion/functions-aggregate/src/approx_distinct.rs‎

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
//! Defines physical expressions that can evaluated at runtime during query execution
1919
2020
use crate::hyperloglog::{HLL_HASH_STATE, HyperLogLog};
21-
use arrow::array::{Array, BinaryArray, StringViewArray};
21+
use arrow::array::{Array, BinaryArray, ByteView, StringViewArray};
2222
use arrow::array::{
2323
GenericBinaryArray, GenericStringArray, OffsetSizeTrait, PrimitiveArray,
2424
};
@@ -213,16 +213,43 @@ impl Accumulator for StringViewHLLAccumulator {
213213
fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
214214
let array: &StringViewArray = downcast_value!(values[0], StringViewArray);
215215

216-
// When all strings are stored inline in the StringView (≤ 12 bytes),
217-
// hash the raw u128 view directly instead of materializing a &str.
216+
// When strings are stored inline in the StringView (≤ 12 bytes), hash
217+
// the raw u128 view directly. Whether a string is inline is determined
218+
// solely by its length, so the same value always takes the same code
219+
// path and gets the same hash.
218220
if array.data_buffers().is_empty() {
221+
// All strings are inline — skip per-element length check
219222
for (i, &view) in array.views().iter().enumerate() {
220223
if !array.is_null(i) {
221224
self.hll.add_hashed(HLL_HASH_STATE.hash_one(view));
222225
}
223226
}
224227
} else {
225-
self.hll.extend(array.iter().flatten());
228+
// At least some strings stored out-of-line
229+
let buffers = array.data_buffers();
230+
for (i, &view) in array.views().iter().enumerate() {
231+
if array.is_null(i) {
232+
continue;
233+
}
234+
let view_len = view as u32;
235+
if view_len <= 12 {
236+
self.hll.add_hashed(HLL_HASH_STATE.hash_one(view));
237+
} else {
238+
// For out-of-line strings, it is faster to hash the bytes
239+
// in the appropriate data buffer directly, rather than
240+
// constructing a &str to call self.hll.add()
241+
let bv = ByteView::from(view);
242+
let offset = bv.offset as usize;
243+
// SAFETY: view came from the array so buffer_index
244+
// and offset..offset+len are in bounds.
245+
let bytes = unsafe {
246+
buffers
247+
.get_unchecked(bv.buffer_index as usize)
248+
.get_unchecked(offset..offset + view_len as usize)
249+
};
250+
self.hll.add_hashed(HLL_HASH_STATE.hash_one(bytes));
251+
}
252+
}
226253
}
227254

228255
Ok(())

0 commit comments

Comments
 (0)