Skip to content

Commit b33ccf6

Browse files
committed
refactor: share hash buffer management
1 parent b1b6240 commit b33ccf6

2 files changed

Lines changed: 51 additions & 69 deletions

File tree

‎datafusion/common/src/hash_utils.rs‎

Lines changed: 48 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,37 @@ thread_local! {
114114
static HASH_BUFFER: RefCell<Vec<u64>> = const { RefCell::new(Vec::new()) };
115115
}
116116

117+
#[inline]
118+
fn with_hash_buffer<R>(
119+
required_size: usize,
120+
operation: &'static str,
121+
f: impl FnOnce(&mut [u64]) -> Result<R>,
122+
) -> Result<R> {
123+
HASH_BUFFER.try_with(|cell| {
124+
let mut buffer = cell.try_borrow_mut().map_err(|_| {
125+
_internal_datafusion_err!(
126+
"{operation} cannot be called reentrantly on the same thread"
127+
)
128+
})?;
129+
130+
buffer.clear();
131+
buffer.resize(required_size, 0);
132+
133+
let result = f(&mut buffer[..required_size])?;
134+
135+
if buffer.capacity() > MAX_BUFFER_SIZE {
136+
buffer.truncate(MAX_BUFFER_SIZE);
137+
buffer.shrink_to_fit();
138+
}
139+
140+
Ok(result)
141+
}).map_err(|_| {
142+
_internal_datafusion_err!(
143+
"{operation} cannot access thread-local storage during or after thread destruction"
144+
)
145+
})?
146+
}
147+
117148
/// Creates hashes for the given arrays using a thread-local buffer, then calls the provided callback
118149
/// with an immutable reference to the computed hashes.
119150
///
@@ -164,28 +195,13 @@ where
164195
None => return _internal_err!("with_hashes requires at least one array"),
165196
};
166197

167-
HASH_BUFFER.try_with(|cell| {
168-
let mut buffer = cell.try_borrow_mut()
169-
.map_err(|_| _internal_datafusion_err!("with_hashes cannot be called reentrantly on the same thread"))?;
170-
171-
// Ensure buffer has sufficient length, clearing old values
172-
buffer.clear();
173-
buffer.resize(required_size, 0);
174-
198+
with_hash_buffer(required_size, "with_hashes", |buffer| {
175199
// Create hashes in the buffer - this consumes the iterator
176-
create_hashes(iter, random_state, &mut buffer[..required_size])?;
200+
create_hashes(iter, random_state, buffer)?;
177201

178202
// Execute the callback with an immutable slice
179-
let result = callback(&buffer[..required_size])?;
180-
181-
// Cleanup: truncate if buffer grew too large
182-
if buffer.capacity() > MAX_BUFFER_SIZE {
183-
buffer.truncate(MAX_BUFFER_SIZE);
184-
buffer.shrink_to_fit();
185-
}
186-
187-
Ok(result)
188-
}).map_err(|_| _internal_datafusion_err!("with_hashes cannot access thread-local storage during or after thread destruction"))?
203+
callback(buffer)
204+
})
189205
}
190206

191207
/// Creates hashes for the given arrays using a thread-local buffer and a custom
@@ -204,7 +220,19 @@ where
204220
F: FnOnce(&[u64]) -> Result<R>,
205221
S: BuildHasher,
206222
{
207-
build_hasher::with_hashes_with_hasher(arrays, hash_builder, callback)
223+
let mut iter = arrays.into_iter().peekable();
224+
225+
let required_size = match iter.peek() {
226+
Some(arr) => arr.as_dyn_array().len(),
227+
None => {
228+
return _internal_err!("with_hashes_with_hasher requires at least one array");
229+
}
230+
};
231+
232+
with_hash_buffer(required_size, "with_hashes_with_hasher", |buffer| {
233+
build_hasher::create_hashes_with_hasher(iter, hash_builder, buffer)?;
234+
callback(buffer)
235+
})
208236
}
209237

210238
#[cfg(not(feature = "force_hash_collisions"))]

‎datafusion/common/src/hash_utils/build_hasher.rs‎

Lines changed: 3 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use super::{AsDynArray, HASH_BUFFER, MAX_BUFFER_SIZE};
18+
use super::AsDynArray;
1919
#[cfg(not(feature = "force_hash_collisions"))]
2020
use super::{
2121
ChildHashing, combine_hashes, hash_dictionary_with_child_hashing,
@@ -29,8 +29,9 @@ use crate::cast::{
2929
as_list_array, as_list_view_array, as_map_array, as_string_array,
3030
as_string_view_array, as_struct_array, as_union_array,
3131
};
32+
#[cfg(not(feature = "force_hash_collisions"))]
33+
use crate::error::_internal_err;
3234
use crate::error::Result;
33-
use crate::error::{_internal_datafusion_err, _internal_err};
3435
#[cfg(feature = "force_hash_collisions")]
3536
use arrow::array::Array;
3637
#[cfg(not(feature = "force_hash_collisions"))]
@@ -43,53 +44,6 @@ use arrow::datatypes::*;
4344
use arrow::{downcast_dictionary_array, downcast_primitive_array};
4445
use std::hash::BuildHasher;
4546

46-
pub(super) fn with_hashes_with_hasher<I, T, F, R, S>(
47-
arrays: I,
48-
hash_builder: &S,
49-
callback: F,
50-
) -> Result<R>
51-
where
52-
I: IntoIterator<Item = T>,
53-
T: AsDynArray,
54-
F: FnOnce(&[u64]) -> Result<R>,
55-
S: BuildHasher,
56-
{
57-
let mut iter = arrays.into_iter().peekable();
58-
59-
let required_size = match iter.peek() {
60-
Some(arr) => arr.as_dyn_array().len(),
61-
None => {
62-
return _internal_err!("with_hashes_with_hasher requires at least one array");
63-
}
64-
};
65-
66-
HASH_BUFFER.try_with(|cell| {
67-
let mut buffer = cell.try_borrow_mut().map_err(|_| {
68-
_internal_datafusion_err!(
69-
"with_hashes_with_hasher cannot be called reentrantly on the same thread"
70-
)
71-
})?;
72-
73-
buffer.clear();
74-
buffer.resize(required_size, 0);
75-
76-
create_hashes_with_hasher_impl(iter, hash_builder, &mut buffer[..required_size])?;
77-
78-
let result = callback(&buffer[..required_size])?;
79-
80-
if buffer.capacity() > MAX_BUFFER_SIZE {
81-
buffer.truncate(MAX_BUFFER_SIZE);
82-
buffer.shrink_to_fit();
83-
}
84-
85-
Ok(result)
86-
}).map_err(|_| {
87-
_internal_datafusion_err!(
88-
"with_hashes_with_hasher cannot access thread-local storage during or after thread destruction"
89-
)
90-
})?
91-
}
92-
9347
pub(super) fn create_hashes_with_hasher<'a, I, T, S>(
9448
arrays: I,
9549
hash_builder: &S,

0 commit comments

Comments
 (0)