Skip to content

Commit fbe0593

Browse files
committed
perf: avoid hashing row-wise substr arguments
Decode wide-key dictionaries once and wrap the row results with identity keys, avoiding HashMap overhead when start and length combinations rarely repeat.
1 parent 7b8a1aa commit fbe0593

1 file changed

Lines changed: 11 additions & 59 deletions

File tree

‎datafusion/functions/src/unicode/substr.rs‎

Lines changed: 11 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,13 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use std::collections::HashMap;
1918
use std::sync::Arc;
2019

2120
use crate::strings::sub_view;
2221
use crate::utils::make_scalar_function;
2322
use arrow::array::{
2423
Array, ArrayRef, AsArray, DictionaryArray, GenericStringArray, Int64Array,
25-
OffsetSizeTrait, PrimitiveBuilder, StringArrayType, StringViewArray, UInt64Array,
24+
OffsetSizeTrait, PrimitiveArray, StringArrayType, StringViewArray,
2625
};
2726
use arrow::buffer::{NullBuffer, ScalarBuffer};
2827
use arrow::compute::take;
@@ -205,63 +204,16 @@ fn dictionary_substr<K: ArrowDictionaryKeyType>(
205204
dictionary: &DictionaryArray<K>,
206205
args: &[ArrayRef],
207206
) -> Result<ArrayRef> {
208-
let start_array = as_int64_array(&args[0])?;
209-
let count_array = args.get(1).map(|a| as_int64_array(a)).transpose()?;
210-
211-
let mut combinations = HashMap::new();
212-
let mut value_indices = Vec::new();
213-
let mut starts = Vec::new();
214-
let mut counts = count_array.map(|_| Vec::new());
215-
let mut keys = PrimitiveBuilder::<K>::with_capacity(dictionary.len());
216-
217-
for i in 0..dictionary.len() {
218-
let Some(value_index) = dictionary.key(i) else {
219-
keys.append_null();
220-
continue;
221-
};
222-
if dictionary.values().is_null(value_index)
223-
|| start_array.is_null(i)
224-
|| count_array.is_some_and(|a| a.is_null(i))
225-
{
226-
keys.append_null();
227-
continue;
228-
}
229-
230-
let start = start_array.value(i);
231-
let count = count_array.map(|a| a.value(i));
232-
let combination = (value_index, start, count);
233-
let next_key = combinations.len();
234-
let key = *combinations.entry(combination).or_insert_with(|| {
235-
value_indices.push(value_index as u64);
236-
starts.push(start);
237-
if let (Some(counts), Some(count)) = (&mut counts, count) {
238-
counts.push(count);
239-
}
240-
next_key
241-
});
242-
let key = K::Native::from_usize(key).ok_or_else(|| {
243-
datafusion_common::DataFusionError::Execution(format!(
244-
"substr result has too many dictionary values for key type {}",
245-
K::DATA_TYPE
246-
))
247-
})?;
248-
keys.append_value(key);
249-
}
250-
251-
let values = take(
252-
dictionary.values().as_ref(),
253-
&UInt64Array::from(value_indices),
254-
None,
255-
)?;
256-
let mut distinct_args = vec![values, Arc::new(Int64Array::from(starts)) as ArrayRef];
257-
if let Some(counts) = counts {
258-
distinct_args.push(Arc::new(Int64Array::from(counts)));
259-
}
260-
let values = substr(&distinct_args)?;
261-
Ok(Arc::new(DictionaryArray::<K>::try_new(
262-
keys.finish(),
263-
values,
264-
)?))
207+
// Only wide key types reach here, so one key per row always fits.
208+
let decoded = take(dictionary.values().as_ref(), dictionary.keys(), None)?;
209+
let mut row_args = vec![decoded];
210+
row_args.extend(args.iter().cloned());
211+
let values = substr(&row_args)?;
212+
let keys = (0..values.len())
213+
.map(|i| K::Native::from_usize(i).unwrap())
214+
.collect::<Vec<_>>();
215+
let keys = PrimitiveArray::<K>::new(keys.into(), values.nulls().cloned());
216+
Ok(Arc::new(DictionaryArray::<K>::try_new(keys, values)?))
265217
}
266218

267219
/// Convert the given `start` and `count` to valid byte indices within `input` string.

0 commit comments

Comments
 (0)