Skip to content

Commit 3a90a9f

Browse files
committed
fix: preserve substr dictionaries without casts
1 parent 6262d66 commit 3a90a9f

2 files changed

Lines changed: 78 additions & 9 deletions

File tree

‎datafusion/functions/src/unicode/substr.rs‎

Lines changed: 75 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,19 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18+
use std::collections::HashMap;
1819
use std::sync::Arc;
1920

2021
use crate::strings::sub_view;
2122
use crate::utils::make_scalar_function;
2223
use arrow::array::{
23-
Array, ArrayRef, AsArray, GenericStringArray, Int64Array, OffsetSizeTrait,
24-
StringArrayType, StringViewArray,
24+
Array, ArrayRef, AsArray, DictionaryArray, GenericStringArray, Int64Array,
25+
OffsetSizeTrait, PrimitiveBuilder, StringArrayType, StringViewArray, UInt64Array,
2526
};
2627
use arrow::buffer::{NullBuffer, ScalarBuffer};
27-
use arrow::compute::kernels::cast::cast;
28-
use arrow::datatypes::DataType;
28+
use arrow::compute::take;
29+
use arrow::datatypes::{ArrowDictionaryKeyType, ArrowNativeType, DataType};
30+
use arrow::downcast_dictionary_array;
2931
use datafusion_common::cast::as_int64_array;
3032
use datafusion_common::types::{
3133
NativeType, logical_int32, logical_int64, logical_string,
@@ -162,10 +164,12 @@ fn substr(args: &[ArrayRef]) -> Result<ArrayRef> {
162164
let string_array = args[0].as_string_view();
163165
string_view_substr(string_array, &args[1..])
164166
}
165-
DataType::Dictionary(_, value_type) => {
166-
let mut flat_args = args.to_vec();
167-
flat_args[0] = cast(&args[0], value_type)?;
168-
Ok(cast(&substr(&flat_args)?, args[0].data_type())?)
167+
DataType::Dictionary(_, _) => {
168+
let array = &args[0];
169+
downcast_dictionary_array! {
170+
array => dictionary_substr(array, &args[1..]),
171+
_ => unreachable!("substr matched a dictionary array"),
172+
}
169173
}
170174
other => exec_err!(
171175
"Unsupported data type {other:?} for function substr,\
@@ -174,6 +178,69 @@ fn substr(args: &[ArrayRef]) -> Result<ArrayRef> {
174178
}
175179
}
176180

181+
fn dictionary_substr<K: ArrowDictionaryKeyType>(
182+
dictionary: &DictionaryArray<K>,
183+
args: &[ArrayRef],
184+
) -> Result<ArrayRef> {
185+
let start_array = as_int64_array(&args[0])?;
186+
let count_array = args.get(1).map(|a| as_int64_array(a)).transpose()?;
187+
188+
let mut combinations = HashMap::new();
189+
let mut value_indices = Vec::new();
190+
let mut starts = Vec::new();
191+
let mut counts = count_array.map(|_| Vec::new());
192+
let mut keys = PrimitiveBuilder::<K>::with_capacity(dictionary.len());
193+
194+
for i in 0..dictionary.len() {
195+
let Some(value_index) = dictionary.key(i) else {
196+
keys.append_null();
197+
continue;
198+
};
199+
if dictionary.values().is_null(value_index)
200+
|| start_array.is_null(i)
201+
|| count_array.is_some_and(|a| a.is_null(i))
202+
{
203+
keys.append_null();
204+
continue;
205+
}
206+
207+
let start = start_array.value(i);
208+
let count = count_array.map(|a| a.value(i));
209+
let combination = (value_index, start, count);
210+
let next_key = combinations.len();
211+
let key = *combinations.entry(combination).or_insert_with(|| {
212+
value_indices.push(value_index as u64);
213+
starts.push(start);
214+
if let (Some(counts), Some(count)) = (&mut counts, count) {
215+
counts.push(count);
216+
}
217+
next_key
218+
});
219+
let key = K::Native::from_usize(key).ok_or_else(|| {
220+
datafusion_common::DataFusionError::Execution(format!(
221+
"substr result has too many dictionary values for key type {}",
222+
K::DATA_TYPE
223+
))
224+
})?;
225+
keys.append_value(key);
226+
}
227+
228+
let values = take(
229+
dictionary.values().as_ref(),
230+
&UInt64Array::from(value_indices),
231+
None,
232+
)?;
233+
let mut distinct_args = vec![values, Arc::new(Int64Array::from(starts)) as ArrayRef];
234+
if let Some(counts) = counts {
235+
distinct_args.push(Arc::new(Int64Array::from(counts)));
236+
}
237+
let values = substr(&distinct_args)?;
238+
Ok(Arc::new(DictionaryArray::<K>::try_new(
239+
keys.finish(),
240+
values,
241+
)?))
242+
}
243+
177244
/// Convert the given `start` and `count` to valid byte indices within `input` string.
178245
///
179246
/// Input `start` and `count` are equivalent to PostgreSQL's `substr(s, start, count)`.

‎datafusion/sqllogictest/test_files/string/dictionary_utf8.slt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ create table test_substr_arguments (
8989
statement ok
9090
insert into test_substr_arguments values
9191
('hello', 1, 2),
92+
('hello', 2, 1),
9293
('world', NULL, 3),
9394
('arrow', 2, NULL);
9495

@@ -99,10 +100,11 @@ SELECT
99100
substr(arrow_cast(c, 'Dictionary(Int32, Utf8)'), start_pos),
100101
substr(arrow_cast(c, 'Dictionary(Int32, Utf8)'), 1, length)
101102
FROM test_substr_arguments
102-
ORDER BY c;
103+
ORDER BY c, start_pos;
103104
----
104105
rrow rrow NULL
105106
hello hello he
107+
ello ello h
106108
NULL NULL wor
107109

108110
query TT

0 commit comments

Comments
 (0)