Skip to content

Commit 019dde2

Browse files
asubiottoampagent
andcommitted
perf: preserve run-end encoded UTF-8 LIKE operands
Keep an outer RunEndEncoded over Utf8 or dictionary-encoded Utf8 when the coerced pattern is Utf8. Preserve ordinary casts for other layouts and SIMILAR TO. Requires Arrow LIKE kernel support from apache/arrow-rs#12154. Amp-Thread-ID: https://ampcode.com/threads/T-01a1c09d-06c3-76af-8d8a-d3c4ebe25726 Co-authored-by: Amp <[email protected]>
1 parent 0a363f6 commit 019dde2

3 files changed

Lines changed: 158 additions & 8 deletions

File tree

‎datafusion/optimizer/src/analyzer/type_coercion.rs‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -661,16 +661,16 @@ impl<'a> TypeCoercionRewriter<'a> {
661661

662662
/// Coerce the value and pattern expressions of a string pattern matching
663663
/// expression (`LIKE`, `ILIKE` or `SIMILAR TO`) to a common type using
664-
/// the provided coercion rules. `LIKE` can preserve a dictionary-encoded
665-
/// value expression, while regex array kernels require both operands to
666-
/// have the same physical string type.
664+
/// the provided coercion rules. `LIKE` can preserve dictionary-encoded or
665+
/// run-end encoded Utf8 values, while regex array kernels require both
666+
/// operands to have the same physical string type.
667667
fn coerce_like_operands(
668668
&self,
669669
expr: Expr,
670670
pattern: Expr,
671671
coercion: fn(&DataType, &DataType) -> Option<DataType>,
672672
op_name: &str,
673-
preserve_utf8_dictionary: bool,
673+
preserve_utf8_encoding: bool,
674674
) -> Result<(Box<Expr>, Box<Expr>)> {
675675
let left_type = expr.get_type(self.schema)?;
676676
let right_type = pattern.get_type(self.schema)?;
@@ -681,7 +681,18 @@ impl<'a> TypeCoercionRewriter<'a> {
681681
})?;
682682
let expr = match left_type {
683683
DataType::Dictionary(_, inner)
684-
if preserve_utf8_dictionary && *inner == DataType::Utf8 =>
684+
if preserve_utf8_encoding && *inner == DataType::Utf8 =>
685+
{
686+
Box::new(expr)
687+
}
688+
DataType::RunEndEncoded(_, values)
689+
if preserve_utf8_encoding
690+
&& coerced_type == DataType::Utf8
691+
&& match values.data_type() {
692+
DataType::Utf8 => true,
693+
DataType::Dictionary(_, inner) => **inner == DataType::Utf8,
694+
_ => false,
695+
} =>
685696
{
686697
Box::new(expr)
687698
}

‎datafusion/physical-expr/src/expressions/like.rs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -199,10 +199,17 @@ impl LikeExpr {
199199
}
200200
}
201201

202-
/// used for optimize Dictionary like
203-
fn can_like_type(from_type: &DataType) -> bool {
202+
/// Whether LIKE can use the encoded value without a cast.
203+
fn can_like_type(from_type: &DataType, pattern_type: &DataType) -> bool {
204204
match from_type {
205205
DataType::Dictionary(_, inner_type_from) => **inner_type_from == DataType::Utf8,
206+
DataType::RunEndEncoded(_, values) if *pattern_type == DataType::Utf8 => {
207+
match values.data_type() {
208+
DataType::Utf8 => true,
209+
DataType::Dictionary(_, inner) => **inner == DataType::Utf8,
210+
_ => false,
211+
}
212+
}
206213
_ => false,
207214
}
208215
}
@@ -218,7 +225,7 @@ pub fn like(
218225
let expr_type = &expr.data_type(input_schema)?;
219226
let pattern_type = &pattern.data_type(input_schema)?;
220227
assert_or_internal_err!(
221-
expr_type.eq(pattern_type) || can_like_type(expr_type),
228+
expr_type.eq(pattern_type) || can_like_type(expr_type, pattern_type),
222229
"The type of {expr_type} AND {pattern_type} of like physical should be same"
223230
);
224231
Ok(Arc::new(LikeExpr::new(

‎datafusion/sqllogictest/test_files/run_end_encoded.slt‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,3 +187,135 @@ sensor_B Utf8
187187

188188
statement ok
189189
DROP TABLE ree_readings;
190+
191+
# LIKE keeps an outer REE layer over Utf8 or dictionary-encoded Utf8.
192+
# A dictionary around REE or another dictionary still requires coercion.
193+
statement ok
194+
CREATE TABLE ree_strings AS
195+
SELECT id,
196+
arrow_cast(value, 'RunEndEncoded("run_ends": non-null Int32, "values": Dictionary(UInt32, Utf8))') AS value,
197+
arrow_cast(value, 'RunEndEncoded("run_ends": non-null Int32, "values": Utf8)') AS plain_value,
198+
arrow_cast(arrow_cast(value, 'Dictionary(UInt32, Utf8)'),
199+
'Dictionary(UInt32, RunEndEncoded("run_ends": non-null Int32, "values": Utf8))') AS dict_run,
200+
arrow_cast(arrow_cast(value, 'Dictionary(UInt32, Utf8)'),
201+
'Dictionary(UInt32, Dictionary(UInt32, Utf8))') AS dict_dict,
202+
pattern
203+
FROM (VALUES
204+
(1, 'Ab_c', '%b\_%'),
205+
(2, 'Ab_c', 'a%'),
206+
(3, 'Äpfel', 'ä%'),
207+
(4, NULL, '%'),
208+
(5, 'Other', NULL)
209+
) AS t(id, value, pattern);
210+
211+
query IBBBBBBBB
212+
SELECT id, value LIKE '%b\_%', value NOT LIKE '%b\_%',
213+
value ILIKE 'ä%', value NOT ILIKE 'ä%',
214+
plain_value LIKE '%b\_%', plain_value NOT LIKE '%b\_%',
215+
plain_value ILIKE 'ä%', plain_value NOT ILIKE 'ä%'
216+
FROM ree_strings ORDER BY id;
217+
----
218+
1 true false false true true false false true
219+
2 true false false true true false false true
220+
3 false true true false false true true false
221+
4 NULL NULL NULL NULL NULL NULL NULL NULL
222+
5 false true false true false true false true
223+
224+
query TT
225+
EXPLAIN SELECT id FROM ree_strings WHERE value ILIKE '%b\_%';
226+
----
227+
logical_plan
228+
01)Projection: ree_strings.id
229+
02)--Filter: ree_strings.value ILIKE Utf8("%b\_%")
230+
03)----TableScan: ree_strings projection=[id, value]
231+
physical_plan
232+
01)FilterExec: value@1 ILIKE %b\_%, projection=[id@0]
233+
02)--DataSourceExec: partitions=1, partition_sizes=[1]
234+
235+
query I
236+
SELECT id FROM ree_strings WHERE value ILIKE '%b\_%' ORDER BY id;
237+
----
238+
1
239+
2
240+
241+
query IBBBB
242+
SELECT id, value ILIKE pattern, plain_value ILIKE pattern,
243+
dict_run ILIKE pattern, dict_dict ILIKE pattern
244+
FROM ree_strings ORDER BY id;
245+
----
246+
1 true true true true
247+
2 true true true true
248+
3 true true true true
249+
4 NULL NULL NULL NULL
250+
5 NULL NULL NULL NULL
251+
252+
query TT
253+
EXPLAIN SELECT value ILIKE pattern, plain_value ILIKE pattern,
254+
dict_run ILIKE pattern, dict_dict ILIKE pattern
255+
FROM ree_strings;
256+
----
257+
logical_plan
258+
01)Projection: ree_strings.value ILIKE ree_strings.pattern, ree_strings.plain_value ILIKE ree_strings.pattern, CAST(ree_strings.dict_run AS Utf8) ILIKE ree_strings.pattern, CAST(ree_strings.dict_dict AS Utf8) ILIKE ree_strings.pattern
259+
02)--TableScan: ree_strings projection=[value, plain_value, dict_run, dict_dict, pattern]
260+
physical_plan
261+
01)ProjectionExec: expr=[value@0 ILIKE pattern@4 as ree_strings.value ILIKE ree_strings.pattern, plain_value@1 ILIKE pattern@4 as ree_strings.plain_value ILIKE ree_strings.pattern, CAST(dict_run@2 AS Utf8) ILIKE pattern@4 as ree_strings.dict_run ILIKE ree_strings.pattern, CAST(dict_dict@3 AS Utf8) ILIKE pattern@4 as ree_strings.dict_dict ILIKE ree_strings.pattern]
262+
02)--DataSourceExec: partitions=1, partition_sizes=[1]
263+
264+
# Nested dictionaries keep their existing coercion for all four operators.
265+
query IBBBBBBBB
266+
SELECT id, dict_run LIKE '%b\_%', dict_run NOT LIKE '%b\_%',
267+
dict_run ILIKE 'ä%', dict_run NOT ILIKE 'ä%',
268+
dict_dict LIKE '%b\_%', dict_dict NOT LIKE '%b\_%',
269+
dict_dict ILIKE 'ä%', dict_dict NOT ILIKE 'ä%'
270+
FROM ree_strings ORDER BY id;
271+
----
272+
1 true false false true true false false true
273+
2 true false false true true false false true
274+
3 false true true false false true true false
275+
4 NULL NULL NULL NULL NULL NULL NULL NULL
276+
5 false true false true false true false true
277+
278+
# A different pattern layout must still coerce the REE operand's values.
279+
query IBB
280+
SELECT id, value ILIKE arrow_cast(pattern, 'LargeUtf8'),
281+
value ILIKE arrow_cast(pattern, 'Utf8View')
282+
FROM ree_strings ORDER BY id;
283+
----
284+
1 true true
285+
2 true true
286+
3 true true
287+
4 NULL NULL
288+
5 NULL NULL
289+
290+
query TT
291+
EXPLAIN SELECT value ILIKE arrow_cast('a%', 'LargeUtf8') FROM ree_strings;
292+
----
293+
logical_plan
294+
01)Projection: CAST(ree_strings.value AS LargeUtf8) ILIKE LargeUtf8("a%") AS ree_strings.value ILIKE arrow_cast(Utf8("a%"),Utf8("LargeUtf8"))
295+
02)--TableScan: ree_strings projection=[value]
296+
physical_plan
297+
01)ProjectionExec: expr=[CAST(value@0 AS LargeUtf8) ILIKE a% as ree_strings.value ILIKE arrow_cast(Utf8("a%"),Utf8("LargeUtf8"))]
298+
02)--DataSourceExec: partitions=1, partition_sizes=[1]
299+
300+
# SIMILAR TO uses the regex kernel, which still needs decoded strings.
301+
query IB
302+
SELECT id, value SIMILAR TO 'Ab%' FROM ree_strings ORDER BY id;
303+
----
304+
1 true
305+
2 true
306+
3 false
307+
4 NULL
308+
5 false
309+
310+
query TT
311+
EXPLAIN SELECT value SIMILAR TO 'Ab%' FROM ree_strings;
312+
----
313+
logical_plan
314+
01)Projection: CAST(ree_strings.value AS Utf8) SIMILAR TO Utf8("Ab%")
315+
02)--TableScan: ree_strings projection=[value]
316+
physical_plan
317+
01)ProjectionExec: expr=[CAST(value@0 AS Utf8) ~ ^(?:Ab(?s:.*))$ as ree_strings.value SIMILAR TO Utf8("Ab%")]
318+
02)--DataSourceExec: partitions=1, partition_sizes=[1]
319+
320+
statement ok
321+
DROP TABLE ree_strings;

0 commit comments

Comments
 (0)