Skip to content

Commit f248f45

Browse files
authored
fix: handle empty patterns in regexp_instr (apache#24054)
## Which issue does this PR close? - Closes apache#22257. ## Rationale for this change PostgreSQL treats empty and other zero-width regular-expression matches as valid character-boundary matches. Before this change, `regexp_instr` returned 0 before compiling an empty pattern, excluded the terminal boundary from its start-position mapping, and returned 0 instead of propagating NULL when the input was NULL and the pattern was empty. ## What changes are included in this PR? - Let the regex implementation evaluate empty patterns instead of returning early. - Map the 1-based start position to UTF-8 byte boundaries including the terminal boundary. - Preserve NULL propagation when the input is NULL and the pattern is empty. - Add unit coverage for empty patterns and potentially zero-width patterns across Utf8, LargeUtf8, and Utf8View arrays. - Add SQL logic coverage for terminal zero-width matches and empty-pattern NULL input. ## Are these changes tested? Yes. The following checks pass on the current head: - `cargo fmt --all -- --check` - `cargo test -p datafusion-functions regexpinstr::tests::test_regexp_instr` - `cargo test --profile=ci --test sqllogictests -- regexp_instr.slt` - `cargo clippy -p datafusion-functions --tests -- -D warnings` The implementation commit was also verified with the repository-prescribed extended workspace suite and `cargo test --profile ci -p datafusion-cli`; the review follow-up changes add tests only. ## Are there any user-facing changes? Yes. `regexp_instr` now returns PostgreSQL-compatible positions for empty and potentially zero-width patterns, including at the terminal character boundary, and returns NULL for NULL input with an empty pattern. This does not change the public API.
1 parent b2ac10f commit f248f45

2 files changed

Lines changed: 89 additions & 13 deletions

File tree

‎datafusion/functions/src/regex/regexpinstr.rs‎

Lines changed: 77 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -302,20 +302,12 @@ where
302302
continue;
303303
}
304304
let regex = regex_array.value(i);
305-
if regex.is_empty() {
306-
result.append_value(0);
307-
continue;
308-
}
309305

310306
if values.is_null(i) {
311307
result.append_null();
312308
continue;
313309
}
314310
let value = values.value(i);
315-
if value.is_empty() {
316-
result.append_value(0);
317-
continue;
318-
}
319311

320312
let flags = match flags_array {
321313
Some(flags) if !flags.is_null(i) => Some(flags.value(i)),
@@ -376,7 +368,7 @@ impl<'a> RegexCache<'a> {
376368
/// Returns the 1-based character position of the `n`-th match of `pattern` in
377369
/// `value`, or 0 if there is no such match. The search begins at the 1-based
378370
/// character position `start`. A positive `subexpr` selects that capture group
379-
/// of the first match instead of the `n`-th match. `value` is non-empty.
371+
/// of the first match instead of the `n`-th match.
380372
fn get_index(
381373
value: &str,
382374
pattern: &Regex,
@@ -396,9 +388,16 @@ fn get_index(
396388
));
397389
}
398390

399-
// Byte offset of the `start`-th character. A `start` past the end of the
400-
// string leaves nothing to search, so no match is possible.
401-
let Some((byte_start_offset, _)) = value.char_indices().nth((start - 1) as usize)
391+
let Ok(start_index) = usize::try_from(start - 1) else {
392+
return Ok(0);
393+
};
394+
// Include the terminal byte boundary so an empty pattern can match after
395+
// the last character, including in an empty string.
396+
let Some(byte_start_offset) = value
397+
.char_indices()
398+
.map(|(offset, _)| offset)
399+
.chain(std::iter::once(value.len()))
400+
.nth(start_index)
402401
else {
403402
return Ok(0);
404403
};
@@ -451,6 +450,14 @@ mod tests {
451450
test_case_sensitive_regexp_instr_array_nth::<GenericStringArray<i32>>();
452451
test_case_sensitive_regexp_instr_array_nth::<GenericStringArray<i64>>();
453452
test_case_sensitive_regexp_instr_array_nth::<StringViewArray>();
453+
454+
test_case_sensitive_regexp_instr_empty_pattern::<GenericStringArray<i32>>();
455+
test_case_sensitive_regexp_instr_empty_pattern::<GenericStringArray<i64>>();
456+
test_case_sensitive_regexp_instr_empty_pattern::<StringViewArray>();
457+
458+
test_case_sensitive_regexp_instr_zero_width_pattern::<GenericStringArray<i32>>();
459+
test_case_sensitive_regexp_instr_zero_width_pattern::<GenericStringArray<i64>>();
460+
test_case_sensitive_regexp_instr_zero_width_pattern::<StringViewArray>();
454461
}
455462

456463
fn regexp_instr_with_scalar_values(args: &[ScalarValue]) -> Result<ColumnarValue> {
@@ -479,7 +486,7 @@ mod tests {
479486
fn test_case_sensitive_regexp_instr_nulls() {
480487
let v = "";
481488
let r = "";
482-
let expected = 0;
489+
let expected = 1;
483490
let regex_sv = ScalarValue::Utf8(Some(r.to_string()));
484491
let re = regexp_instr_with_scalar_values(&[v.to_string().into(), regex_sv]);
485492
// let res_exp = re.unwrap();
@@ -489,6 +496,29 @@ mod tests {
489496
}
490497
_ => panic!("Unexpected result"),
491498
}
499+
500+
for (value, regex) in [
501+
(
502+
ScalarValue::Utf8(None),
503+
ScalarValue::Utf8(Some(String::new())),
504+
),
505+
(
506+
ScalarValue::LargeUtf8(None),
507+
ScalarValue::LargeUtf8(Some(String::new())),
508+
),
509+
(
510+
ScalarValue::Utf8View(None),
511+
ScalarValue::Utf8View(Some(String::new())),
512+
),
513+
] {
514+
let re = regexp_instr_with_scalar_values(&[value, regex]);
515+
match re {
516+
Ok(ColumnarValue::Scalar(ScalarValue::Int64(v))) => {
517+
assert_eq!(v, None, "regexp_instr NULL scalar test failed");
518+
}
519+
_ => panic!("Unexpected result"),
520+
}
521+
}
492522
}
493523
fn test_case_sensitive_regexp_instr_scalar() {
494524
let values = [
@@ -800,4 +830,38 @@ mod tests {
800830
.unwrap();
801831
assert_eq!(re.as_ref(), &expected);
802832
}
833+
834+
fn test_case_sensitive_regexp_instr_empty_pattern<A>()
835+
where
836+
A: From<Vec<&'static str>> + Array + 'static,
837+
{
838+
let values = A::from(vec!["abc", "", "abc", "abc", "😀"]);
839+
let regex = A::from(vec!["", "", "", "", ""]);
840+
let start = Int64Array::from(vec![1, 1, 4, 5, 1]);
841+
let nth = Int64Array::from(vec![1, 1, 1, 1, 2]);
842+
let expected = Int64Array::from(vec![1, 1, 4, 0, 2]);
843+
844+
let re = regexp_instr_func(&[
845+
Arc::new(values),
846+
Arc::new(regex),
847+
Arc::new(start),
848+
Arc::new(nth),
849+
])
850+
.unwrap();
851+
assert_eq!(re.as_ref(), &expected);
852+
}
853+
854+
fn test_case_sensitive_regexp_instr_zero_width_pattern<A>()
855+
where
856+
A: From<Vec<&'static str>> + Array + 'static,
857+
{
858+
let values = A::from(vec!["abc"]);
859+
let regex = A::from(vec!["x*"]);
860+
let start = Int64Array::from(vec![4]);
861+
let expected = Int64Array::from(vec![4]);
862+
863+
let re = regexp_instr_func(&[Arc::new(values), Arc::new(regex), Arc::new(start)])
864+
.unwrap();
865+
assert_eq!(re.as_ref(), &expected);
866+
}
803867
}

‎datafusion/sqllogictest/test_files/regexp/regexp_instr.slt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,18 @@ SELECT regexp_instr('123123123123123', '(12)3');
2323
----
2424
1
2525

26+
query IIIIIII
27+
SELECT
28+
regexp_instr('abc', ''),
29+
regexp_instr('', ''),
30+
regexp_instr('abc', '', 4),
31+
regexp_instr('abc', '', 5),
32+
regexp_instr('😀', '', 1, 2),
33+
regexp_instr('abc', 'x*', 4),
34+
regexp_instr(NULL, '');
35+
----
36+
1 1 4 0 2 4 NULL
37+
2638
query I
2739
SELECT regexp_instr('123123123123', '123', 1);
2840
----

0 commit comments

Comments
 (0)