Skip to content

Commit cc1326a

Browse files
authored
fix: preserve dictionary-value nulls in scalar regex operators (apache#23966)
## Which issue does this PR close? - Closes apache#23963. ## Rationale for this change The scalar regex fast path for Dictionary arrays evaluates each dictionary value once and gathers the resulting booleans with `BooleanArray::take_iter`. That gather preserves null keys but drops null validity from the evaluated dictionary values. A key referencing a null dictionary value therefore becomes valid `false`, and negated regex operators can incorrectly turn it into `true`. This is distinct from apache#23722, which updates the array-pattern `regex_match_dyn` path. This PR fixes null propagation in `regex_match_dyn_scalar`. ## What changes are included in this PR? - Replace the validity-dropping dictionary gather with Arrow `take`, preserving both key validity and evaluated child validity. - Add a physical-expression regression test for `RegexMatch`, `RegexIMatch`, `RegexNotMatch`, and `RegexNotIMatch`. The test covers a null key and a key referencing a null dictionary value, and compares the Dictionary result with the equivalent cast-to-`Utf8` evaluation. ## Are these changes tested? Yes. - Focused regression: 1 passed. - `datafusion-physical-expr`: 1,649 passed, including the new regression. - Remaining workspace: 7,928 passed, 0 failed, 7 ignored; sqllogictest completed 499/499 files. - `datafusion` library: 461 passed. - `datafusion` core integration: 1,057 passed; four local memory-validation runners exceeded their host RSS thresholds and were isolated as environment-sensitive, unrelated to this patch. - `datafusion-cli`: 106 passed. - Formatting, workspace clippy with warnings denied, TOML formatting, license headers, changed-file typo checks, and `git diff --check` passed. ## Are there any user-facing changes? Dictionary-encoded string inputs now preserve SQL null semantics for scalar regex operators. There are no public API changes. --------- Signed-off-by: discord9 <[email protected]>
1 parent 802d8d2 commit cc1326a

2 files changed

Lines changed: 83 additions & 11 deletions

File tree

‎datafusion/physical-expr/src/expressions/binary.rs‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3353,6 +3353,78 @@ mod tests {
33533353
Ok(())
33543354
}
33553355

3356+
#[test]
3357+
fn regex_scalar_with_dictionary_nulls() -> Result<()> {
3358+
let dictionary_values = Arc::new(StringArray::from(vec![
3359+
Some("abc"),
3360+
None,
3361+
Some("ABC"),
3362+
Some("def"),
3363+
]));
3364+
let keys = UInt32Array::from(vec![Some(0), None, Some(1), Some(2), Some(3)]);
3365+
let dictionary =
3366+
Arc::new(DictionaryArray::try_new(keys, dictionary_values)?) as ArrayRef;
3367+
let utf8 = cast(&dictionary, &DataType::Utf8)?;
3368+
let pattern = ScalarValue::Utf8(Some("^abc$".to_string()));
3369+
let dictionary_schema = Arc::new(Schema::new(vec![Field::new(
3370+
"a",
3371+
dictionary.data_type().clone(),
3372+
true,
3373+
)]));
3374+
let utf8_schema =
3375+
Arc::new(Schema::new(vec![Field::new("a", DataType::Utf8, true)]));
3376+
3377+
let evaluate =
3378+
|schema: &SchemaRef, array: &ArrayRef, op: Operator| -> Result<ArrayRef> {
3379+
let expr = binary(col("a", schema)?, op, lit(pattern.clone()), schema)?;
3380+
let batch =
3381+
RecordBatch::try_new(Arc::clone(schema), vec![Arc::clone(array)])?;
3382+
Ok(expr
3383+
.evaluate(&batch)?
3384+
.into_array(batch.num_rows())
3385+
.expect("Failed to convert to array"))
3386+
};
3387+
3388+
for (op, expected) in [
3389+
(
3390+
Operator::RegexMatch,
3391+
BooleanArray::from(vec![
3392+
Some(true),
3393+
None,
3394+
None,
3395+
Some(false),
3396+
Some(false),
3397+
]),
3398+
),
3399+
(
3400+
Operator::RegexIMatch,
3401+
BooleanArray::from(vec![Some(true), None, None, Some(true), Some(false)]),
3402+
),
3403+
(
3404+
Operator::RegexNotMatch,
3405+
BooleanArray::from(vec![Some(false), None, None, Some(true), Some(true)]),
3406+
),
3407+
(
3408+
Operator::RegexNotIMatch,
3409+
BooleanArray::from(vec![
3410+
Some(false),
3411+
None,
3412+
None,
3413+
Some(false),
3414+
Some(true),
3415+
]),
3416+
),
3417+
] {
3418+
let dictionary_result = evaluate(&dictionary_schema, &dictionary, op)?;
3419+
let utf8_result = evaluate(&utf8_schema, &utf8, op)?;
3420+
3421+
assert_eq!(dictionary_result.as_ref(), &expected);
3422+
assert_eq!(&dictionary_result, &utf8_result);
3423+
}
3424+
3425+
Ok(())
3426+
}
3427+
33563428
#[test]
33573429
fn regex_mismatched_array_types_error() -> Result<()> {
33583430
// The analyzer coerces both operands of a regex operator to a common

‎datafusion/physical-expr/src/expressions/binary/kernels.rs‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,8 @@ pub(crate) fn regex_match_dyn_scalar(
270270
regexp_is_match_flag_scalar!(left, right, LargeStringArray, not_match, flag)
271271
}
272272
DataType::Dictionary(_, _) => {
273-
let values = left.as_any_dictionary().values();
273+
let dictionary = left.as_any_dictionary();
274+
let values = dictionary.values();
274275

275276
match values.data_type() {
276277
DataType::Utf8 => regexp_is_match_flag_scalar!(values, right, StringArray, not_match, flag),
@@ -280,16 +281,15 @@ pub(crate) fn regex_match_dyn_scalar(
280281
"Data type {} not supported as a dictionary value type for operation 'regex_match_dyn_scalar' on string array",
281282
other
282283
),
283-
}.map(
284-
// downcast_dictionary_array duplicates code per possible key type, so we aim to do all prep work before
285-
|evaluated_values| downcast_dictionary_array! {
286-
left => {
287-
let unpacked_dict = evaluated_values.take_iter(left.keys().iter().map(|opt| opt.map(|v| v as _))).collect::<BooleanArray>();
288-
Arc::new(unpacked_dict) as ArrayRef
289-
},
290-
_ => unreachable!(),
291-
}
292-
)
284+
}
285+
.and_then(|evaluated_values| {
286+
// Expand back to rows while preserving nulls from both keys and values.
287+
Ok(arrow::compute::take(
288+
evaluated_values.as_ref(),
289+
dictionary.keys(),
290+
None,
291+
)?)
292+
})
293293
}
294294
other => internal_err!(
295295
"Data type {} not supported for operation 'regex_match_dyn_scalar' on string array",

0 commit comments

Comments
 (0)