Skip to content

Commit 8f065c9

Browse files
airborne12claude
andcommitted
[refactor](nereids) Declare that MATCH and SEARCH depend on the column index
### What problem does this PR solve? Issue Number: None Related PR: #69292 Problem Summary: The previous commit stopped predicate inference from copying MATCH and SEARCH with a check for these classes in the OR collector of InferPredicateByReplace. That check named concrete expressions inside a generic rule, and it covered only an OR: a comparison such as (c1 MATCH_ANY 'apple') = TRUE still reached inference and was copied to an equal column. Declare the property on the expressions instead. Match, Search and SearchExpression implement the new trait DependsOnColumnIndex: their result depends on the inverted index of their column, not only on its value. Inference skips every input that contains such an expression, next to the existing NoneMovableFunction and non-deterministic checks, and the OR collector is back to the version on master. ### Release note None ### Check List (For Author) - Test: Unit Test and Regression test - InferPredicateByReplaceTest.testDoNotInferPredicatesThatDependOnColumnIndex: an OR of MATCH, (MATCH) = TRUE and an OR with SEARCH next to a = b infer nothing. The comparison case fails with the previous commit. InferPredicateByReplaceTest, InferPredicatesTest and UnequalPredicateInferTest pass. Checkstyle passes. - Regression test nereids_rules_p0/infer_predicate/infer_match_predicate passes unchanged on a local cluster. - Behavior changed: No, beyond the previous commit - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 888ef79 commit 8f065c9

6 files changed

Lines changed: 40 additions & 14 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,12 @@
2828
import org.apache.doris.nereids.trees.expressions.Expression;
2929
import org.apache.doris.nereids.trees.expressions.InPredicate;
3030
import org.apache.doris.nereids.trees.expressions.Like;
31-
import org.apache.doris.nereids.trees.expressions.Match;
3231
import org.apache.doris.nereids.trees.expressions.Not;
3332
import org.apache.doris.nereids.trees.expressions.Or;
34-
import org.apache.doris.nereids.trees.expressions.SearchExpression;
3533
import org.apache.doris.nereids.trees.expressions.Slot;
34+
import org.apache.doris.nereids.trees.expressions.functions.DependsOnColumnIndex;
3635
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
3736
import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction;
38-
import org.apache.doris.nereids.trees.expressions.functions.scalar.Search;
3937
import org.apache.doris.nereids.trees.expressions.literal.Literal;
4038
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
4139
import org.apache.doris.nereids.trees.plans.Plan;
@@ -134,11 +132,6 @@ public Void visitLike(Like like, Map<Expression, Set<Expression>> context) {
134132

135133
@Override
136134
public Void visitOr(Or or, Map<Expression, Set<Expression>> context) {
137-
// MATCH and SEARCH depend on the inverted index of their column, so they can differ on an equal
138-
// column, and a copy would also add to score().
139-
if (or.containsType(Match.class, Search.class, SearchExpression.class)) {
140-
return null;
141-
}
142135
for (Expression expr : getAllSubExpressions(or)) {
143136
context.computeIfAbsent(expr, k -> new LinkedHashSet<>()).add(or);
144137
}
@@ -244,8 +237,9 @@ public static Set<Expression> infer(Set<Expression> inputs) {
244237
}
245238
Map<Expression, Set<Expression>> exprPredicates = new HashMap<>();
246239
for (Expression input : inputs) {
247-
// Inference can evaluate a predicate on rows that never reach its original filter.
248-
if (input.anyMatch(expr -> expr instanceof NoneMovableFunction
240+
// Inference can evaluate a predicate on rows that never reach its original filter,
241+
// and it replaces the column of the predicate with an equal one.
242+
if (input.anyMatch(expr -> expr instanceof NoneMovableFunction || expr instanceof DependsOnColumnIndex
249243
|| !((ExpressionTrait) expr).isDeterministic())
250244
|| input.getInputSlots().size() != 1) {
251245
continue;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Match.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.doris.analysis.MatchPredicate.Operator;
2222
import org.apache.doris.nereids.exceptions.AnalysisException;
2323
import org.apache.doris.nereids.exceptions.UnboundException;
24+
import org.apache.doris.nereids.trees.expressions.functions.DependsOnColumnIndex;
2425
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
2526
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
2627
import org.apache.doris.nereids.types.BooleanType;
@@ -36,7 +37,8 @@
3637
/**
3738
* like expression: a MATCH 'hello'.
3839
*/
39-
public abstract class Match extends BinaryOperator implements PropagateNullable, PreferPushDownProject {
40+
public abstract class Match extends BinaryOperator
41+
implements PropagateNullable, PreferPushDownProject, DependsOnColumnIndex {
4042

4143
private final Optional<String> analyzer;
4244

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import org.apache.doris.analysis.SearchDslParser;
2121
import org.apache.doris.nereids.exceptions.UnboundException;
22+
import org.apache.doris.nereids.trees.expressions.functions.DependsOnColumnIndex;
2223
import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
2324
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
2425
import org.apache.doris.nereids.types.BooleanType;
@@ -31,7 +32,7 @@
3132
* SearchExpression represents a search query with bound slot references.
3233
* This is created by RewriteSearchToSlots rule from Search scalar function.
3334
*/
34-
public class SearchExpression extends Expression {
35+
public class SearchExpression extends Expression implements DependsOnColumnIndex {
3536
private final String dslString;
3637
private final SearchDslParser.QsPlan qsPlan;
3738

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.trees.expressions.functions;
19+
20+
/**
21+
* Expression trait. The expression evaluates a column through the inverted index of the column, so its result
22+
* depends on that index, e.g. its parser, and not only on the value of the column. A column with an equal value can
23+
* give another result, so the column must not be replaced by an equal one, e.g. by predicate inference.
24+
*/
25+
public interface DependsOnColumnIndex {
26+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Search.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.doris.catalog.FunctionSignature;
2222
import org.apache.doris.nereids.trees.expressions.Expression;
2323
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
24+
import org.apache.doris.nereids.trees.expressions.functions.DependsOnColumnIndex;
2425
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
2526
import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
2627
import org.apache.doris.nereids.types.BooleanType;
@@ -48,7 +49,7 @@
4849
* Example options: '{"default_field":"title","mode":"lucene","minimum_should_match":0}'
4950
*/
5051
public class Search extends ScalarFunction
51-
implements ExplicitlyCastableSignature, AlwaysNotNullable {
52+
implements ExplicitlyCastableSignature, AlwaysNotNullable, DependsOnColumnIndex {
5253

5354
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
5455
// Original signature: search(dsl_string)

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.doris.nereids.trees.expressions.functions.scalar.Search;
3636
import org.apache.doris.nereids.trees.expressions.functions.scalar.SignBit;
3737
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
38+
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
3839
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
3940
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
4041
import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
@@ -93,11 +94,12 @@ public void testDoNotInferNoneMovablePredicateInsideOr() {
9394
}
9495

9596
@Test
96-
public void testDoNotInferMatchOrSearchInsideOr() {
97+
public void testDoNotInferPredicatesThatDependOnColumnIndex() {
9798
SlotReference a = new SlotReference("a", StringType.INSTANCE);
9899
SlotReference b = new SlotReference("b", StringType.INSTANCE);
99100
for (Expression predicate : ImmutableList.of(
100101
new Or(new MatchAny(a, new StringLiteral("apple")), new MatchAny(a, new StringLiteral("pear"))),
102+
new EqualTo(new MatchAny(a, new StringLiteral("apple")), BooleanLiteral.TRUE),
101103
new Or(new Search(new StringLiteral("a:apple")), new EqualTo(a, new StringLiteral("pear"))))) {
102104
Set<Expression> inputs = new LinkedHashSet<>(ImmutableList.of(new EqualTo(a, b), predicate));
103105

0 commit comments

Comments
 (0)