Skip to content

Commit 888ef79

Browse files
airborne12claude
andcommitted
[fix](nereids) Do not infer MATCH and SEARCH predicates across column equalities
### What problem does this PR solve? Issue Number: None Related PR: #69292 Problem Summary: Predicate inference copies an OR on one column to every column that is equal to it. When the OR contains MATCH, the copy changes the result: CREATE TABLE t (id INT, c1 TEXT, c2 TEXT, INDEX i1 (c1) USING INVERTED PROPERTIES ("parser" = "english"), INDEX i2 (c2) USING INVERTED) ...; INSERT INTO t VALUES (1, 'apple red', 'apple red'), (2, 'banana', 'banana'); SELECT id FROM t WHERE c1 = c2 AND (c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear'); -- no rows; row 1 is expected InferPredicateByReplace added (c2 MATCH_ANY 'apple' OR c2 MATCH_ANY 'pear'). A MATCH depends on the inverted index of its column: c2 has no parser, so a MATCH on c2 only matches a whole value, and the copy dropped row 1. When both columns have the same index, the copy is scored too. With 'apple red' and 'apple green' in both columns, score() of the query above ordered by score() with a LIMIT doubles from 0.4700036 to 0.9400072, and score() > 0.6 returns both rows, although their score is below the minimum. InferPredicateByReplace no longer infers from an OR that contains MATCH or SEARCH; it never inferred from a MATCH outside an OR. SEARCH is left out for the same reasons. ### Release note Fix wrong results and scores when a query has an equality between two columns and an OR of MATCH or SEARCH predicates on one of them. ### Check List (For Author) - Test: Unit Test and Regression test - InferPredicateByReplaceTest.testDoNotInferMatchOrSearchInsideOr: an OR of MATCH and an OR with SEARCH next to a = b infer nothing. It fails without this change, where the MATCH OR is copied to b. InferPredicateByReplaceTest, InferPredicatesTest and UnequalPredicateInferTest pass (104 tests). Checkstyle passes. - Regression test: nereids_rules_p0/infer_predicate/infer_match_predicate runs the queries above, with its .out generated by the framework on an FE built from this branch and a BE built from 41a58af plus the SegmentIterator change of #69292 (this change is FE only). On an FE built from master, the same queries return no rows, 0.9400072 and two rows. - Behavior changed: Yes. Predicate inference no longer copies an OR that contains MATCH or SEARCH to an equal column. - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 8851108 commit 888ef79

4 files changed

Lines changed: 97 additions & 0 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,14 @@
2828
import org.apache.doris.nereids.trees.expressions.Expression;
2929
import org.apache.doris.nereids.trees.expressions.InPredicate;
3030
import org.apache.doris.nereids.trees.expressions.Like;
31+
import org.apache.doris.nereids.trees.expressions.Match;
3132
import org.apache.doris.nereids.trees.expressions.Not;
3233
import org.apache.doris.nereids.trees.expressions.Or;
34+
import org.apache.doris.nereids.trees.expressions.SearchExpression;
3335
import org.apache.doris.nereids.trees.expressions.Slot;
3436
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
3537
import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction;
38+
import org.apache.doris.nereids.trees.expressions.functions.scalar.Search;
3639
import org.apache.doris.nereids.trees.expressions.literal.Literal;
3740
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
3841
import org.apache.doris.nereids.trees.plans.Plan;
@@ -131,6 +134,11 @@ public Void visitLike(Like like, Map<Expression, Set<Expression>> context) {
131134

132135
@Override
133136
public Void visitOr(Or or, Map<Expression, Set<Expression>> context) {
137+
// MATCH and SEARCH depend on the inverted index of their column, so they can differ on an equal
138+
// column, and a copy would also add to score().
139+
if (or.containsType(Match.class, Search.class, SearchExpression.class)) {
140+
return null;
141+
}
134142
for (Expression expr : getAllSubExpressions(or)) {
135143
context.computeIfAbsent(expr, k -> new LinkedHashSet<>()).add(or);
136144
}

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,15 @@
2424
import org.apache.doris.nereids.trees.expressions.InPredicate;
2525
import org.apache.doris.nereids.trees.expressions.LessThan;
2626
import org.apache.doris.nereids.trees.expressions.Like;
27+
import org.apache.doris.nereids.trees.expressions.MatchAny;
2728
import org.apache.doris.nereids.trees.expressions.Not;
2829
import org.apache.doris.nereids.trees.expressions.Or;
2930
import org.apache.doris.nereids.trees.expressions.SlotReference;
3031
import org.apache.doris.nereids.trees.expressions.functions.scalar.Abs;
3132
import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue;
3233
import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
3334
import org.apache.doris.nereids.trees.expressions.functions.scalar.Length;
35+
import org.apache.doris.nereids.trees.expressions.functions.scalar.Search;
3436
import org.apache.doris.nereids.trees.expressions.functions.scalar.SignBit;
3537
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
3638
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
@@ -90,6 +92,20 @@ public void testDoNotInferNoneMovablePredicateInsideOr() {
9092
Assertions.assertEquals(inputs, PredicateInferUtils.inferAllPredicate(inputs));
9193
}
9294

95+
@Test
96+
public void testDoNotInferMatchOrSearchInsideOr() {
97+
SlotReference a = new SlotReference("a", StringType.INSTANCE);
98+
SlotReference b = new SlotReference("b", StringType.INSTANCE);
99+
for (Expression predicate : ImmutableList.of(
100+
new Or(new MatchAny(a, new StringLiteral("apple")), new MatchAny(a, new StringLiteral("pear"))),
101+
new Or(new Search(new StringLiteral("a:apple")), new EqualTo(a, new StringLiteral("pear"))))) {
102+
Set<Expression> inputs = new LinkedHashSet<>(ImmutableList.of(new EqualTo(a, b), predicate));
103+
104+
Assertions.assertEquals(inputs, InferPredicateByReplace.infer(inputs));
105+
Assertions.assertEquals(inputs, PredicateInferUtils.inferAllPredicate(inputs));
106+
}
107+
}
108+
93109
@Test
94110
public void testInferWithEqualTo() {
95111
SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !different_index --
3+
1
4+
5+
-- !score --
6+
1 0.4700036
7+
2 0.4700036
8+
9+
-- !minimum_score --
10+
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
// An equality between two columns does not let a MATCH on one column be copied to the other:
19+
// MATCH depends on the inverted index of its column, and a copy would add to score().
20+
suite("infer_match_predicate") {
21+
// c2 has an index without a parser, so MATCH on c2 only matches a whole value.
22+
sql "DROP TABLE IF EXISTS infer_match_predicate_parser"
23+
sql """
24+
CREATE TABLE infer_match_predicate_parser (
25+
id INT,
26+
c1 TEXT,
27+
c2 TEXT,
28+
INDEX idx_c1 (c1) USING INVERTED PROPERTIES ("parser" = "english"),
29+
INDEX idx_c2 (c2) USING INVERTED
30+
) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
31+
PROPERTIES ("replication_num" = "1")
32+
"""
33+
sql "INSERT INTO infer_match_predicate_parser VALUES (1, 'apple red', 'apple red'), (2, 'banana', 'banana')"
34+
order_qt_different_index """
35+
SELECT id FROM infer_match_predicate_parser
36+
WHERE c1 = c2 AND (c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear')
37+
"""
38+
39+
sql "DROP TABLE IF EXISTS infer_match_predicate_score"
40+
sql """
41+
CREATE TABLE infer_match_predicate_score (
42+
id INT,
43+
c1 TEXT,
44+
c2 TEXT,
45+
INDEX idx_c1 (c1) USING INVERTED PROPERTIES ("parser" = "english"),
46+
INDEX idx_c2 (c2) USING INVERTED PROPERTIES ("parser" = "english")
47+
) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1
48+
PROPERTIES ("replication_num" = "1")
49+
"""
50+
sql """
51+
INSERT INTO infer_match_predicate_score VALUES
52+
(1, 'apple red', 'apple red'), (2, 'apple green', 'apple green'), (3, 'banana yellow', 'banana yellow')
53+
"""
54+
order_qt_score """
55+
SELECT id, score() AS s FROM infer_match_predicate_score
56+
WHERE c1 = c2 AND (c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear') ORDER BY s DESC LIMIT 10
57+
"""
58+
order_qt_minimum_score """
59+
SELECT id, score() AS s FROM infer_match_predicate_score
60+
WHERE c1 = c2 AND (c1 MATCH_ANY 'apple' OR c1 MATCH_ANY 'pear') AND score() > 0.6
61+
ORDER BY s DESC LIMIT 10
62+
"""
63+
}

0 commit comments

Comments
 (0)