Skip to content

Commit b5be6ee

Browse files
[fix](nereids) Preserve source-dependent predicates above set operations
### Background Search predicates depend on the source column's index and analyzer, not only on its value. Set-operation pushdown substitutes each branch's output columns. ### What problem does this PR solve? Issue Number: None Related PR: #69344 (follow-up) PR Type: Bugfix Problem Summary: Outer SEARCH predicates could be copied to branches with different analyzers, changing EXCEPT results. Rename NonInferableExpression to SourceDependentExpression and retain marked conjuncts above every set operation, including UNION ALL. Existing final SEARCH validation rejects unsupported placement. ### Details - **Frequency:** Outer SEARCH on set operations with source-dependent matching. - **Root cause:** Ordinary set-operation pushdown bypassed inference guards; UNION ALL had an unconditional fast path. - **Why existing tests missed it:** Tests expected outer UNION SEARCH to be copied into each branch. - **Fix:** Classify all conjuncts using the source-dependent marker while preserving existing volatile-expression handling. - **Other paths and branches:** Inference guards and implementations use the renamed interface. Same-source alias pushdown remains allowed. MATCH/MultiMatch final validation is deferred. - **Behavior change:** Outer SEARCH retained above set operations is rejected by existing FE validation; branch-local SEARCH remains supported. ### Testing On ssh dev: 160 FE unit tests passed across CheckSearchUsageTest, PushDownFilterThroughSetOperationTest, InferPredicateByReplaceTest and InferPredicatesTest. Standard build.sh --fe with DISABLE_BUILD_UI=ON passed, including Checkstyle. Updated FE deployed. Generated regression output using run-regression-test.sh. SEARCH directory: 47/48 suites passed, including changed suites. test_search_inverted_index_profile failed and failed independently on retry because InvertedIndexSearcherCacheHit was 0 instead of positive; its single-table query does not exercise set-operation pushdown. ### Release note SEARCH predicates over set-operation outputs are rejected instead of being evaluated with different branch indexes or analyzers. SEARCH inside individual branches remains supported. ### Check List (For Author) - Behavior changed: Yes (see Details) - Does this need documentation: No - Pre-submission status: Partially verified: one cache-profile regression remains failing as described in Testing. ### AI Disclosure - Tool: Codex - Generated: Code, tests, and commit description. - Human review: Pending.
1 parent 7846086 commit b5be6ee

14 files changed

Lines changed: 81 additions & 70 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,10 @@
2828
import org.apache.doris.nereids.trees.expressions.Expression;
2929
import org.apache.doris.nereids.trees.expressions.InPredicate;
3030
import org.apache.doris.nereids.trees.expressions.Like;
31-
import org.apache.doris.nereids.trees.expressions.NonInferableExpression;
3231
import org.apache.doris.nereids.trees.expressions.Not;
3332
import org.apache.doris.nereids.trees.expressions.Or;
3433
import org.apache.doris.nereids.trees.expressions.Slot;
34+
import org.apache.doris.nereids.trees.expressions.SourceDependentExpression;
3535
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
3636
import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction;
3737
import org.apache.doris.nereids.trees.expressions.literal.Literal;
@@ -239,7 +239,7 @@ public static Set<Expression> infer(Set<Expression> inputs) {
239239
for (Expression input : inputs) {
240240
// Inference can evaluate a predicate on rows that never reach its original filter.
241241
// MATCH, multi_match and SEARCH depend on column indexes, and copies can change score().
242-
if (input.containsType(NonInferableExpression.class)
242+
if (input.containsType(SourceDependentExpression.class)
243243
|| input.anyMatch(expr -> expr instanceof NoneMovableFunction
244244
|| !((ExpressionTrait) expr).isDeterministic())
245245
|| input.getInputSlots().size() != 1) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@
2323
import org.apache.doris.nereids.trees.expressions.Expression;
2424
import org.apache.doris.nereids.trees.expressions.IsNull;
2525
import org.apache.doris.nereids.trees.expressions.NamedExpression;
26-
import org.apache.doris.nereids.trees.expressions.NonInferableExpression;
2726
import org.apache.doris.nereids.trees.expressions.Or;
2827
import org.apache.doris.nereids.trees.expressions.Slot;
28+
import org.apache.doris.nereids.trees.expressions.SourceDependentExpression;
2929
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
3030
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
3131
import org.apache.doris.nereids.trees.plans.Plan;
@@ -220,7 +220,7 @@ private Plan inferNewPredicate(Plan plan, Set<Expression> expressions) {
220220
Set<Slot> planOutputs = plan.getOutputSet();
221221
for (Expression expr : expressions) {
222222
// Search predicates depend on the source index/analyzer, which a sibling may not have.
223-
if (expr.containsType(NonInferableExpression.class)
223+
if (expr.containsType(SourceDependentExpression.class)
224224
|| expr.containsVolatileOrNoneMovableExpression()) {
225225
// Volatile expressions (e.g. rand(), uuid()) must not be cloned into
226226
// subtrees that did not already evaluate them. Otherwise, callers that perform
@@ -254,7 +254,7 @@ private Plan inferNewPredicateRemoveUselessIsNull(Plan plan, Set<Expression> exp
254254
Set<Slot> planOutputs = plan.getOutputSet();
255255
for (Expression expr : expressions) {
256256
// Search predicates depend on the source index/analyzer, which a sibling may not have.
257-
if (expr.containsType(NonInferableExpression.class)
257+
if (expr.containsType(SourceDependentExpression.class)
258258
|| expr.containsVolatileOrNoneMovableExpression()) {
259259
// See inferNewPredicate for rationale: never clone volatile
260260
// predicates into a subtree that did not already evaluate them.

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughSetOperation.java‎

Lines changed: 14 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.doris.nereids.trees.expressions.Expression;
2626
import org.apache.doris.nereids.trees.expressions.NamedExpression;
2727
import org.apache.doris.nereids.trees.expressions.SlotReference;
28+
import org.apache.doris.nereids.trees.expressions.SourceDependentExpression;
2829
import org.apache.doris.nereids.trees.plans.Plan;
2930
import org.apache.doris.nereids.trees.plans.algebra.EmptyRelation;
3031
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
@@ -76,30 +77,21 @@ public Rule build() {
7677
// "half of A intersect half of B" instead of "half of (A intersect B)").
7778
boolean canPushVolatileExpr = setOperation instanceof LogicalUnion
7879
&& setOperation.getQualifier() == Qualifier.ALL;
79-
Set<Expression> pushableConjuncts;
80-
Set<Expression> keptAboveConjuncts;
81-
boolean allConjunctsPushable;
82-
if (canPushVolatileExpr) {
83-
pushableConjuncts = origFilter.getConjuncts();
84-
keptAboveConjuncts = ImmutableSet.of();
85-
allConjunctsPushable = true;
86-
} else {
87-
pushableConjuncts = new LinkedHashSet<>();
88-
Set<Expression> kept = new LinkedHashSet<>();
89-
for (Expression c : origFilter.getConjuncts()) {
90-
if (c.containsVolatileOrNoneMovableExpression()) {
91-
kept.add(c);
92-
} else {
93-
pushableConjuncts.add(c);
94-
}
95-
}
96-
keptAboveConjuncts = kept;
97-
if (pushableConjuncts.isEmpty()) {
98-
return null;
80+
Set<Expression> pushableConjuncts = new LinkedHashSet<>();
81+
Set<Expression> keptAboveConjuncts = new LinkedHashSet<>();
82+
for (Expression conjunct : origFilter.getConjuncts()) {
83+
// Equal values in different branches may use different indexes or analyzers.
84+
if (conjunct.containsType(SourceDependentExpression.class)
85+
|| (!canPushVolatileExpr && conjunct.containsVolatileOrNoneMovableExpression())) {
86+
keptAboveConjuncts.add(conjunct);
87+
} else {
88+
pushableConjuncts.add(conjunct);
9989
}
100-
allConjunctsPushable = false;
10190
}
102-
LogicalFilter<LogicalSetOperation> filter = allConjunctsPushable
91+
if (pushableConjuncts.isEmpty()) {
92+
return null;
93+
}
94+
LogicalFilter<LogicalSetOperation> filter = keptAboveConjuncts.isEmpty()
10395
? origFilter
10496
: new LogicalFilter<>(ImmutableSet.copyOf(pushableConjuncts), setOperation);
10597

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Match.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
* like expression: a MATCH 'hello'.
3838
*/
3939
public abstract class Match extends BinaryOperator
40-
implements PropagateNullable, PreferPushDownProject, NonInferableExpression {
40+
implements PropagateNullable, PreferPushDownProject, SourceDependentExpression {
4141

4242
private final Optional<String> analyzer;
4343

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SearchExpression.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
* SearchExpression represents a search query with bound slot references.
3333
* This is created by RewriteSearchToSlots rule from Search scalar function.
3434
*/
35-
public class SearchExpression extends Expression implements NonInferableExpression {
35+
public class SearchExpression extends Expression implements SourceDependentExpression {
3636
private final String dslString;
3737
// The second argument of search(dsl, options), kept so that toSql() reproduces the call.
3838
private final Optional<String> optionsJson;

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NonInferableExpression.java renamed to fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SourceDependentExpression.java

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,12 @@
1818
package org.apache.doris.nereids.trees.expressions;
1919

2020
/**
21-
* Marks expressions that must not be used to derive additional predicates by equivalent-expression
22-
* substitution or propagation to sibling plan branches. For example, a search predicate depends on
23-
* its source column's index and analyzer, which an equivalent column may not share.
21+
* Marks expressions whose semantics depend on their source columns, including index and analyzer
22+
* metadata, rather than column values alone.
2423
*
25-
* <p>This trait does not prevent normal predicate pushdown, alias substitution, or pruning.
24+
* <p>Rewrites must preserve the original column source. Value equivalence alone does not justify
25+
* substitution or propagation to another source, including pushdown through set operations.
26+
* Alias substitution and predicate movement that preserve the source are allowed.
2627
*/
27-
public interface NonInferableExpression {
28+
public interface SourceDependentExpression {
2829
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MultiMatch.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
import org.apache.doris.catalog.FunctionSignature;
2121
import org.apache.doris.nereids.trees.expressions.Expression;
22-
import org.apache.doris.nereids.trees.expressions.NonInferableExpression;
22+
import org.apache.doris.nereids.trees.expressions.SourceDependentExpression;
2323
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
2424
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
2525
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
@@ -36,7 +36,7 @@
3636
* ScalarFunction 'multi_match'.
3737
*/
3838
public class MultiMatch extends ScalarFunction
39-
implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, NonInferableExpression {
39+
implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNotNullable, SourceDependentExpression {
4040

4141
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
4242
FunctionSignature.ret(BooleanType.INSTANCE).varArgs(StringType.INSTANCE)

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Search.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
import org.apache.doris.analysis.SearchDslParser;
2121
import org.apache.doris.catalog.FunctionSignature;
2222
import org.apache.doris.nereids.trees.expressions.Expression;
23-
import org.apache.doris.nereids.trees.expressions.NonInferableExpression;
23+
import org.apache.doris.nereids.trees.expressions.SourceDependentExpression;
2424
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
2525
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
2626
import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
@@ -49,7 +49,7 @@
4949
* Example options: '{"default_field":"title","mode":"lucene","minimum_should_match":0}'
5050
*/
5151
public class Search extends ScalarFunction
52-
implements ExplicitlyCastableSignature, AlwaysNotNullable, NonInferableExpression {
52+
implements ExplicitlyCastableSignature, AlwaysNotNullable, SourceDependentExpression {
5353

5454
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
5555
// Original signature: search(dsl_string)

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckSearchUsageTest.java‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -417,17 +417,20 @@ public void testSearchBindsVariantAlias() {
417417
}
418418

419419
@Test
420-
public void testSearchBindsUnionOutput() {
421-
PlanChecker.from(connectContext)
422-
.analyze("SELECT id FROM (SELECT id, title AS label FROM test_search_table "
423-
+ "UNION ALL SELECT id, body FROM test_search_input_other) t WHERE search('label:hello')")
424-
.rewrite()
425-
.matches(logicalFilter(logicalOlapScan()).when(filter -> filter.getExpressions().stream()
426-
.anyMatch(expr -> expr.anyMatch(e -> e instanceof SearchExpression
427-
&& ((SlotReference) e.child(0)).getOriginalColumn().get().getName().equals("title")))))
428-
.matches(logicalFilter(logicalOlapScan()).when(filter -> filter.getExpressions().stream()
429-
.anyMatch(expr -> expr.anyMatch(e -> e instanceof SearchExpression
430-
&& ((SlotReference) e.child(0)).getOriginalColumn().get().getName().equals("body")))));
420+
public void testSearchAboveSetOperationRejected() {
421+
for (String operator : new String[] {"UNION ALL", "UNION", "INTERSECT", "EXCEPT"}) {
422+
for (String predicate : new String[] {"search('label:hello')",
423+
"search('label:hello') AND id > 0", "search('label:hello') OR id > 0",
424+
"NOT search('label:hello')"}) {
425+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () ->
426+
PlanChecker.from(connectContext)
427+
.analyze("SELECT id FROM (SELECT id, title AS label FROM test_search_table "
428+
+ operator + " SELECT id, body FROM test_search_input_other) t WHERE " + predicate)
429+
.rewrite());
430+
Assertions.assertTrue(exception.getMessage().contains("filters directly on an OLAP table scan"),
431+
exception.getMessage());
432+
}
433+
}
431434
}
432435

433436
@Test
@@ -459,12 +462,12 @@ public void testSearchAcrossTablesRejectedAfterRewrite() {
459462
}
460463

461464
@Test
462-
public void testSearchChecksEachUnionBranchIndex() {
465+
public void testSearchAboveUnionWithoutBranchIndexRejected() {
463466
PlanChecker checker = PlanChecker.from(connectContext)
464467
.analyze("SELECT id FROM (SELECT id, title AS label FROM test_search_table "
465468
+ "UNION ALL SELECT id, plain FROM test_search_input_other) t WHERE search('label:hello')");
466469
AnalysisException exception = Assertions.assertThrows(AnalysisException.class, checker::rewrite);
467-
Assertions.assertTrue(exception.getMessage().contains("Field 'plain' has no inverted index"),
470+
Assertions.assertTrue(exception.getMessage().contains("filters directly on an OLAP table scan"),
468471
exception.getMessage());
469472
}
470473

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplaceTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,11 @@
2727
import org.apache.doris.nereids.trees.expressions.Like;
2828
import org.apache.doris.nereids.trees.expressions.MatchAny;
2929
import org.apache.doris.nereids.trees.expressions.MatchPhrase;
30-
import org.apache.doris.nereids.trees.expressions.NonInferableExpression;
3130
import org.apache.doris.nereids.trees.expressions.Not;
3231
import org.apache.doris.nereids.trees.expressions.Or;
3332
import org.apache.doris.nereids.trees.expressions.SearchExpression;
3433
import org.apache.doris.nereids.trees.expressions.SlotReference;
34+
import org.apache.doris.nereids.trees.expressions.SourceDependentExpression;
3535
import org.apache.doris.nereids.trees.expressions.functions.scalar.Abs;
3636
import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue;
3737
import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
@@ -140,7 +140,7 @@ public void testNonInferableTraitDoesNotRequireConcreteTypeChecks() {
140140
Assertions.assertEquals(inputs, PredicateInferUtils.inferAllPredicate(inputs));
141141
}
142142

143-
private static class NonInferableLike extends Like implements NonInferableExpression {
143+
private static class NonInferableLike extends Like implements SourceDependentExpression {
144144
NonInferableLike(Expression left, Expression right) {
145145
super(left, right);
146146
}

0 commit comments

Comments
 (0)