Skip to content

Commit 5f107ef

Browse files
[fix](rbo) Preserve semantics in predicate inference (apache#67919)
### What problem does this PR solve? Problem Summary: Fix two predicate inference errors that change query results: - `chooseInputPredicates` records a retained `GT`/`GTE` predicate as `EQ`. The false equality can make another necessary predicate appear redundant. For example, `a > b AND rn > b AND a > rn` can lose `a > rn` and return extra rows in a QUALIFY query. Record the actual relation in the working graph. - Comparison-equal operands are not interchangeable inside arbitrary expressions. DATE and DATETIME values can compare equal but have different string lengths; negative and positive floating-point zero compare equal but have different SIGNBIT results. Substitution can therefore add filters that discard matching rows. Restrict expression substitution to exactly matching supported scalar types with value-preserving equality. Preserve direct comparisons, IN and their negations on the unwrapped operand, along with the existing determinism and cast-analysis checks. The fixes are in separate commits. Shared cast extraction is unchanged, and existing regression expectations are unchanged. ### Release note Fix extra or missing rows caused by incorrect predicate inference, including QUALIFY inequality chains and substitutions inside type-sensitive or representation-sensitive expressions.
1 parent c1b2bdb commit 5f107ef

30 files changed

Lines changed: 712 additions & 59 deletions

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CollectFilterAboveConsumer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ public Rule build() {
3838
LogicalCTEConsumer cteConsumer = filter.child();
3939
Set<Expression> exprs = filter.getConjuncts();
4040
for (Expression expr : exprs) {
41-
if (expr.containsVolatileExpression()) {
41+
if (expr.containsVolatileOrNoneMovableExpression()) {
4242
continue;
4343
}
4444
Expression rewrittenExpr = expr.rewriteUp(e -> {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicateByReplace.java‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,11 @@
3232
import org.apache.doris.nereids.trees.expressions.Or;
3333
import org.apache.doris.nereids.trees.expressions.Slot;
3434
import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
35+
import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction;
3536
import org.apache.doris.nereids.trees.expressions.literal.Literal;
3637
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
3738
import org.apache.doris.nereids.trees.plans.Plan;
39+
import org.apache.doris.nereids.types.DataType;
3840
import org.apache.doris.nereids.types.DecimalV2Type;
3941
import org.apache.doris.nereids.types.DecimalV3Type;
4042
import org.apache.doris.nereids.util.ExpressionUtils;
@@ -153,12 +155,15 @@ private static <T extends Expression> Set<Expression> getEqualSetAndDoReplace(T
153155
ExpressionAnalyzer analyzer = new ReplaceAnalyzer(null, new Scope(ImmutableList.of()), null, false, false);
154156
Set<Expression> res = new LinkedHashSet<>();
155157
for (T equals : equalSet) {
156-
Map<Expression, Expression> replaceMap = new HashMap<>();
157-
replaceMap.put(equals, replaceToThis);
158158
if (!exprPredicates.containsKey(equals)) {
159159
continue;
160160
}
161+
Map<Expression, Expression> replaceMap = new HashMap<>();
162+
replaceMap.put(equals, replaceToThis);
161163
for (Expression predicate : exprPredicates.get(equals)) {
164+
if (!canReplace(equals, replaceToThis, predicate)) {
165+
continue;
166+
}
162167
Expression newPredicates = ExpressionUtils.replace(predicate, replaceMap);
163168
try {
164169
Expression analyzed = analyzer.analyze(newPredicates);
@@ -171,6 +176,27 @@ private static <T extends Expression> Set<Expression> getEqualSetAndDoReplace(T
171176
return res;
172177
}
173178

179+
private static boolean canReplace(Expression source, Expression target, Expression predicate) {
180+
Expression comparison = predicate instanceof Not ? predicate.child(0) : predicate;
181+
// Direct comparisons observe comparison equality rather than a value's type or representation.
182+
// Do not descend through functions, casts or OR to apply this exception.
183+
if ((comparison instanceof ComparisonPredicate || comparison instanceof InPredicate)
184+
&& comparison.child(0).equals(source)) {
185+
return true;
186+
}
187+
DataType type = source.getDataType();
188+
// Comparison equality across types does not preserve type-sensitive expressions such as CAST to STRING.
189+
if (!type.equals(target.getDataType())) {
190+
return false;
191+
}
192+
// Only substitute types whose equality preserves the value observed by enclosing expressions.
193+
// In particular, FLOAT/DOUBLE equality cannot distinguish signed zero, but SIGNBIT can.
194+
// Comparisons can still be propagated separately by UnequalPredicateInfer.
195+
return type.isBooleanType() || type.isIntegralType() || type.isDecimalLikeType()
196+
|| type.isStringLikeType() || type.isIPType()
197+
|| (type.isDateLikeType() && !type.isTimeStampTzType());
198+
}
199+
174200
/* Extract the equivalence relationship a=b, and when case (d_tinyint as int)=d_int is encountered,
175201
remove the cast and extract d_tinyint=d_int
176202
EqualPairs is the output parameter and the equivalent pair of predicate derivation input,
@@ -210,7 +236,9 @@ public static Set<Expression> infer(Set<Expression> inputs) {
210236
}
211237
Map<Expression, Set<Expression>> exprPredicates = new HashMap<>();
212238
for (Expression input : inputs) {
213-
if (input.anyMatch(expr -> !((ExpressionTrait) expr).isDeterministic())
239+
// Inference can evaluate a predicate on rows that never reach its original filter.
240+
if (input.anyMatch(expr -> expr instanceof NoneMovableFunction
241+
|| !((ExpressionTrait) expr).isDeterministic())
214242
|| input.getInputSlots().size() != 1) {
215243
continue;
216244
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferPredicates.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@ private Plan inferNewPredicate(Plan plan, Set<Expression> expressions) {
218218
Set<Expression> predicates = new LinkedHashSet<>();
219219
Set<Slot> planOutputs = plan.getOutputSet();
220220
for (Expression expr : expressions) {
221-
if (expr.containsVolatileExpression()) {
221+
if (expr.containsVolatileOrNoneMovableExpression()) {
222222
// Volatile expressions (e.g. rand(), uuid()) must not be cloned into
223223
// subtrees that did not already evaluate them. Otherwise, callers that perform
224224
// slot substitution (e.g. SetOp visitors below) would introduce a fresh
@@ -250,7 +250,7 @@ private Plan inferNewPredicateRemoveUselessIsNull(Plan plan, Set<Expression> exp
250250
Set<Expression> predicates = new LinkedHashSet<>();
251251
Set<Slot> planOutputs = plan.getOutputSet();
252252
for (Expression expr : expressions) {
253-
if (expr.containsVolatileExpression()) {
253+
if (expr.containsVolatileOrNoneMovableExpression()) {
254254
// See inferNewPredicate for rationale: never clone volatile
255255
// predicates into a subtree that did not already evaluate them.
256256
continue;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ProjectOtherJoinConditionForNestedLoopJoin.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ public Expression visit(Expression expression, ReplacerContext ctx) {
124124
// pair" to "per row of that child", which silently changes results. Keep such
125125
// expressions inline in otherJoinConjuncts, but still recurse to extract deterministic
126126
// child expressions.
127-
if (expression.containsVolatileExpression()) {
127+
if (expression.containsVolatileOrNoneMovableExpression()) {
128128
return super.visit(expression, ctx);
129129
}
130130
if (ctx.leftSlots.containsAll(input)) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughAggregation.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ public Rule build() {
6969
// 2. if the conjunct contains unique function, it should not be pushed down;
7070
// e.g. 'select a, sum(a) from t group by a having a + random() > 10'
7171
// not equals 'select a, sum(a) from t where a + random() > 10 group by a'
72-
if (!conjunct.containsVolatileExpression()
72+
if (!conjunct.containsVolatileOrNoneMovableExpression()
7373
&& !conjunctSlots.isEmpty() && canPushDownSlots.containsAll(conjunctSlots)) {
7474
pushDownPredicates.add(conjunct);
7575
} else {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughGenerate.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public Rule build() {
5050
filter.getConjuncts().forEach(conjunct -> {
5151
Set<Slot> conjunctSlots = conjunct.getInputSlots();
5252
if (!conjunctSlots.isEmpty() && childOutputs.containsAll(conjunctSlots)
53-
&& !conjunct.containsVolatileExpression()) {
53+
&& !conjunct.containsVolatileOrNoneMovableExpression()) {
5454
pushDownPredicates.add(conjunct);
5555
} else {
5656
remainPredicates.add(conjunct);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughJoin.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ public Rule build() {
120120
Set<Expression> rightPredicates = Sets.newLinkedHashSet();
121121
Set<Expression> remainingPredicates = Sets.newLinkedHashSet();
122122
for (Expression p : filterPredicates) {
123-
if (p.containsVolatileExpression()) {
123+
if (p.containsVolatileOrNoneMovableExpression()) {
124124
remainingPredicates.add(p);
125125
continue;
126126
}
@@ -162,7 +162,7 @@ private boolean convertJoinCondition(Expression predicate, Set<Slot> leftOutputs
162162
if (!(predicate instanceof EqualTo)) {
163163
return false;
164164
}
165-
if (predicate.containsVolatileExpression()) {
165+
if (predicate.containsVolatileOrNoneMovableExpression()) {
166166
return false;
167167
}
168168

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughPartitionTopN.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ public Rule build() {
7777
// top-N", and the surviving rows of every partition would no longer be the true
7878
// top-N. Empty-input-slot predicates like `rand() > 0.5` would also bypass the
7979
// `containsAll` check otherwise.
80-
if (!expr.containsVolatileExpression() && partitionKeySlots.containsAll(exprInputSlots)) {
80+
if (!expr.containsVolatileOrNoneMovableExpression() && partitionKeySlots.containsAll(exprInputSlots)) {
8181
bottomConjunctsBuilder.add(expr);
8282
} else {
8383
upperConjunctsBuilder.add(expr);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughProject.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ private static Pair<Set<Expression>, Set<Expression>> splitConjunctsByChildOutpu
125125
// `project(b + random(1, 10) as a) -> filter(b + random(1, 10) > 1)`, it contains two distinct RANDOM.
126126
if (childOutputs.containsAll(conjunctSlots)
127127
&& conjunctSlots.stream().map(childAlias::get).filter(Objects::nonNull)
128-
.noneMatch(Expression::containsVolatileExpression)) {
128+
.noneMatch(Expression::containsVolatileOrNoneMovableExpression)) {
129129
pushDownPredicates.add(conjunct);
130130
} else {
131131
remainPredicates.add(conjunct);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughSetOperation.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public Rule build() {
8787
pushableConjuncts = new LinkedHashSet<>();
8888
Set<Expression> kept = new LinkedHashSet<>();
8989
for (Expression c : origFilter.getConjuncts()) {
90-
if (c.containsVolatileExpression()) {
90+
if (c.containsVolatileOrNoneMovableExpression()) {
9191
kept.add(c);
9292
} else {
9393
pushableConjuncts.add(c);

0 commit comments

Comments
 (0)