Skip to content

Commit 8475be8

Browse files
authored
branch-4.1: [fix](dereference) Resolve correlated qualified columns before dereference (apache#67438) (apache#68260) (apache#68062)
apache#67438 apache#68260
1 parent 2eb1ec3 commit 8475be8

10 files changed

Lines changed: 1166 additions & 75 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/Scope.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.doris.nereids.util.Utils;
2222

2323
import com.google.common.base.Suppliers;
24+
import com.google.common.collect.ImmutableSetMultimap;
2425
import com.google.common.collect.LinkedListMultimap;
2526
import com.google.common.collect.ListMultimap;
2627
import com.google.common.collect.Sets;
@@ -66,6 +67,7 @@ public class Scope {
6667
private final boolean buildNameToSlot;
6768
private final Supplier<ListMultimap<String, Slot>> nameToSlot;
6869
private final Supplier<ListMultimap<String, Slot>> nameToAsteriskSlot;
70+
private final Supplier<ImmutableSetMultimap<String, List<String>>> relationNameToQualifiers;
6971

7072
public Scope(List<Slot> slots) {
7173
this(Optional.empty(), slots);
@@ -87,6 +89,7 @@ public Scope(Optional<Scope> outerScope, List<Slot> slots, List<Slot> asteriskSl
8789
this.buildNameToSlot = slots.size() > 500;
8890
this.nameToSlot = buildNameToSlot ? Suppliers.memoize(this::buildNameToSlot) : null;
8991
this.nameToAsteriskSlot = buildNameToSlot ? Suppliers.memoize(this::buildNameToAsteriskSlot) : null;
92+
this.relationNameToQualifiers = Suppliers.memoize(this::buildRelationNameToQualifiers);
9093
this.asteriskSlots = Utils.fastToImmutableList(
9194
Objects.requireNonNull(asteriskSlots, "asteriskSlots can not be null"));
9295
}
@@ -107,6 +110,11 @@ public Set<Slot> getCorrelatedSlots() {
107110
return correlatedSlots;
108111
}
109112

113+
/** Find distinct relation qualifiers by relation name, ignoring case. */
114+
public Set<List<String>> findRelationQualifiersIgnoreCase(String relationName) {
115+
return relationNameToQualifiers.get().get(relationName.toUpperCase(Locale.ROOT));
116+
}
117+
110118
/** findSlotIgnoreCase */
111119
public List<Slot> findSlotIgnoreCase(String slotName, boolean all) {
112120
List<Slot> slots = all ? this.slots : this.asteriskSlots;
@@ -140,4 +148,15 @@ private ListMultimap<String, Slot> buildNameToAsteriskSlot() {
140148
}
141149
return map;
142150
}
151+
152+
private ImmutableSetMultimap<String, List<String>> buildRelationNameToQualifiers() {
153+
ImmutableSetMultimap.Builder<String, List<String>> builder = ImmutableSetMultimap.builder();
154+
for (Slot slot : slots) {
155+
if (!slot.getQualifier().isEmpty()) {
156+
List<String> qualifier = slot.getQualifier();
157+
builder.put(qualifier.get(qualifier.size() - 1).toUpperCase(Locale.ROOT), qualifier);
158+
}
159+
}
160+
return builder.build();
161+
}
143162
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java‎

Lines changed: 103 additions & 56 deletions
Large diffs are not rendered by default.

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java‎

Lines changed: 203 additions & 18 deletions
Large diffs are not rendered by default.

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ProjectOtherJoinConditionForNestedLoopJoin.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.doris.nereids.trees.expressions.Expression;
2424
import org.apache.doris.nereids.trees.expressions.NamedExpression;
2525
import org.apache.doris.nereids.trees.expressions.Slot;
26+
import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
2627
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
2728
import org.apache.doris.nereids.trees.plans.Plan;
2829
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@@ -103,6 +104,15 @@ public ReplacerContext(Set<Slot> leftSlots, Set<Slot> rightSlots) {
103104
private static class AliasReplacer extends DefaultExpressionRewriter<ReplacerContext> {
104105
public static AliasReplacer INSTANCE = new AliasReplacer();
105106

107+
@Override
108+
public Expression visitLambda(Lambda lambda, ReplacerContext ctx) {
109+
// A lambda body is evaluated per array item. An expression in it may reference the lambda
110+
// arguments, which are not input slots and which no child of the join outputs, so it can
111+
// not be evaluated in a child Project:
112+
// array_map(x -> x + t2.b, [0]) > t1.a -- `x + t2.b` must stay inside the lambda
113+
return lambda;
114+
}
115+
106116
@Override
107117
public Expression visit(Expression expression, ReplacerContext ctx) {
108118
Set<Slot> input = expression.getInputSlots();
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.analyzer;
19+
20+
import org.apache.doris.nereids.trees.expressions.ExprId;
21+
import org.apache.doris.nereids.trees.expressions.SlotReference;
22+
import org.apache.doris.nereids.types.IntegerType;
23+
24+
import com.google.common.collect.ImmutableList;
25+
import org.junit.jupiter.api.Assertions;
26+
import org.junit.jupiter.api.Test;
27+
28+
import java.util.List;
29+
30+
class ScopeTest {
31+
32+
@Test
33+
void testFindRelationQualifiersIgnoreCase() {
34+
List<String> qualifier1 = ImmutableList.of("internal", "db", "t1");
35+
List<String> qualifier2 = ImmutableList.of("internal", "db", "t2");
36+
Scope scope = new Scope(ImmutableList.of(
37+
new SlotReference(new ExprId(1), "c1", IntegerType.INSTANCE, true, qualifier1),
38+
new SlotReference(new ExprId(2), "c2", IntegerType.INSTANCE, true, qualifier1),
39+
new SlotReference(new ExprId(3), "c1", IntegerType.INSTANCE, true, qualifier2),
40+
new SlotReference(new ExprId(4), "unqualified", IntegerType.INSTANCE, true, ImmutableList.of())));
41+
42+
Assertions.assertEquals(ImmutableList.of(qualifier1),
43+
ImmutableList.copyOf(scope.findRelationQualifiersIgnoreCase("T1")));
44+
Assertions.assertEquals(ImmutableList.of(qualifier2),
45+
ImmutableList.copyOf(scope.findRelationQualifiersIgnoreCase("t2")));
46+
Assertions.assertTrue(scope.findRelationQualifiersIgnoreCase("unqualified").isEmpty());
47+
}
48+
}

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzerTest.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,60 @@
3535
import org.junit.jupiter.api.Test;
3636

3737
import java.util.List;
38+
import java.util.Optional;
39+
import java.util.Set;
3840

3941
public class ExpressionAnalyzerTest {
4042

43+
@Test
44+
void testSkipQualifierOccupancyForLocalBindingHit() {
45+
SlotReference localSlot = new SlotReference(
46+
new ExprId(1), "c", BigIntType.INSTANCE, true, ImmutableList.of("t"));
47+
Scope outerScope = new Scope(ImmutableList.of());
48+
Scope localScope = new Scope(Optional.of(outerScope), ImmutableList.of(localSlot)) {
49+
@Override
50+
public Set<List<String>> findRelationQualifiersIgnoreCase(String relationName) {
51+
throw new AssertionError("Qualifier occupancy should not be evaluated for a binding hit");
52+
}
53+
};
54+
ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, localScope, null, true, true);
55+
56+
Assertions.assertEquals(localSlot, analyzer.analyze(new UnboundSlot("t", "c")));
57+
}
58+
59+
@Test
60+
void testOuterRelationProbeDoesNotEvaluateQualifierOccupancy() {
61+
SlotReference outerSlot = new SlotReference(
62+
new ExprId(1), "c", BigIntType.INSTANCE, true, ImmutableList.of("t"));
63+
Scope outerScope = new Scope(ImmutableList.of(outerSlot)) {
64+
@Override
65+
public Set<List<String>> findRelationQualifiersIgnoreCase(String relationName) {
66+
throw new AssertionError("Outer relation probe should only bind slots");
67+
}
68+
};
69+
Scope localScope = new Scope(Optional.of(outerScope), ImmutableList.of());
70+
ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, localScope, null, true, true);
71+
72+
Assertions.assertEquals(outerSlot, analyzer.analyze(new UnboundSlot("t", "c")));
73+
Assertions.assertEquals(ImmutableList.of(outerSlot), ImmutableList.copyOf(outerScope.getCorrelatedSlots()));
74+
}
75+
76+
@Test
77+
void testKeepQualifierOccupancyLazyInExactBinding() {
78+
Scope scope = new Scope(ImmutableList.of()) {
79+
@Override
80+
public Set<List<String>> findRelationQualifiersIgnoreCase(String relationName) {
81+
throw new AssertionError("Exact binding should preserve lazy qualifier occupancy");
82+
}
83+
};
84+
ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, scope, null, true, true);
85+
86+
ExpressionAnalyzer.SlotBinding binding = Assertions.assertDoesNotThrow(
87+
() -> analyzer.bindExactSlotsByThisScope(new UnboundSlot("t", "c"), scope, true));
88+
Assertions.assertTrue(binding.getBoundSlots().isEmpty());
89+
Assertions.assertThrows(AssertionError.class, binding::isRelationQualifierOccupied);
90+
}
91+
4192
@Test
4293
void testPreProcessUnboundFunctionForThreeArgsDataTimeFunction() {
4394
ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, new Scope(ImmutableList.of()),

0 commit comments

Comments
 (0)