Skip to content

Commit 09a22aa

Browse files
authored
[fix](variable) Track user variables in SQL cache during binding (apache#67787)
### What problem does this PR solve? Related PR: apache#62524 Problem Summary: User-variable dependencies were registered in the SQL cache context only by the late `ReplaceVariableByLiteral` rewrite. Function binding unwraps a direct `Variable` argument to its real expression before that rewrite, so expressions such as `ABS(@v)` lost the variable dependency. After `@v` changed, the same SQL could therefore reuse a stale `PhysicalSqlCache` result and return the old value. This PR records the complete variable dependency while binding `UnboundVariable`, then directly returns its real expression. It removes the now-redundant `VariableToLiteral` / `ReplaceVariableByLiteral` path and the `ExpressionTrait` variable special case. It also moves the generated-column variable check to the parsed expression so variables inside functions remain rejected.
1 parent a6751eb commit 09a22aa

12 files changed

Lines changed: 75 additions & 232 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@
5252
import org.apache.doris.nereids.rules.analysis.QualifyToFilter;
5353
import org.apache.doris.nereids.rules.analysis.ReplaceExpressionByChildOutput;
5454
import org.apache.doris.nereids.rules.analysis.SubqueryToApply;
55-
import org.apache.doris.nereids.rules.analysis.VariableToLiteral;
5655
import org.apache.doris.nereids.rules.rewrite.AdjustNullable;
5756
import org.apache.doris.nereids.rules.rewrite.MergeFilters;
5857
import org.apache.doris.nereids.rules.rewrite.SimplifyAggGroupBy;
@@ -168,17 +167,6 @@ private static List<RewriteJob> buildAnalyzerJobs() {
168167
// LogicalProject for normalize. This rule depends on FillUpMissingSlots to fill up slots.
169168
new NormalizeRepeat()
170169
),
171-
// consider sql with user defined var @t_zone
172-
// set @t_zone='GMT';
173-
// SELECT
174-
// DATE_FORMAT(convert_tz(dt, time_zone, @t_zone),'%Y-%m-%d') day
175-
// FROM
176-
// t
177-
// GROUP BY
178-
// 1;
179-
// @t_zone must be replaced as 'GMT' before EliminateGroupByConstant and NormalizeAggregate rule.
180-
// So need run VariableToLiteral rule before the two rules.
181-
topDown(new VariableToLiteral()),
182170
// run CheckSearchUsage before CheckAnalysis to detect search() in GROUP BY before it gets optimized
183171
bottomUp(new CheckSearchUsage()),
184172
// run CheckAnalysis before EliminateGroupByConstant in order to report error message correctly like bellow

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,12 @@ public Expression visit(Expression expr, ExpressionRewriteContext context) {
225225
* ******************************************************************************************** */
226226
@Override
227227
public Expression visitUnboundVariable(UnboundVariable unboundVariable, ExpressionRewriteContext context) {
228-
return resolveUnboundVariable(unboundVariable);
228+
Variable variable = resolveUnboundVariable(unboundVariable);
229+
if (wantToParseSqlFromSqlCache) {
230+
getCascadesContext().getStatementContext().getSqlCacheContext()
231+
.ifPresent(sqlCacheContext -> sqlCacheContext.addUsedVariable(variable));
232+
}
233+
return variable.getRealExpression();
229234
}
230235

231236
/** resolveUnboundVariable */

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/VariableToLiteral.java‎

Lines changed: 0 additions & 40 deletions
This file was deleted.

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@ public enum ExpressionRuleType {
5050
NORMALIZE_BINARY_PREDICATES,
5151
NULL_SAFE_EQUAL_TO_EQUAL,
5252
PUSH_INTO_CASE_WHEN_BRANCH,
53-
REPLACE_VARIABLE_BY_LITERAL,
5453
SIMPLIFY_ARITHMETIC_COMPARISON,
5554
SIMPLIFY_ARITHMETIC,
5655
SIMPLIFY_CAST,

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ReplaceVariableByLiteral.java‎

Lines changed: 0 additions & 53 deletions
This file was deleted.

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTrait.java‎

Lines changed: 2 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import org.apache.doris.nereids.exceptions.UnboundException;
2222
import org.apache.doris.nereids.trees.TreeNode;
2323
import org.apache.doris.nereids.trees.expressions.Expression;
24-
import org.apache.doris.nereids.trees.expressions.Variable;
2524
import org.apache.doris.nereids.trees.expressions.VolatileExpression;
2625
import org.apache.doris.nereids.types.DataType;
2726

@@ -52,38 +51,14 @@ default void checkLegalityAfterRewrite() {}
5251
* getArguments.
5352
*/
5453
default List<Expression> getArguments() {
55-
boolean hasVariableArg = false;
56-
for (Expression arg : children()) {
57-
if (arg instanceof Variable) {
58-
hasVariableArg = true;
59-
break;
60-
}
61-
}
62-
if (hasVariableArg) {
63-
ImmutableList.Builder<Expression> arguments = ImmutableList.builder();
64-
for (Expression arg : children()) {
65-
if (arg instanceof Variable) {
66-
arguments.add(((Variable) arg).getRealExpression());
67-
} else {
68-
arguments.add(arg);
69-
}
70-
}
71-
return arguments.build();
72-
} else {
73-
return children();
74-
}
54+
return children();
7555
}
7656

7757
/**
7858
* getArgument.
7959
*/
8060
default Expression getArgument(int index) {
81-
Expression arg = child(index);
82-
if (arg instanceof Variable) {
83-
return ((Variable) arg).getRealExpression();
84-
} else {
85-
return arg;
86-
}
61+
return child(index);
8762
}
8863

8964
default List<DataType> getArgumentsTypes() {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@
5656
import org.apache.doris.nereids.analyzer.Scope;
5757
import org.apache.doris.nereids.analyzer.UnboundFunction;
5858
import org.apache.doris.nereids.analyzer.UnboundSlot;
59+
import org.apache.doris.nereids.analyzer.UnboundVariable;
5960
import org.apache.doris.nereids.exceptions.AnalysisException;
6061
import org.apache.doris.nereids.glue.translator.ExpressionTranslator;
6162
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
@@ -66,7 +67,6 @@
6667
import org.apache.doris.nereids.trees.expressions.Slot;
6768
import org.apache.doris.nereids.trees.expressions.SlotReference;
6869
import org.apache.doris.nereids.trees.expressions.SubqueryExpr;
69-
import org.apache.doris.nereids.trees.expressions.Variable;
7070
import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
7171
import org.apache.doris.nereids.trees.expressions.functions.Udf;
7272
import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction;
@@ -1209,16 +1209,16 @@ void checkParsedExpressionInGeneratedColumn(Expression expr) {
12091209
throw new AnalysisException("Generated column does not support subquery.");
12101210
} else if (e instanceof Lambda) {
12111211
throw new AnalysisException("Generated column does not support lambda.");
1212+
} else if (e instanceof UnboundVariable) {
1213+
throw new AnalysisException("Generated column expression cannot contain variable.");
12121214
}
12131215
});
12141216
}
12151217

12161218
void checkExpressionInGeneratedColumn(Expression expr, ColumnDefinition column,
12171219
Map<String, ColumnDefinition> nameToColumnDefinition) {
12181220
expr.foreach(e -> {
1219-
if (e instanceof Variable) {
1220-
throw new AnalysisException("Generated column expression cannot contain variable.");
1221-
} else if (e instanceof Slot && nameToColumnDefinition.containsKey(((Slot) e).getName())) {
1221+
if (e instanceof Slot && nameToColumnDefinition.containsKey(((Slot) e).getName())) {
12221222
ColumnDefinition columnDefinition = nameToColumnDefinition.get(((Slot) e).getName());
12231223
if (columnDefinition.getAutoIncInitValue() != -1) {
12241224
throw new AnalysisException(

‎fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,8 @@
2929
import org.apache.doris.nereids.properties.DataTrait;
3030
import org.apache.doris.nereids.properties.PhysicalProperties;
3131
import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer;
32-
import org.apache.doris.nereids.rules.expression.ExpressionRewrite;
3332
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
34-
import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor;
3533
import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule;
36-
import org.apache.doris.nereids.rules.expression.rules.ReplaceVariableByLiteral;
3734
import org.apache.doris.nereids.rules.expression.rules.TrySimplifyPredicateWithMarkJoinSlot;
3835
import org.apache.doris.nereids.trees.SuperClassId;
3936
import org.apache.doris.nereids.trees.TreeNode;
@@ -1504,10 +1501,7 @@ public static Literal analyzeAndFoldToLiteral(ConnectContext ctx, Expression exp
15041501
throw new UserException(expression + " must be constant value");
15051502
}
15061503
ExpressionRewriteContext context = new ExpressionRewriteContext(cascadesContext);
1507-
ExpressionRuleExecutor executor = new ExpressionRuleExecutor(ImmutableList.of(
1508-
ExpressionRewrite.bottomUp(ReplaceVariableByLiteral.INSTANCE)));
1509-
Expression rewrittenExpression = executor.rewrite(analyzedExpr, context);
1510-
Expression foldExpression = FoldConstantRule.evaluate(rewrittenExpression, context);
1504+
Expression foldExpression = FoldConstantRule.evaluate(analyzedExpr, context);
15111505
if (foldExpression instanceof Literal) {
15121506
return (Literal) foldExpression;
15131507
} else {

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/UserVariableAnalysisTest.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@
2222
import org.apache.doris.analysis.LargeIntLiteral;
2323
import org.apache.doris.analysis.TimeStampNsLiteral;
2424
import org.apache.doris.catalog.ScalarType;
25+
import org.apache.doris.nereids.CascadesContext;
26+
import org.apache.doris.nereids.analyzer.Scope;
27+
import org.apache.doris.nereids.analyzer.UnboundVariable;
28+
import org.apache.doris.nereids.analyzer.UnboundVariable.VariableType;
29+
import org.apache.doris.nereids.trees.expressions.Expression;
30+
import org.apache.doris.nereids.trees.expressions.Variable;
2531
import org.apache.doris.nereids.trees.expressions.literal.Literal;
2632
import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral;
2733
import org.apache.doris.nereids.types.BigIntType;
@@ -34,9 +40,12 @@
3440
import org.apache.doris.nereids.util.MemoTestUtils;
3541
import org.apache.doris.qe.ConnectContext;
3642

43+
import com.google.common.collect.ImmutableList;
3744
import org.junit.jupiter.api.Assertions;
3845
import org.junit.jupiter.api.Test;
3946

47+
import java.util.List;
48+
4049
/** Tests for user variable handling in expression analysis. */
4150
public class UserVariableAnalysisTest {
4251

@@ -84,4 +93,23 @@ public void testUserVarTimeStampNsTypeAndPrecision() {
8493
Assertions.assertEquals(TimeStampNsType.INSTANCE, literal.getDataType());
8594
Assertions.assertEquals("2024-02-29 12:34:56.123456789", literal.getStringValue());
8695
}
96+
97+
@Test
98+
public void testBindUserVariableToRealExpressionAndRecordSqlCacheDependency() {
99+
ConnectContext ctx = MemoTestUtils.createConnectContext();
100+
ctx.setUserVar("v", new IntLiteral(42));
101+
CascadesContext cascadesContext = MemoTestUtils.createCascadesContext(ctx, "select @v");
102+
ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, new Scope(ImmutableList.of()),
103+
cascadesContext, false, false);
104+
105+
Expression analyzed = analyzer.analyze(new UnboundVariable("v", VariableType.USER));
106+
107+
Assertions.assertInstanceOf(Literal.class, analyzed);
108+
List<Variable> usedVariables = cascadesContext.getStatementContext().getSqlCacheContext()
109+
.orElseThrow().getUsedVariables();
110+
Assertions.assertEquals(1, usedVariables.size());
111+
Assertions.assertEquals("v", usedVariables.get(0).getName());
112+
Assertions.assertEquals(VariableType.USER, usedVariables.get(0).getType());
113+
Assertions.assertEquals(analyzed, usedVariables.get(0).getRealExpression());
114+
}
87115
}

‎fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/ExpressionTraitTest.java‎

Lines changed: 0 additions & 86 deletions
This file was deleted.

0 commit comments

Comments
 (0)