Skip to content

Commit 81f3c48

Browse files
authored
[enhancement](nereids) improve lots of values in insert into values statement (apache#40202)
improve lots of values in `insert into values` statement by bypass NereidsPlanner the main logic is 1. `InsertUtils.normalizePlan` use `FoldConstantRuleOnFE` to reduce the expression, e.g. `values(date(now())` 2. `FastInsertIntoValuesPlanner` skip most of rules to analyze and rewrite `LogicalInlineTable` to `LogicalUnion` or `LogicalOneRowRelation` 3. fast parse date time string without date format 4. getHintMap and normal lexer share the same tokens 5. `set enable_fast_analyze_into_values=false` can force to execute all optimize rules, when we meet some bugs in `FastInsertIntoValuesPlanner` test: insert 1000 rows with 1000 columns, the columns contains int, bigint, decimal(26,7), date, datetime, varchar(10 chinese chars) +---------------------------------+------------------------------------------------------+--------------------------+--------------------------+ |FastInsertIntoValuesPlanner |NereidsPlanner(enable_fast_analyze_into_values=false) |Legacy optimizer in 2.1.6 | Nereids planner in 2.1.6 | +---------------------------------+------------------------------------------------------+--------------------------+--------------------------+ |16s(bottleneck is antlr's lexer) |32s |16s |80s | +---------------------------------+------------------------------------------------------+--------------------------+--------------------------+ If you use FastInsertIntoValuesPlanner with group commit in a transaction, the time can reduce to 12s. TODO: build a custom lexer. in my hand write lexer test, FastInsertIntoValuesPlanner without group commit can reduce 16s to 12s, but it will take more effort: RegularExpression -> NFA -> DFA -> minimal DFA -> Lexer codegen
1 parent 1853d15 commit 81f3c48

48 files changed

Lines changed: 1217 additions & 228 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/http/http_channel.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,8 @@ void HttpChannel::send_files(HttpRequest* request, const std::string& root_dir,
123123
VLOG_DEBUG << "http channel send file " << file_path << ", size: " << file_size;
124124

125125
evbuffer_add_printf(evb.get(), "File-Name: %s\r\n", file.c_str());
126-
evbuffer_add_printf(evb.get(), "Content-Length: %ld\r\n", file_size);
126+
evbuffer_add_printf(evb.get(), "Content-Length: %" PRIi64 "\r\n", file_size);
127+
127128
evbuffer_add_printf(evb.get(), "\r\n");
128129
if (file_size > 0) {
129130
evbuffer_add_file(evb.get(), fd, 0, file_size);

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1476,7 +1476,9 @@ rowConstructor
14761476
;
14771477

14781478
rowConstructorItem
1479-
: namedExpression | DEFAULT
1479+
: constant // duplicate constant rule for improve the parse of `insert into tbl values`
1480+
| DEFAULT
1481+
| namedExpression
14801482
;
14811483

14821484
predicate
@@ -1678,7 +1680,7 @@ constant
16781680
| LEFT_BRACE (items+=constant COLON items+=constant)?
16791681
(COMMA items+=constant COLON items+=constant)* RIGHT_BRACE #mapLiteral
16801682
| LEFT_BRACE items+=constant (COMMA items+=constant)* RIGHT_BRACE #structLiteral
1681-
| PLACEHOLDER #placeholder
1683+
| PLACEHOLDER #placeholder
16821684
;
16831685

16841686
comparisonOperator

‎fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -473,9 +473,16 @@ public void setCurrentRootRewriteJobContext(RootRewriteJobContext currentRootRew
473473
this.currentRootRewriteJobContext = Optional.ofNullable(currentRootRewriteJobContext);
474474
}
475475

476+
/** showPlanProcess */
476477
public boolean showPlanProcess() {
477478
Boolean show = showPlanProcess.get();
478-
return show != null && show;
479+
if (show != null && show) {
480+
return true;
481+
}
482+
if (parent.isPresent()) {
483+
return parent.get().showPlanProcess();
484+
}
485+
return false;
479486
}
480487

481488
/** set showPlanProcess in task scope */

‎fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -98,17 +98,19 @@
9898
*/
9999
public class NereidsPlanner extends Planner {
100100
public static final Logger LOG = LogManager.getLogger(NereidsPlanner.class);
101+
102+
protected Plan parsedPlan;
103+
protected Plan analyzedPlan;
104+
protected Plan rewrittenPlan;
105+
protected Plan optimizedPlan;
106+
protected PhysicalPlan physicalPlan;
107+
101108
private CascadesContext cascadesContext;
102109
private final StatementContext statementContext;
103110
private final List<ScanNode> scanNodeList = Lists.newArrayList();
104111
private final List<PhysicalRelation> physicalRelations = Lists.newArrayList();
105112
private DescriptorTable descTable;
106113

107-
private Plan parsedPlan;
108-
private Plan analyzedPlan;
109-
private Plan rewrittenPlan;
110-
private Plan optimizedPlan;
111-
private PhysicalPlan physicalPlan;
112114
private FragmentIdMapping<DistributedPlan> distributedPlans;
113115
// The cost of optimized plan
114116
private double cost = 0;
@@ -552,7 +554,7 @@ public Group getRoot() {
552554
return cascadesContext.getMemo().getRoot();
553555
}
554556

555-
private PhysicalPlan chooseNthPlan(Group rootGroup, PhysicalProperties physicalProperties, int nthPlan) {
557+
protected PhysicalPlan chooseNthPlan(Group rootGroup, PhysicalProperties physicalProperties, int nthPlan) {
556558
if (nthPlan <= 1) {
557559
cost = rootGroup.getLowestCostPlan(physicalProperties).orElseThrow(
558560
() -> new AnalysisException("lowestCostPlans with physicalProperties("
@@ -605,6 +607,9 @@ private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties physical
605607
}
606608

607609
private long getGarbageCollectionTime() {
610+
if (!ConnectContext.get().getSessionVariable().enableProfile()) {
611+
return 0;
612+
}
608613
List<GarbageCollectorMXBean> gcMxBeans = ManagementFactory.getGarbageCollectorMXBeans();
609614
long initialGCTime = 0;
610615
for (GarbageCollectorMXBean gcBean : gcMxBeans) {
@@ -881,7 +886,7 @@ private boolean showPlanProcess(ExplainOptions explainOptions) {
881886
return explainOptions != null && explainOptions.showPlanProcess();
882887
}
883888

884-
private void keepOrShowPlanProcess(boolean showPlanProcess, Runnable task) {
889+
protected void keepOrShowPlanProcess(boolean showPlanProcess, Runnable task) {
885890
if (showPlanProcess) {
886891
cascadesContext.withPlanProcess(showPlanProcess, task);
887892
} else {
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.analyzer;
19+
20+
import org.apache.doris.nereids.exceptions.UnboundException;
21+
import org.apache.doris.nereids.memo.GroupExpression;
22+
import org.apache.doris.nereids.properties.LogicalProperties;
23+
import org.apache.doris.nereids.trees.expressions.Expression;
24+
import org.apache.doris.nereids.trees.expressions.NamedExpression;
25+
import org.apache.doris.nereids.trees.expressions.Slot;
26+
import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation;
27+
import org.apache.doris.nereids.trees.plans.Plan;
28+
import org.apache.doris.nereids.trees.plans.PlanType;
29+
import org.apache.doris.nereids.trees.plans.algebra.InlineTable;
30+
import org.apache.doris.nereids.trees.plans.logical.LogicalLeaf;
31+
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
32+
import org.apache.doris.nereids.util.Utils;
33+
34+
import com.google.common.collect.ImmutableList;
35+
36+
import java.util.List;
37+
import java.util.Objects;
38+
import java.util.Optional;
39+
40+
/** UnboundInlineTable */
41+
public class UnboundInlineTable extends LogicalLeaf implements InlineTable, BlockFuncDepsPropagation, UnboundPlan {
42+
private final List<List<NamedExpression>> constantExprsList;
43+
44+
public UnboundInlineTable(List<List<NamedExpression>> constantExprsList) {
45+
super(PlanType.LOGICAL_UNBOUND_INLINE_TABLE, Optional.empty(), Optional.empty());
46+
this.constantExprsList = Utils.fastToImmutableList(
47+
Objects.requireNonNull(constantExprsList, "constantExprsList can not be null")
48+
);
49+
}
50+
51+
public List<List<NamedExpression>> getConstantExprsList() {
52+
return constantExprsList;
53+
}
54+
55+
@Override
56+
public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
57+
return visitor.visitUnboundInlineTable(this, context);
58+
}
59+
60+
@Override
61+
public List<? extends Expression> getExpressions() {
62+
ImmutableList.Builder<Expression> expressions = ImmutableList.builderWithExpectedSize(
63+
constantExprsList.size() * constantExprsList.get(0).size());
64+
65+
for (List<NamedExpression> namedExpressions : constantExprsList) {
66+
expressions.addAll(namedExpressions);
67+
}
68+
69+
return expressions.build();
70+
}
71+
72+
@Override
73+
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
74+
return this;
75+
}
76+
77+
@Override
78+
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
79+
Optional<LogicalProperties> logicalProperties, List<Plan> children) {
80+
return this;
81+
}
82+
83+
@Override
84+
public List<Slot> computeOutput() {
85+
throw new UnboundException("output");
86+
}
87+
}
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.analyzer;
19+
20+
import org.apache.doris.nereids.exceptions.UnboundException;
21+
import org.apache.doris.nereids.properties.LogicalProperties;
22+
import org.apache.doris.nereids.properties.UnboundLogicalProperties;
23+
import org.apache.doris.nereids.trees.expressions.Slot;
24+
import org.apache.doris.nereids.trees.plans.Plan;
25+
26+
import java.util.List;
27+
28+
/** UnboundPlan */
29+
public interface UnboundPlan extends Plan {
30+
@Override
31+
default LogicalProperties computeLogicalProperties() {
32+
return UnboundLogicalProperties.INSTANCE;
33+
}
34+
35+
@Override
36+
default List<Slot> computeOutput() {
37+
throw new UnboundException("output");
38+
}
39+
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434

3535
import com.google.common.base.Preconditions;
3636
import com.google.common.collect.ImmutableList;
37+
import org.apache.commons.lang3.StringUtils;
3738

3839
import java.util.List;
3940
import java.util.Objects;
@@ -176,4 +177,12 @@ public LogicalProperties computeLogicalProperties() {
176177
public List<Slot> computeOutput() {
177178
throw new UnboundException("output");
178179
}
180+
181+
@Override
182+
public String toString() {
183+
return Utils.toSqlString("UnboundTableSink",
184+
"nameParts", StringUtils.join(nameParts, "."),
185+
"colNames", colNames,
186+
"hints", hints);
187+
}
179188
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,7 @@
207207

208208
import com.google.common.base.Preconditions;
209209
import com.google.common.collect.ImmutableList;
210+
import com.google.common.collect.ImmutableList.Builder;
210211
import com.google.common.collect.ImmutableSet;
211212
import com.google.common.collect.Lists;
212213
import com.google.common.collect.Maps;
@@ -2058,17 +2059,22 @@ public PlanFragment visitPhysicalSetOperation(
20582059
}
20592060
setOperationNode.setNereidsId(setOperation.getId());
20602061

2061-
setOperation.getRegularChildrenOutputs().stream()
2062-
.map(o -> o.stream()
2063-
.map(e -> ExpressionTranslator.translate(e, context))
2064-
.collect(ImmutableList.toImmutableList()))
2065-
.forEach(setOperationNode::addResultExprLists);
2062+
for (List<SlotReference> regularChildrenOutput : setOperation.getRegularChildrenOutputs()) {
2063+
Builder<Expr> translateOutputs = ImmutableList.builderWithExpectedSize(regularChildrenOutput.size());
2064+
for (SlotReference childOutput : regularChildrenOutput) {
2065+
translateOutputs.add(ExpressionTranslator.translate(childOutput, context));
2066+
}
2067+
setOperationNode.addResultExprLists(translateOutputs.build());
2068+
}
2069+
20662070
if (setOperation instanceof PhysicalUnion) {
2067-
((PhysicalUnion) setOperation).getConstantExprsList().stream()
2068-
.map(l -> l.stream()
2069-
.map(e -> ExpressionTranslator.translate(e, context))
2070-
.collect(ImmutableList.toImmutableList()))
2071-
.forEach(setOperationNode::addConstExprList);
2071+
for (List<NamedExpression> unionConsts : ((PhysicalUnion) setOperation).getConstantExprsList()) {
2072+
Builder<Expr> translateConsts = ImmutableList.builderWithExpectedSize(unionConsts.size());
2073+
for (NamedExpression unionConst : unionConsts) {
2074+
translateConsts.add(ExpressionTranslator.translate(unionConst, context));
2075+
}
2076+
setOperationNode.addConstExprList(translateConsts.build());
2077+
}
20722078
}
20732079

20742080
for (PlanFragment childFragment : childrenFragments) {

0 commit comments

Comments
 (0)