Skip to content

Commit 03bd2a3

Browse files
authored
[enhancement](Nereids) Enable parse sql from sql cache (apache#33262)
Before this pr, the query must pass through parser, analyzer, rewriter, optimizer and translator, then we can check whether this query can use sql cache, if the query is too long, or the number of join tables too big, the plan time usually >= 500ms. This pr reduce this time by skip the fashion plan path, because we can reuse the previous physical plan and query result if no any changed. In some cases we should not parse sql from sql cache, e.g. table structure changed, data changed, user policies changed, privileges changed, contains non-deterministic functions, and user variables changed. In my test case: query a view which has lots of join and union, and the tables has empty partition, the query latency is about 3ms. if not parse sql from sql cache, the plan time is about 550ms ## Features 1. use Config.sql_cache_manage_num to control how many sql cache be reused in on fe 2. if explain plan appear some plans contains `LogicalSqlCache` or `PhysicalSqlCache`, it means the query can use sql cache, like this: ```sql mysql> set enable_sql_cache=true; Query OK, 0 rows affected (0.00 sec) mysql> explain physical plan select * from test.t; +----------------------------------------------------------------------------------+ | Explain String(Nereids Planner) | +----------------------------------------------------------------------------------+ | cost = 3.135 | | PhysicalResultSink[53] ( outputExprs=[c1#0, c2#1] ) | | +--PhysicalDistribute[50]@0 ( stats=3, distributionSpec=DistributionSpecGather ) | | +--PhysicalOlapScan[t]@0 ( stats=3 ) | +----------------------------------------------------------------------------------+ 4 rows in set (0.02 sec) mysql> select * from test.t; +------+------+ | c1 | c2 | +------+------+ | 1 | 2 | | -2 | -2 | | NULL | 30 | +------+------+ 3 rows in set (0.05 sec) mysql> explain physical plan select * from test.t; +-------------------------------------------------------------------------------------------+ | Explain String(Nereids Planner) | +-------------------------------------------------------------------------------------------+ | cost = 0.0 | | PhysicalSqlCache[2] ( queryId=78511f515cda466b-95385d892d6c68d0, backend=127.0.0.1:9050 ) | | +--PhysicalResultSink[52] ( outputExprs=[c1#0, c2#1] ) | | +--PhysicalDistribute[49]@0 ( stats=3, distributionSpec=DistributionSpecGather ) | | +--PhysicalOlapScan[t]@0 ( stats=3 ) | +-------------------------------------------------------------------------------------------+ 5 rows in set (0.01 sec) ```
1 parent c5a81e9 commit 03bd2a3

54 files changed

Lines changed: 3025 additions & 427 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@
1717

1818
package org.apache.doris.common;
1919

20-
import org.apache.doris.common.ConfigBase.ConfField;
21-
2220
public class Config extends ConfigBase {
2321

2422
@ConfField(description = {"用户自定义配置文件的路径,用于存放 fe_custom.conf。该文件中的配置会覆盖 fe.conf 中的配置",
@@ -2004,6 +2002,14 @@ public class Config extends ConfigBase {
20042002
+ "the old load statement will be degraded."})
20052003
public static boolean enable_nereids_load = false;
20062004

2005+
/**
2006+
* the plan cache num which can be reused for the next query
2007+
*/
2008+
@ConfField(mutable = false, varType = VariableAnnotation.EXPERIMENTAL, description = {
2009+
"当前默认设置为 100,用来控制控制NereidsSqlCacheManager管理的sql cache数量。",
2010+
"Now default set to 100, this config is used to control the number of "
2011+
+ "sql cache managed by NereidsSqlCacheManager"})
2012+
public static int sql_cache_manage_num = 100;
20072013

20082014
/**
20092015
* Maximum number of events to poll in each RPC.

‎fe/fe-common/src/main/java/org/apache/doris/common/Pair.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ private Pair(F first, S second) {
4343
this.second = second;
4444
}
4545

46+
public static <P, K extends P> Pair<K, K> ofSame(K same) {
47+
return new Pair<>(same, same);
48+
}
49+
4650
public static <F, S> Pair<F, S> of(F first, S second) {
4751
return new Pair<>(first, second);
4852
}

‎fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -269,10 +269,10 @@ public void checkLimitations(Long partitionNum, Long tabletNum, Long cardinality
269269
return;
270270
}
271271
// match global rule
272-
List<SqlBlockRule> globalRules =
273-
nameToSqlBlockRuleMap.values().stream().filter(SqlBlockRule::getGlobal).collect(Collectors.toList());
274-
for (SqlBlockRule rule : globalRules) {
275-
checkLimitations(rule, partitionNum, tabletNum, cardinality);
272+
for (SqlBlockRule rule : nameToSqlBlockRuleMap.values()) {
273+
if (rule.getGlobal()) {
274+
checkLimitations(rule, partitionNum, tabletNum, cardinality);
275+
}
276276
}
277277
// match user rule
278278
String[] bindSqlBlockRules = Env.getCurrentEnv().getAuth().getSqlBlockRules(user);

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@
105105
import org.apache.doris.common.FeConstants;
106106
import org.apache.doris.common.FeMetaVersion;
107107
import org.apache.doris.common.MetaNotFoundException;
108+
import org.apache.doris.common.NereidsSqlCacheManager;
108109
import org.apache.doris.common.Pair;
109110
import org.apache.doris.common.ThreadPoolManager;
110111
import org.apache.doris.common.UserException;
@@ -530,6 +531,8 @@ public class Env {
530531

531532
private DNSCache dnsCache;
532533

534+
private final NereidsSqlCacheManager sqlCacheManager;
535+
533536
public List<TFrontendInfo> getFrontendInfos() {
534537
List<TFrontendInfo> res = new ArrayList<>();
535538

@@ -766,6 +769,9 @@ public Env(boolean isCheckpointCatalog) {
766769
this.mtmvService = new MTMVService();
767770
this.insertOverwriteManager = new InsertOverwriteManager();
768771
this.dnsCache = new DNSCache();
772+
this.sqlCacheManager = new NereidsSqlCacheManager(
773+
Config.sql_cache_manage_num, Config.cache_last_version_interval_second
774+
);
769775
}
770776

771777
public static void destroyCheckpoint() {
@@ -6092,6 +6098,10 @@ public MasterDaemon getTabletStatMgr() {
60926098
return tabletStatMgr;
60936099
}
60946100

6101+
public NereidsSqlCacheManager getSqlCacheManager() {
6102+
return sqlCacheManager;
6103+
}
6104+
60956105
public void alterMTMVRefreshInfo(AlterMTMVRefreshInfo info) {
60966106
AlterMTMV alter = new AlterMTMV(info.getMvName(), info.getRefreshInfo(), MTMVAlterOpType.ALTER_REFRESH_INFO);
60976107
this.alter.processAlterMTMV(alter, false);

0 commit comments

Comments
 (0)