Skip to content

Commit 90abd76

Browse files
authored
[fix](Nereids) fix some sql cache consistence bug between multiple frontends (#33722)
fix some sql cache consistence bug between multiple frontends which introduced by [enhancement](Nereids) Enable parse sql from sql cache #33262, fix by use row policy as the part of sql cache key. support dynamic update the num of fe manage sql cache key
1 parent d16b7ce commit 90abd76

17 files changed

Lines changed: 343 additions & 112 deletions

File tree

‎fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -744,15 +744,19 @@ public void toThrift(TTypeDesc container) {
744744
case DECIMAL128:
745745
case DECIMAL256:
746746
case DATETIMEV2: {
747-
Preconditions.checkArgument(precision >= scale,
748-
String.format("given precision %d is out of scale bound %d", precision, scale));
747+
if (precision < scale) {
748+
throw new IllegalArgumentException(
749+
String.format("given precision %d is out of scale bound %d", precision, scale));
750+
}
749751
scalarType.setScale(scale);
750752
scalarType.setPrecision(precision);
751753
break;
752754
}
753755
case TIMEV2: {
754-
Preconditions.checkArgument(precision >= scale,
755-
String.format("given precision %d is out of scale bound %d", precision, scale));
756+
if (precision < scale) {
757+
throw new IllegalArgumentException(
758+
String.format("given precision %d is out of scale bound %d", precision, scale));
759+
}
756760
scalarType.setScale(scale);
757761
scalarType.setPrecision(precision);
758762
break;

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1300,7 +1300,11 @@ public class Config extends ConfigBase {
13001300
* Minimum interval between last version when caching results,
13011301
* This parameter distinguishes between offline and real-time updates
13021302
*/
1303-
@ConfField(mutable = true, masterOnly = false)
1303+
@ConfField(
1304+
mutable = true,
1305+
masterOnly = false,
1306+
callbackClassString = "org.apache.doris.common.NereidsSqlCacheManager$UpdateConfig"
1307+
)
13041308
public static int cache_last_version_interval_second = 30;
13051309

13061310
/**
@@ -2011,10 +2015,16 @@ public class Config extends ConfigBase {
20112015
/**
20122016
* the plan cache num which can be reused for the next query
20132017
*/
2014-
@ConfField(mutable = false, varType = VariableAnnotation.EXPERIMENTAL, description = {
2015-
"当前默认设置为 100,用来控制控制NereidsSqlCacheManager管理的sql cache数量。",
2016-
"Now default set to 100, this config is used to control the number of "
2017-
+ "sql cache managed by NereidsSqlCacheManager"})
2018+
@ConfField(
2019+
mutable = true,
2020+
varType = VariableAnnotation.EXPERIMENTAL,
2021+
callbackClassString = "org.apache.doris.common.NereidsSqlCacheManager$UpdateConfig",
2022+
description = {
2023+
"当前默认设置为 100,用来控制控制NereidsSqlCacheManager管理的sql cache数量。",
2024+
"Now default set to 100, this config is used to control the number of "
2025+
+ "sql cache managed by NereidsSqlCacheManager"
2026+
}
2027+
)
20182028
public static int sql_cache_manage_num = 100;
20192029

20202030
/**

‎fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ public class ConfigBase {
5656

5757
Class<? extends ConfHandler> callback() default DefaultConfHandler.class;
5858

59+
String callbackClassString() default "";
60+
5961
// description for this config item.
6062
// There should be 2 elements in the array.
6163
// The first element is the description in Chinese.
@@ -329,6 +331,16 @@ public static synchronized void setMutableConfig(String key, String value) throw
329331
throw new ConfigException("Failed to set config '" + key + "'. err: " + e.getMessage());
330332
}
331333

334+
String callbackClassString = anno.callbackClassString();
335+
if (!Strings.isNullOrEmpty(callbackClassString)) {
336+
try {
337+
ConfHandler confHandler = (ConfHandler) Class.forName(anno.callbackClassString()).newInstance();
338+
confHandler.handle(field, value);
339+
} catch (Exception e) {
340+
throw new ConfigException("Failed to set config '" + key + "'. err: " + e.getMessage());
341+
}
342+
}
343+
332344
LOG.info("set config {} to {}", key, value);
333345
}
334346

‎fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java‎

Lines changed: 54 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,13 @@
1818
package org.apache.doris.common;
1919

2020
import org.apache.doris.analysis.UserIdentity;
21-
import org.apache.doris.catalog.Database;
2221
import org.apache.doris.catalog.DatabaseIf;
2322
import org.apache.doris.catalog.Env;
2423
import org.apache.doris.catalog.OlapTable;
2524
import org.apache.doris.catalog.Partition;
26-
import org.apache.doris.catalog.Table;
2725
import org.apache.doris.catalog.TableIf;
2826
import org.apache.doris.catalog.View;
27+
import org.apache.doris.common.ConfigBase.DefaultConfHandler;
2928
import org.apache.doris.datasource.CatalogIf;
3029
import org.apache.doris.metric.MetricRepo;
3130
import org.apache.doris.mysql.privilege.DataMaskPolicy;
@@ -50,6 +49,7 @@
5049
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
5150
import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache;
5251
import org.apache.doris.proto.InternalService;
52+
import org.apache.doris.proto.Types.PUniqueId;
5353
import org.apache.doris.qe.ConnectContext;
5454
import org.apache.doris.qe.cache.CacheAnalyzer;
5555
import org.apache.doris.qe.cache.SqlCache;
@@ -59,6 +59,7 @@
5959
import com.google.common.collect.ImmutableList;
6060
import org.apache.commons.collections.CollectionUtils;
6161

62+
import java.lang.reflect.Field;
6263
import java.time.Duration;
6364
import java.util.List;
6465
import java.util.Map.Entry;
@@ -69,11 +70,36 @@
6970
/** NereidsSqlCacheManager */
7071
public class NereidsSqlCacheManager {
7172
// key: <user>:<sql>
72-
// value: CacheAnalyzer
73-
private final Cache<String, SqlCacheContext> sqlCache;
73+
// value: SqlCacheContext
74+
private volatile Cache<String, SqlCacheContext> sqlCaches;
7475

7576
public NereidsSqlCacheManager(int sqlCacheNum, long cacheIntervalSeconds) {
76-
sqlCache = Caffeine.newBuilder()
77+
sqlCaches = buildSqlCaches(sqlCacheNum, cacheIntervalSeconds);
78+
}
79+
80+
public static synchronized void updateConfig() {
81+
Env currentEnv = Env.getCurrentEnv();
82+
if (currentEnv == null) {
83+
return;
84+
}
85+
NereidsSqlCacheManager sqlCacheManager = currentEnv.getSqlCacheManager();
86+
if (sqlCacheManager == null) {
87+
return;
88+
}
89+
90+
Cache<String, SqlCacheContext> sqlCaches = buildSqlCaches(
91+
Config.sql_cache_manage_num,
92+
Config.cache_last_version_interval_second
93+
);
94+
sqlCaches.putAll(sqlCacheManager.sqlCaches.asMap());
95+
sqlCacheManager.sqlCaches = sqlCaches;
96+
}
97+
98+
private static Cache<String, SqlCacheContext> buildSqlCaches(int sqlCacheNum, long cacheIntervalSeconds) {
99+
sqlCacheNum = sqlCacheNum <= 0 ? 100 : sqlCacheNum;
100+
cacheIntervalSeconds = cacheIntervalSeconds <= 0 ? 30 : cacheIntervalSeconds;
101+
102+
return Caffeine.newBuilder()
77103
.maximumSize(sqlCacheNum)
78104
.expireAfterAccess(Duration.ofSeconds(cacheIntervalSeconds))
79105
// auto evict cache when jvm memory too low
@@ -95,10 +121,9 @@ public void tryAddCache(
95121
SqlCacheContext sqlCacheContext = sqlCacheContextOpt.get();
96122
UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity();
97123
String key = currentUserIdentity.toString() + ":" + sql.trim();
98-
if (analyzer.getCache() instanceof SqlCache
99-
&& (currentMissParseSqlFromSqlCache || sqlCache.getIfPresent(key) == null)) {
124+
if ((currentMissParseSqlFromSqlCache || sqlCaches.getIfPresent(key) == null)
125+
&& sqlCacheContext.getOrComputeCacheKeyMd5() != null) {
100126
SqlCache cache = (SqlCache) analyzer.getCache();
101-
sqlCacheContext.setCacheKeyMd5(cache.getOrComputeCacheMd5());
102127
sqlCacheContext.setSumOfPartitionNum(cache.getSumOfPartitionNum());
103128
sqlCacheContext.setLatestPartitionId(cache.getLatestId());
104129
sqlCacheContext.setLatestPartitionVersion(cache.getLatestVersion());
@@ -109,23 +134,16 @@ public void tryAddCache(
109134
sqlCacheContext.addScanTable(scanTable);
110135
}
111136

112-
sqlCache.put(key, sqlCacheContext);
137+
sqlCaches.put(key, sqlCacheContext);
113138
}
114139
}
115140

116-
/** invalidateCache */
117-
public void invalidateCache(ConnectContext connectContext, String sql) {
118-
UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity();
119-
String key = currentUserIdentity.toString() + ":" + sql.trim();
120-
sqlCache.invalidate(key);
121-
}
122-
123141
/** tryParseSql */
124142
public Optional<LogicalSqlCache> tryParseSql(ConnectContext connectContext, String sql) {
125143
UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity();
126144
Env env = connectContext.getEnv();
127145
String key = currentUserIdentity.toString() + ":" + sql.trim();
128-
SqlCacheContext sqlCacheContext = sqlCache.getIfPresent(key);
146+
SqlCacheContext sqlCacheContext = sqlCaches.getIfPresent(key);
129147
if (sqlCacheContext == null) {
130148
return Optional.empty();
131149
}
@@ -161,16 +179,17 @@ public Optional<LogicalSqlCache> tryParseSql(ConnectContext connectContext, Stri
161179

162180
try {
163181
Status status = new Status();
182+
PUniqueId cacheKeyMd5 = sqlCacheContext.getOrComputeCacheKeyMd5();
164183
InternalService.PFetchCacheResult cacheData =
165184
SqlCache.getCacheData(sqlCacheContext.getCacheProxy(),
166-
sqlCacheContext.getCacheKeyMd5(), sqlCacheContext.getLatestPartitionId(),
185+
cacheKeyMd5, sqlCacheContext.getLatestPartitionId(),
167186
sqlCacheContext.getLatestPartitionVersion(), sqlCacheContext.getLatestPartitionTime(),
168187
sqlCacheContext.getSumOfPartitionNum(), status);
169188

170189
if (status.ok() && cacheData != null && cacheData.getStatus() == InternalService.PCacheStatus.CACHE_OK) {
171190
List<InternalService.PCacheValue> cacheValues = cacheData.getValuesList();
172191
String cachedPlan = sqlCacheContext.getPhysicalPlan();
173-
String backendAddress = SqlCache.findCacheBe(sqlCacheContext.getCacheKeyMd5()).getAddress();
192+
String backendAddress = SqlCache.findCacheBe(cacheKeyMd5).getAddress();
174193

175194
MetricRepo.COUNTER_CACHE_HIT_SQL.increase(1L);
176195

@@ -179,9 +198,9 @@ public Optional<LogicalSqlCache> tryParseSql(ConnectContext connectContext, Stri
179198
sqlCacheContext.getResultExprs(), cacheValues, backendAddress, cachedPlan);
180199
return Optional.of(logicalSqlCache);
181200
}
182-
return Optional.empty();
201+
return invalidateCache(key);
183202
} catch (Throwable t) {
184-
return Optional.empty();
203+
return invalidateCache(key);
185204
}
186205
}
187206

@@ -309,7 +328,8 @@ private boolean nondeterministicFunctionChanged(
309328
return true;
310329
}
311330

312-
List<Pair<Expression, Expression>> nondeterministicFunctions = sqlCacheContext.getFoldNondeterministicPairs();
331+
List<Pair<Expression, Expression>> nondeterministicFunctions
332+
= sqlCacheContext.getFoldFullNondeterministicPairs();
313333
if (nondeterministicFunctions.isEmpty()) {
314334
return false;
315335
}
@@ -328,21 +348,8 @@ private boolean nondeterministicFunctionChanged(
328348
return false;
329349
}
330350

331-
private boolean isValidDbAndTable(TableIf tableIf, Env env) {
332-
return getTableFromEnv(tableIf, env) != null;
333-
}
334-
335-
private TableIf getTableFromEnv(TableIf tableIf, Env env) {
336-
Optional<Database> db = env.getInternalCatalog().getDb(tableIf.getDatabase().getId());
337-
if (!db.isPresent()) {
338-
return null;
339-
}
340-
Optional<Table> table = db.get().getTable(tableIf.getId());
341-
return table.orElse(null);
342-
}
343-
344351
private Optional<LogicalSqlCache> invalidateCache(String key) {
345-
sqlCache.invalidate(key);
352+
sqlCaches.invalidate(key);
346353
return Optional.empty();
347354
}
348355

@@ -357,4 +364,15 @@ private TableIf findTableIf(Env env, FullTableName fullTableName) {
357364
}
358365
return db.get().getTable(fullTableName.table).orElse(null);
359366
}
367+
368+
// NOTE: used in Config.sql_cache_manage_num.callbackClassString and
369+
// Config.cache_last_version_interval_second.callbackClassString,
370+
// don't remove it!
371+
public static class UpdateConfig extends DefaultConfHandler {
372+
@Override
373+
public void handle(Field field, String confVal) throws Exception {
374+
super.handle(field, confVal);
375+
NereidsSqlCacheManager.updateConfig();
376+
}
377+
}
360378
}

0 commit comments

Comments
 (0)