1818package org .apache .doris .common ;
1919
2020import org .apache .doris .analysis .UserIdentity ;
21- import org .apache .doris .catalog .Database ;
2221import org .apache .doris .catalog .DatabaseIf ;
2322import org .apache .doris .catalog .Env ;
2423import org .apache .doris .catalog .OlapTable ;
2524import org .apache .doris .catalog .Partition ;
26- import org .apache .doris .catalog .Table ;
2725import org .apache .doris .catalog .TableIf ;
2826import org .apache .doris .catalog .View ;
27+ import org .apache .doris .common .ConfigBase .DefaultConfHandler ;
2928import org .apache .doris .datasource .CatalogIf ;
3029import org .apache .doris .metric .MetricRepo ;
3130import org .apache .doris .mysql .privilege .DataMaskPolicy ;
5049import org .apache .doris .nereids .trees .plans .logical .LogicalEmptyRelation ;
5150import org .apache .doris .nereids .trees .plans .logical .LogicalSqlCache ;
5251import org .apache .doris .proto .InternalService ;
52+ import org .apache .doris .proto .Types .PUniqueId ;
5353import org .apache .doris .qe .ConnectContext ;
5454import org .apache .doris .qe .cache .CacheAnalyzer ;
5555import org .apache .doris .qe .cache .SqlCache ;
5959import com .google .common .collect .ImmutableList ;
6060import org .apache .commons .collections .CollectionUtils ;
6161
62+ import java .lang .reflect .Field ;
6263import java .time .Duration ;
6364import java .util .List ;
6465import java .util .Map .Entry ;
6970/** NereidsSqlCacheManager */
7071public class NereidsSqlCacheManager {
7172 // key: <user>:<sql>
72- // value: CacheAnalyzer
73- private final Cache <String , SqlCacheContext > sqlCache ;
73+ // value: SqlCacheContext
74+ private volatile Cache <String , SqlCacheContext > sqlCaches ;
7475
7576 public NereidsSqlCacheManager (int sqlCacheNum , long cacheIntervalSeconds ) {
76- sqlCache = Caffeine .newBuilder ()
77+ sqlCaches = buildSqlCaches (sqlCacheNum , cacheIntervalSeconds );
78+ }
79+
80+ public static synchronized void updateConfig () {
81+ Env currentEnv = Env .getCurrentEnv ();
82+ if (currentEnv == null ) {
83+ return ;
84+ }
85+ NereidsSqlCacheManager sqlCacheManager = currentEnv .getSqlCacheManager ();
86+ if (sqlCacheManager == null ) {
87+ return ;
88+ }
89+
90+ Cache <String , SqlCacheContext > sqlCaches = buildSqlCaches (
91+ Config .sql_cache_manage_num ,
92+ Config .cache_last_version_interval_second
93+ );
94+ sqlCaches .putAll (sqlCacheManager .sqlCaches .asMap ());
95+ sqlCacheManager .sqlCaches = sqlCaches ;
96+ }
97+
98+ private static Cache <String , SqlCacheContext > buildSqlCaches (int sqlCacheNum , long cacheIntervalSeconds ) {
99+ sqlCacheNum = sqlCacheNum <= 0 ? 100 : sqlCacheNum ;
100+ cacheIntervalSeconds = cacheIntervalSeconds <= 0 ? 30 : cacheIntervalSeconds ;
101+
102+ return Caffeine .newBuilder ()
77103 .maximumSize (sqlCacheNum )
78104 .expireAfterAccess (Duration .ofSeconds (cacheIntervalSeconds ))
79105 // auto evict cache when jvm memory too low
@@ -95,10 +121,9 @@ public void tryAddCache(
95121 SqlCacheContext sqlCacheContext = sqlCacheContextOpt .get ();
96122 UserIdentity currentUserIdentity = connectContext .getCurrentUserIdentity ();
97123 String key = currentUserIdentity .toString () + ":" + sql .trim ();
98- if (analyzer . getCache () instanceof SqlCache
99- && ( currentMissParseSqlFromSqlCache || sqlCache . getIfPresent ( key ) == null ) ) {
124+ if (( currentMissParseSqlFromSqlCache || sqlCaches . getIfPresent ( key ) == null )
125+ && sqlCacheContext . getOrComputeCacheKeyMd5 () != null ) {
100126 SqlCache cache = (SqlCache ) analyzer .getCache ();
101- sqlCacheContext .setCacheKeyMd5 (cache .getOrComputeCacheMd5 ());
102127 sqlCacheContext .setSumOfPartitionNum (cache .getSumOfPartitionNum ());
103128 sqlCacheContext .setLatestPartitionId (cache .getLatestId ());
104129 sqlCacheContext .setLatestPartitionVersion (cache .getLatestVersion ());
@@ -109,23 +134,16 @@ public void tryAddCache(
109134 sqlCacheContext .addScanTable (scanTable );
110135 }
111136
112- sqlCache .put (key , sqlCacheContext );
137+ sqlCaches .put (key , sqlCacheContext );
113138 }
114139 }
115140
116- /** invalidateCache */
117- public void invalidateCache (ConnectContext connectContext , String sql ) {
118- UserIdentity currentUserIdentity = connectContext .getCurrentUserIdentity ();
119- String key = currentUserIdentity .toString () + ":" + sql .trim ();
120- sqlCache .invalidate (key );
121- }
122-
123141 /** tryParseSql */
124142 public Optional <LogicalSqlCache > tryParseSql (ConnectContext connectContext , String sql ) {
125143 UserIdentity currentUserIdentity = connectContext .getCurrentUserIdentity ();
126144 Env env = connectContext .getEnv ();
127145 String key = currentUserIdentity .toString () + ":" + sql .trim ();
128- SqlCacheContext sqlCacheContext = sqlCache .getIfPresent (key );
146+ SqlCacheContext sqlCacheContext = sqlCaches .getIfPresent (key );
129147 if (sqlCacheContext == null ) {
130148 return Optional .empty ();
131149 }
@@ -161,16 +179,17 @@ public Optional<LogicalSqlCache> tryParseSql(ConnectContext connectContext, Stri
161179
162180 try {
163181 Status status = new Status ();
182+ PUniqueId cacheKeyMd5 = sqlCacheContext .getOrComputeCacheKeyMd5 ();
164183 InternalService .PFetchCacheResult cacheData =
165184 SqlCache .getCacheData (sqlCacheContext .getCacheProxy (),
166- sqlCacheContext . getCacheKeyMd5 () , sqlCacheContext .getLatestPartitionId (),
185+ cacheKeyMd5 , sqlCacheContext .getLatestPartitionId (),
167186 sqlCacheContext .getLatestPartitionVersion (), sqlCacheContext .getLatestPartitionTime (),
168187 sqlCacheContext .getSumOfPartitionNum (), status );
169188
170189 if (status .ok () && cacheData != null && cacheData .getStatus () == InternalService .PCacheStatus .CACHE_OK ) {
171190 List <InternalService .PCacheValue > cacheValues = cacheData .getValuesList ();
172191 String cachedPlan = sqlCacheContext .getPhysicalPlan ();
173- String backendAddress = SqlCache .findCacheBe (sqlCacheContext . getCacheKeyMd5 () ).getAddress ();
192+ String backendAddress = SqlCache .findCacheBe (cacheKeyMd5 ).getAddress ();
174193
175194 MetricRepo .COUNTER_CACHE_HIT_SQL .increase (1L );
176195
@@ -179,9 +198,9 @@ public Optional<LogicalSqlCache> tryParseSql(ConnectContext connectContext, Stri
179198 sqlCacheContext .getResultExprs (), cacheValues , backendAddress , cachedPlan );
180199 return Optional .of (logicalSqlCache );
181200 }
182- return Optional . empty ( );
201+ return invalidateCache ( key );
183202 } catch (Throwable t ) {
184- return Optional . empty ( );
203+ return invalidateCache ( key );
185204 }
186205 }
187206
@@ -309,7 +328,8 @@ private boolean nondeterministicFunctionChanged(
309328 return true ;
310329 }
311330
312- List <Pair <Expression , Expression >> nondeterministicFunctions = sqlCacheContext .getFoldNondeterministicPairs ();
331+ List <Pair <Expression , Expression >> nondeterministicFunctions
332+ = sqlCacheContext .getFoldFullNondeterministicPairs ();
313333 if (nondeterministicFunctions .isEmpty ()) {
314334 return false ;
315335 }
@@ -328,21 +348,8 @@ private boolean nondeterministicFunctionChanged(
328348 return false ;
329349 }
330350
331- private boolean isValidDbAndTable (TableIf tableIf , Env env ) {
332- return getTableFromEnv (tableIf , env ) != null ;
333- }
334-
335- private TableIf getTableFromEnv (TableIf tableIf , Env env ) {
336- Optional <Database > db = env .getInternalCatalog ().getDb (tableIf .getDatabase ().getId ());
337- if (!db .isPresent ()) {
338- return null ;
339- }
340- Optional <Table > table = db .get ().getTable (tableIf .getId ());
341- return table .orElse (null );
342- }
343-
344351 private Optional <LogicalSqlCache > invalidateCache (String key ) {
345- sqlCache .invalidate (key );
352+ sqlCaches .invalidate (key );
346353 return Optional .empty ();
347354 }
348355
@@ -357,4 +364,15 @@ private TableIf findTableIf(Env env, FullTableName fullTableName) {
357364 }
358365 return db .get ().getTable (fullTableName .table ).orElse (null );
359366 }
367+
368+ // NOTE: used in Config.sql_cache_manage_num.callbackClassString and
369+ // Config.cache_last_version_interval_second.callbackClassString,
370+ // don't remove it!
371+ public static class UpdateConfig extends DefaultConfHandler {
372+ @ Override
373+ public void handle (Field field , String confVal ) throws Exception {
374+ super .handle (field , confVal );
375+ NereidsSqlCacheManager .updateConfig ();
376+ }
377+ }
360378}
0 commit comments