3636import org .apache .doris .datasource .metacache .MetaCache ;
3737import org .apache .doris .datasource .mvcc .MvccUtil ;
3838import org .apache .doris .datasource .paimon .PaimonMetadataCache ;
39- import org .apache .doris .datasource .paimon .PaimonMetadataCacheMgr ;
4039import org .apache .doris .fs .FileSystemCache ;
4140import org .apache .doris .nereids .exceptions .NotSupportedException ;
4241
@@ -87,21 +86,17 @@ public class ExternalMetaCacheMgr {
8786 // This executor is used to schedule the getting split tasks
8887 private ExecutorService scheduleExecutor ;
8988
90- // catalog id -> HiveMetaStoreCache
91- private final Map <Long , HiveMetaStoreCache > hiveCacheMap = Maps .newConcurrentMap ();
92-
93- // catalog id -> IcebergMetadataCache
94- private final Map <Long , IcebergMetadataCache > icebergCacheMap = Maps .newConcurrentMap ();
95- // catalog id -> table schema cache
96- private final Map <Long , ExternalSchemaCache > schemaCacheMap = Maps .newHashMap ();
89+ private final CatalogScopedCacheMgr <HiveMetaStoreCache > hiveMetaStoreCacheMgr ;
90+ private final CatalogScopedCacheMgr <IcebergMetadataCache > icebergMetadataCacheMgr ;
91+ private final CatalogScopedCacheMgr <PaimonMetadataCache > paimonMetadataCacheMgr ;
92+ private final CatalogScopedCacheMgr <ExternalSchemaCache > schemaCacheMgr ;
9793 // hudi partition manager
9894 private final HudiMetadataCacheMgr hudiMetadataCacheMgr ;
9995 // all catalogs could share the same fsCache.
10096 private FileSystemCache fsCache ;
10197 // all external table row count cache.
10298 private ExternalRowCountCache rowCountCache ;
10399 private final MaxComputeMetadataCacheMgr maxComputeMetadataCacheMgr ;
104- private final PaimonMetadataCacheMgr paimonMetadataCacheMgr ;
105100 private final DorisExternalMetaCacheMgr dorisExternalMetaCacheMgr ;
106101
107102 public ExternalMetaCacheMgr (boolean isCheckpointCatalog ) {
@@ -132,7 +127,15 @@ public ExternalMetaCacheMgr(boolean isCheckpointCatalog) {
132127
133128 hudiMetadataCacheMgr = new HudiMetadataCacheMgr (commonRefreshExecutor );
134129 maxComputeMetadataCacheMgr = new MaxComputeMetadataCacheMgr ();
135- paimonMetadataCacheMgr = new PaimonMetadataCacheMgr (commonRefreshExecutor );
130+ hiveMetaStoreCacheMgr = new CatalogScopedCacheMgr <>(
131+ catalog -> new HiveMetaStoreCache ((HMSExternalCatalog ) catalog ,
132+ commonRefreshExecutor , fileListingExecutor ));
133+ icebergMetadataCacheMgr = new CatalogScopedCacheMgr <>(
134+ catalog -> new IcebergMetadataCache (catalog , commonRefreshExecutor ));
135+ schemaCacheMgr = new CatalogScopedCacheMgr <>(
136+ catalog -> new ExternalSchemaCache (catalog , commonRefreshExecutor ));
137+ paimonMetadataCacheMgr = new CatalogScopedCacheMgr <>(
138+ catalog -> new PaimonMetadataCache (catalog , commonRefreshExecutor ));
136139 dorisExternalMetaCacheMgr = new DorisExternalMetaCacheMgr (commonRefreshExecutor );
137140 }
138141
@@ -160,30 +163,11 @@ public ExecutorService getScheduleExecutor() {
160163 }
161164
162165 public HiveMetaStoreCache getMetaStoreCache (HMSExternalCatalog catalog ) {
163- HiveMetaStoreCache cache = hiveCacheMap .get (catalog .getId ());
164- if (cache == null ) {
165- synchronized (hiveCacheMap ) {
166- if (!hiveCacheMap .containsKey (catalog .getId ())) {
167- hiveCacheMap .put (catalog .getId (),
168- new HiveMetaStoreCache (catalog , commonRefreshExecutor , fileListingExecutor ));
169- }
170- cache = hiveCacheMap .get (catalog .getId ());
171- }
172- }
173- return cache ;
166+ return hiveMetaStoreCacheMgr .getCache (catalog );
174167 }
175168
176169 public ExternalSchemaCache getSchemaCache (ExternalCatalog catalog ) {
177- ExternalSchemaCache cache = schemaCacheMap .get (catalog .getId ());
178- if (cache == null ) {
179- synchronized (schemaCacheMap ) {
180- if (!schemaCacheMap .containsKey (catalog .getId ())) {
181- schemaCacheMap .put (catalog .getId (), new ExternalSchemaCache (catalog , commonRefreshExecutor ));
182- }
183- cache = schemaCacheMap .get (catalog .getId ());
184- }
185- }
186- return cache ;
170+ return schemaCacheMgr .getCache (catalog );
187171 }
188172
189173 public HudiPartitionProcessor getHudiPartitionProcess (ExternalCatalog catalog ) {
@@ -203,20 +187,11 @@ public HudiMetadataCacheMgr getHudiMetadataCacheMgr() {
203187 }
204188
205189 public IcebergMetadataCache getIcebergMetadataCache (ExternalCatalog catalog ) {
206- IcebergMetadataCache cache = icebergCacheMap .get (catalog .getId ());
207- if (cache == null ) {
208- synchronized (icebergCacheMap ) {
209- if (!icebergCacheMap .containsKey (catalog .getId ())) {
210- icebergCacheMap .put (catalog .getId (), new IcebergMetadataCache (catalog , commonRefreshExecutor ));
211- }
212- cache = icebergCacheMap .get (catalog .getId ());
213- }
214- }
215- return cache ;
190+ return icebergMetadataCacheMgr .getCache (catalog );
216191 }
217192
218- public PaimonMetadataCache getPaimonMetadataCache () {
219- return paimonMetadataCacheMgr .getPaimonMetadataCache ( );
193+ public PaimonMetadataCache getPaimonMetadataCache (ExternalCatalog catalog ) {
194+ return paimonMetadataCacheMgr .getCache ( catalog );
220195 }
221196
222197 public MaxComputeMetadataCache getMaxComputeMetadataCache (long catalogId ) {
@@ -236,41 +211,43 @@ public DorisExternalMetaCacheMgr getDorisExternalMetaCacheMgr() {
236211 }
237212
238213 public void removeCache (long catalogId ) {
239- if (hiveCacheMap . remove (catalogId ) != null ) {
214+ if (hiveMetaStoreCacheMgr . removeCache (catalogId ) != null ) {
240215 LOG .info ("remove hive metastore cache for catalog {}" , catalogId );
241216 }
242- synchronized (schemaCacheMap ) {
243- if (schemaCacheMap .remove (catalogId ) != null ) {
244- LOG .info ("remove schema cache for catalog {}" , catalogId );
245- }
217+ if (schemaCacheMgr .removeCache (catalogId ) != null ) {
218+ LOG .info ("remove schema cache for catalog {}" , catalogId );
246219 }
247- if (icebergCacheMap . remove (catalogId ) != null ) {
220+ if (icebergMetadataCacheMgr . removeCache (catalogId ) != null ) {
248221 LOG .info ("remove iceberg meta cache for catalog {}" , catalogId );
249222 }
250223 hudiMetadataCacheMgr .removeCache (catalogId );
251224 maxComputeMetadataCacheMgr .removeCache (catalogId );
252- paimonMetadataCacheMgr .removeCache (catalogId );
225+ PaimonMetadataCache paimonMetadataCache = paimonMetadataCacheMgr .removeCache (catalogId );
226+ if (paimonMetadataCache != null ) {
227+ paimonMetadataCache .invalidateCatalogCache (catalogId );
228+ }
253229 dorisExternalMetaCacheMgr .removeCache (catalogId );
254230 }
255231
256232 public void invalidateTableCache (ExternalTable dorisTable ) {
257- synchronized (schemaCacheMap ) {
258- ExternalSchemaCache schemaCache = schemaCacheMap .get (dorisTable .getCatalog ().getId ());
259- if (schemaCache != null ) {
260- schemaCache .invalidateTableCache (dorisTable );
261- }
233+ ExternalSchemaCache schemaCache = schemaCacheMgr .getCache (dorisTable .getCatalog ().getId ());
234+ if (schemaCache != null ) {
235+ schemaCache .invalidateTableCache (dorisTable );
262236 }
263- HiveMetaStoreCache hiveMetaCache = hiveCacheMap . get (dorisTable .getCatalog ().getId ());
237+ HiveMetaStoreCache hiveMetaCache = hiveMetaStoreCacheMgr . getCache (dorisTable .getCatalog ().getId ());
264238 if (hiveMetaCache != null ) {
265239 hiveMetaCache .invalidateTableCache (dorisTable .getOrBuildNameMapping ());
266240 }
267- IcebergMetadataCache icebergMetadataCache = icebergCacheMap . get (dorisTable .getCatalog ().getId ());
241+ IcebergMetadataCache icebergMetadataCache = icebergMetadataCacheMgr . getCache (dorisTable .getCatalog ().getId ());
268242 if (icebergMetadataCache != null ) {
269243 icebergMetadataCache .invalidateTableCache (dorisTable );
270244 }
271245 hudiMetadataCacheMgr .invalidateTableCache (dorisTable );
272246 maxComputeMetadataCacheMgr .invalidateTableCache (dorisTable );
273- paimonMetadataCacheMgr .invalidateTableCache (dorisTable );
247+ PaimonMetadataCache paimonMetadataCache = paimonMetadataCacheMgr .getCache (dorisTable .getCatalog ().getId ());
248+ if (paimonMetadataCache != null ) {
249+ paimonMetadataCache .invalidateTableCache (dorisTable );
250+ }
274251 if (LOG .isDebugEnabled ()) {
275252 LOG .debug ("invalid table cache for {}.{} in catalog {}" , dorisTable .getRemoteDbName (),
276253 dorisTable .getRemoteName (), dorisTable .getCatalog ().getName ());
@@ -279,58 +256,58 @@ public void invalidateTableCache(ExternalTable dorisTable) {
279256
280257 public void invalidateDbCache (long catalogId , String dbName ) {
281258 dbName = ClusterNamespace .getNameFromFullName (dbName );
282- synchronized (schemaCacheMap ) {
283- ExternalSchemaCache schemaCache = schemaCacheMap .get (catalogId );
284- if (schemaCache != null ) {
285- schemaCache .invalidateDbCache (dbName );
286- }
259+ ExternalSchemaCache schemaCache = schemaCacheMgr .getCache (catalogId );
260+ if (schemaCache != null ) {
261+ schemaCache .invalidateDbCache (dbName );
287262 }
288- HiveMetaStoreCache metaCache = hiveCacheMap . get (catalogId );
263+ HiveMetaStoreCache metaCache = hiveMetaStoreCacheMgr . getCache (catalogId );
289264 if (metaCache != null ) {
290265 metaCache .invalidateDbCache (dbName );
291266 }
292- IcebergMetadataCache icebergMetadataCache = icebergCacheMap . get (catalogId );
267+ IcebergMetadataCache icebergMetadataCache = icebergMetadataCacheMgr . getCache (catalogId );
293268 if (icebergMetadataCache != null ) {
294269 icebergMetadataCache .invalidateDbCache (catalogId , dbName );
295270 }
296271 hudiMetadataCacheMgr .invalidateDbCache (catalogId , dbName );
297272 maxComputeMetadataCacheMgr .invalidateDbCache (catalogId , dbName );
298- paimonMetadataCacheMgr .invalidateDbCache (catalogId , dbName );
273+ PaimonMetadataCache paimonMetadataCache = paimonMetadataCacheMgr .getCache (catalogId );
274+ if (paimonMetadataCache != null ) {
275+ paimonMetadataCache .invalidateDbCache (catalogId , dbName );
276+ }
299277 if (LOG .isDebugEnabled ()) {
300278 LOG .debug ("invalid db cache for {} in catalog {}" , dbName , catalogId );
301279 }
302280 }
303281
304282 public void invalidateCatalogCache (long catalogId ) {
305- synchronized (schemaCacheMap ) {
306- schemaCacheMap .remove (catalogId );
307- }
308- HiveMetaStoreCache metaCache = hiveCacheMap .get (catalogId );
283+ schemaCacheMgr .removeCache (catalogId );
284+ HiveMetaStoreCache metaCache = hiveMetaStoreCacheMgr .getCache (catalogId );
309285 if (metaCache != null ) {
310286 metaCache .invalidateAll ();
311287 }
312- IcebergMetadataCache icebergMetadataCache = icebergCacheMap . get (catalogId );
288+ IcebergMetadataCache icebergMetadataCache = icebergMetadataCacheMgr . getCache (catalogId );
313289 if (icebergMetadataCache != null ) {
314290 icebergMetadataCache .invalidateCatalogCache (catalogId );
315291 }
316292 hudiMetadataCacheMgr .invalidateCatalogCache (catalogId );
317293 maxComputeMetadataCacheMgr .invalidateCatalogCache (catalogId );
318- paimonMetadataCacheMgr .invalidateCatalogCache (catalogId );
294+ PaimonMetadataCache paimonMetadataCache = paimonMetadataCacheMgr .getCache (catalogId );
295+ if (paimonMetadataCache != null ) {
296+ paimonMetadataCache .invalidateCatalogCache (catalogId );
297+ }
319298 dorisExternalMetaCacheMgr .invalidateCatalogCache (catalogId );
320299 if (LOG .isDebugEnabled ()) {
321300 LOG .debug ("invalid catalog cache for {}" , catalogId );
322301 }
323302 }
324303
325304 public void invalidSchemaCache (long catalogId ) {
326- synchronized (schemaCacheMap ) {
327- schemaCacheMap .remove (catalogId );
328- }
305+ schemaCacheMgr .removeCache (catalogId );
329306 }
330307
331308 public void addPartitionsCache (long catalogId , HMSExternalTable table , List <String > partitionNames ) {
332309 String dbName = ClusterNamespace .getNameFromFullName (table .getDbName ());
333- HiveMetaStoreCache metaCache = hiveCacheMap . get (catalogId );
310+ HiveMetaStoreCache metaCache = hiveMetaStoreCacheMgr . getCache (catalogId );
334311 if (metaCache != null ) {
335312 List <Type > partitionColumnTypes ;
336313 try {
@@ -348,7 +325,7 @@ public void addPartitionsCache(long catalogId, HMSExternalTable table, List<Stri
348325
349326 public void dropPartitionsCache (long catalogId , HMSExternalTable table , List <String > partitionNames ) {
350327 String dbName = ClusterNamespace .getNameFromFullName (table .getDbName ());
351- HiveMetaStoreCache metaCache = hiveCacheMap . get (catalogId );
328+ HiveMetaStoreCache metaCache = hiveMetaStoreCacheMgr . getCache (catalogId );
352329 if (metaCache != null ) {
353330 metaCache .dropPartitionsCache (table , partitionNames , true );
354331 }
@@ -358,7 +335,7 @@ public void dropPartitionsCache(long catalogId, HMSExternalTable table, List<Str
358335 }
359336
360337 public void invalidatePartitionsCache (ExternalTable dorisTable , List <String > partitionNames ) {
361- HiveMetaStoreCache metaCache = hiveCacheMap . get (dorisTable .getCatalog ().getId ());
338+ HiveMetaStoreCache metaCache = hiveMetaStoreCacheMgr . getCache (dorisTable .getCatalog ().getId ());
362339 if (metaCache != null ) {
363340 for (String partitionName : partitionNames ) {
364341 metaCache .invalidatePartitionCache (dorisTable , partitionName );
0 commit comments