Skip to content

Commit 2c85148

Browse files
[refactor](paimon) Per-catalog Paimon metadata cache with two-level table+snapshot structure (apache#60478)
### What problem does this PR solve? ## Summary Refactor Paimon metadata cache from a single global instance to per-catalog instances, introduce a two-level Table+Snapshot cache structure, and unify TTL resolution logic across Iceberg/Paimon/Schema caches. ## Motivation The previous design shared a single `PaimonMetadataCache` instance and a single global `snapshotCache` across all Paimon catalogs. This caused: - Different catalogs could not independently configure cache TTL. - Cache keys had to carry `catalogId` for isolation; invalidation required scanning all keys and filtering. - `PaimonExternalTable` eagerly fetched the `Table` object at construction time, incurring remote calls even when the table was never subsequently accessed.
1 parent 8ade19f commit 2c85148

25 files changed

Lines changed: 856 additions & 622 deletions
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.datasource;
19+
20+
import java.util.Map;
21+
import java.util.concurrent.ConcurrentHashMap;
22+
import java.util.function.Function;
23+
24+
public class CatalogScopedCacheMgr<T> {
25+
private final Map<Long, T> cacheMap = new ConcurrentHashMap<>();
26+
private final Function<ExternalCatalog, T> cacheFactory;
27+
28+
public CatalogScopedCacheMgr(Function<ExternalCatalog, T> cacheFactory) {
29+
this.cacheFactory = cacheFactory;
30+
}
31+
32+
public T getCache(ExternalCatalog catalog) {
33+
return cacheMap.computeIfAbsent(catalog.getId(), id -> cacheFactory.apply(catalog));
34+
}
35+
36+
public T getCache(long catalogId) {
37+
return cacheMap.get(catalogId);
38+
}
39+
40+
public T removeCache(long catalogId) {
41+
return cacheMap.remove(catalogId);
42+
}
43+
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1450,4 +1450,3 @@ public void reorderColumns(TableIf dorisTable, List<String> newOrder) throws Use
14501450
}
14511451
}
14521452
}
1453-

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java‎

Lines changed: 55 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636
import org.apache.doris.datasource.metacache.MetaCache;
3737
import org.apache.doris.datasource.mvcc.MvccUtil;
3838
import org.apache.doris.datasource.paimon.PaimonMetadataCache;
39-
import org.apache.doris.datasource.paimon.PaimonMetadataCacheMgr;
4039
import org.apache.doris.fs.FileSystemCache;
4140
import org.apache.doris.nereids.exceptions.NotSupportedException;
4241

@@ -87,21 +86,17 @@ public class ExternalMetaCacheMgr {
8786
// This executor is used to schedule the getting split tasks
8887
private ExecutorService scheduleExecutor;
8988

90-
// catalog id -> HiveMetaStoreCache
91-
private final Map<Long, HiveMetaStoreCache> hiveCacheMap = Maps.newConcurrentMap();
92-
93-
// catalog id -> IcebergMetadataCache
94-
private final Map<Long, IcebergMetadataCache> icebergCacheMap = Maps.newConcurrentMap();
95-
// catalog id -> table schema cache
96-
private final Map<Long, ExternalSchemaCache> schemaCacheMap = Maps.newHashMap();
89+
private final CatalogScopedCacheMgr<HiveMetaStoreCache> hiveMetaStoreCacheMgr;
90+
private final CatalogScopedCacheMgr<IcebergMetadataCache> icebergMetadataCacheMgr;
91+
private final CatalogScopedCacheMgr<PaimonMetadataCache> paimonMetadataCacheMgr;
92+
private final CatalogScopedCacheMgr<ExternalSchemaCache> schemaCacheMgr;
9793
// hudi partition manager
9894
private final HudiMetadataCacheMgr hudiMetadataCacheMgr;
9995
// all catalogs could share the same fsCache.
10096
private FileSystemCache fsCache;
10197
// all external table row count cache.
10298
private ExternalRowCountCache rowCountCache;
10399
private final MaxComputeMetadataCacheMgr maxComputeMetadataCacheMgr;
104-
private final PaimonMetadataCacheMgr paimonMetadataCacheMgr;
105100
private final DorisExternalMetaCacheMgr dorisExternalMetaCacheMgr;
106101

107102
public ExternalMetaCacheMgr(boolean isCheckpointCatalog) {
@@ -132,7 +127,15 @@ public ExternalMetaCacheMgr(boolean isCheckpointCatalog) {
132127

133128
hudiMetadataCacheMgr = new HudiMetadataCacheMgr(commonRefreshExecutor);
134129
maxComputeMetadataCacheMgr = new MaxComputeMetadataCacheMgr();
135-
paimonMetadataCacheMgr = new PaimonMetadataCacheMgr(commonRefreshExecutor);
130+
hiveMetaStoreCacheMgr = new CatalogScopedCacheMgr<>(
131+
catalog -> new HiveMetaStoreCache((HMSExternalCatalog) catalog,
132+
commonRefreshExecutor, fileListingExecutor));
133+
icebergMetadataCacheMgr = new CatalogScopedCacheMgr<>(
134+
catalog -> new IcebergMetadataCache(catalog, commonRefreshExecutor));
135+
schemaCacheMgr = new CatalogScopedCacheMgr<>(
136+
catalog -> new ExternalSchemaCache(catalog, commonRefreshExecutor));
137+
paimonMetadataCacheMgr = new CatalogScopedCacheMgr<>(
138+
catalog -> new PaimonMetadataCache(catalog, commonRefreshExecutor));
136139
dorisExternalMetaCacheMgr = new DorisExternalMetaCacheMgr(commonRefreshExecutor);
137140
}
138141

@@ -160,30 +163,11 @@ public ExecutorService getScheduleExecutor() {
160163
}
161164

162165
public HiveMetaStoreCache getMetaStoreCache(HMSExternalCatalog catalog) {
163-
HiveMetaStoreCache cache = hiveCacheMap.get(catalog.getId());
164-
if (cache == null) {
165-
synchronized (hiveCacheMap) {
166-
if (!hiveCacheMap.containsKey(catalog.getId())) {
167-
hiveCacheMap.put(catalog.getId(),
168-
new HiveMetaStoreCache(catalog, commonRefreshExecutor, fileListingExecutor));
169-
}
170-
cache = hiveCacheMap.get(catalog.getId());
171-
}
172-
}
173-
return cache;
166+
return hiveMetaStoreCacheMgr.getCache(catalog);
174167
}
175168

176169
public ExternalSchemaCache getSchemaCache(ExternalCatalog catalog) {
177-
ExternalSchemaCache cache = schemaCacheMap.get(catalog.getId());
178-
if (cache == null) {
179-
synchronized (schemaCacheMap) {
180-
if (!schemaCacheMap.containsKey(catalog.getId())) {
181-
schemaCacheMap.put(catalog.getId(), new ExternalSchemaCache(catalog, commonRefreshExecutor));
182-
}
183-
cache = schemaCacheMap.get(catalog.getId());
184-
}
185-
}
186-
return cache;
170+
return schemaCacheMgr.getCache(catalog);
187171
}
188172

189173
public HudiPartitionProcessor getHudiPartitionProcess(ExternalCatalog catalog) {
@@ -203,20 +187,11 @@ public HudiMetadataCacheMgr getHudiMetadataCacheMgr() {
203187
}
204188

205189
public IcebergMetadataCache getIcebergMetadataCache(ExternalCatalog catalog) {
206-
IcebergMetadataCache cache = icebergCacheMap.get(catalog.getId());
207-
if (cache == null) {
208-
synchronized (icebergCacheMap) {
209-
if (!icebergCacheMap.containsKey(catalog.getId())) {
210-
icebergCacheMap.put(catalog.getId(), new IcebergMetadataCache(catalog, commonRefreshExecutor));
211-
}
212-
cache = icebergCacheMap.get(catalog.getId());
213-
}
214-
}
215-
return cache;
190+
return icebergMetadataCacheMgr.getCache(catalog);
216191
}
217192

218-
public PaimonMetadataCache getPaimonMetadataCache() {
219-
return paimonMetadataCacheMgr.getPaimonMetadataCache();
193+
public PaimonMetadataCache getPaimonMetadataCache(ExternalCatalog catalog) {
194+
return paimonMetadataCacheMgr.getCache(catalog);
220195
}
221196

222197
public MaxComputeMetadataCache getMaxComputeMetadataCache(long catalogId) {
@@ -236,41 +211,43 @@ public DorisExternalMetaCacheMgr getDorisExternalMetaCacheMgr() {
236211
}
237212

238213
public void removeCache(long catalogId) {
239-
if (hiveCacheMap.remove(catalogId) != null) {
214+
if (hiveMetaStoreCacheMgr.removeCache(catalogId) != null) {
240215
LOG.info("remove hive metastore cache for catalog {}", catalogId);
241216
}
242-
synchronized (schemaCacheMap) {
243-
if (schemaCacheMap.remove(catalogId) != null) {
244-
LOG.info("remove schema cache for catalog {}", catalogId);
245-
}
217+
if (schemaCacheMgr.removeCache(catalogId) != null) {
218+
LOG.info("remove schema cache for catalog {}", catalogId);
246219
}
247-
if (icebergCacheMap.remove(catalogId) != null) {
220+
if (icebergMetadataCacheMgr.removeCache(catalogId) != null) {
248221
LOG.info("remove iceberg meta cache for catalog {}", catalogId);
249222
}
250223
hudiMetadataCacheMgr.removeCache(catalogId);
251224
maxComputeMetadataCacheMgr.removeCache(catalogId);
252-
paimonMetadataCacheMgr.removeCache(catalogId);
225+
PaimonMetadataCache paimonMetadataCache = paimonMetadataCacheMgr.removeCache(catalogId);
226+
if (paimonMetadataCache != null) {
227+
paimonMetadataCache.invalidateCatalogCache(catalogId);
228+
}
253229
dorisExternalMetaCacheMgr.removeCache(catalogId);
254230
}
255231

256232
public void invalidateTableCache(ExternalTable dorisTable) {
257-
synchronized (schemaCacheMap) {
258-
ExternalSchemaCache schemaCache = schemaCacheMap.get(dorisTable.getCatalog().getId());
259-
if (schemaCache != null) {
260-
schemaCache.invalidateTableCache(dorisTable);
261-
}
233+
ExternalSchemaCache schemaCache = schemaCacheMgr.getCache(dorisTable.getCatalog().getId());
234+
if (schemaCache != null) {
235+
schemaCache.invalidateTableCache(dorisTable);
262236
}
263-
HiveMetaStoreCache hiveMetaCache = hiveCacheMap.get(dorisTable.getCatalog().getId());
237+
HiveMetaStoreCache hiveMetaCache = hiveMetaStoreCacheMgr.getCache(dorisTable.getCatalog().getId());
264238
if (hiveMetaCache != null) {
265239
hiveMetaCache.invalidateTableCache(dorisTable.getOrBuildNameMapping());
266240
}
267-
IcebergMetadataCache icebergMetadataCache = icebergCacheMap.get(dorisTable.getCatalog().getId());
241+
IcebergMetadataCache icebergMetadataCache = icebergMetadataCacheMgr.getCache(dorisTable.getCatalog().getId());
268242
if (icebergMetadataCache != null) {
269243
icebergMetadataCache.invalidateTableCache(dorisTable);
270244
}
271245
hudiMetadataCacheMgr.invalidateTableCache(dorisTable);
272246
maxComputeMetadataCacheMgr.invalidateTableCache(dorisTable);
273-
paimonMetadataCacheMgr.invalidateTableCache(dorisTable);
247+
PaimonMetadataCache paimonMetadataCache = paimonMetadataCacheMgr.getCache(dorisTable.getCatalog().getId());
248+
if (paimonMetadataCache != null) {
249+
paimonMetadataCache.invalidateTableCache(dorisTable);
250+
}
274251
if (LOG.isDebugEnabled()) {
275252
LOG.debug("invalid table cache for {}.{} in catalog {}", dorisTable.getRemoteDbName(),
276253
dorisTable.getRemoteName(), dorisTable.getCatalog().getName());
@@ -279,58 +256,58 @@ public void invalidateTableCache(ExternalTable dorisTable) {
279256

280257
public void invalidateDbCache(long catalogId, String dbName) {
281258
dbName = ClusterNamespace.getNameFromFullName(dbName);
282-
synchronized (schemaCacheMap) {
283-
ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId);
284-
if (schemaCache != null) {
285-
schemaCache.invalidateDbCache(dbName);
286-
}
259+
ExternalSchemaCache schemaCache = schemaCacheMgr.getCache(catalogId);
260+
if (schemaCache != null) {
261+
schemaCache.invalidateDbCache(dbName);
287262
}
288-
HiveMetaStoreCache metaCache = hiveCacheMap.get(catalogId);
263+
HiveMetaStoreCache metaCache = hiveMetaStoreCacheMgr.getCache(catalogId);
289264
if (metaCache != null) {
290265
metaCache.invalidateDbCache(dbName);
291266
}
292-
IcebergMetadataCache icebergMetadataCache = icebergCacheMap.get(catalogId);
267+
IcebergMetadataCache icebergMetadataCache = icebergMetadataCacheMgr.getCache(catalogId);
293268
if (icebergMetadataCache != null) {
294269
icebergMetadataCache.invalidateDbCache(catalogId, dbName);
295270
}
296271
hudiMetadataCacheMgr.invalidateDbCache(catalogId, dbName);
297272
maxComputeMetadataCacheMgr.invalidateDbCache(catalogId, dbName);
298-
paimonMetadataCacheMgr.invalidateDbCache(catalogId, dbName);
273+
PaimonMetadataCache paimonMetadataCache = paimonMetadataCacheMgr.getCache(catalogId);
274+
if (paimonMetadataCache != null) {
275+
paimonMetadataCache.invalidateDbCache(catalogId, dbName);
276+
}
299277
if (LOG.isDebugEnabled()) {
300278
LOG.debug("invalid db cache for {} in catalog {}", dbName, catalogId);
301279
}
302280
}
303281

304282
public void invalidateCatalogCache(long catalogId) {
305-
synchronized (schemaCacheMap) {
306-
schemaCacheMap.remove(catalogId);
307-
}
308-
HiveMetaStoreCache metaCache = hiveCacheMap.get(catalogId);
283+
schemaCacheMgr.removeCache(catalogId);
284+
HiveMetaStoreCache metaCache = hiveMetaStoreCacheMgr.getCache(catalogId);
309285
if (metaCache != null) {
310286
metaCache.invalidateAll();
311287
}
312-
IcebergMetadataCache icebergMetadataCache = icebergCacheMap.get(catalogId);
288+
IcebergMetadataCache icebergMetadataCache = icebergMetadataCacheMgr.getCache(catalogId);
313289
if (icebergMetadataCache != null) {
314290
icebergMetadataCache.invalidateCatalogCache(catalogId);
315291
}
316292
hudiMetadataCacheMgr.invalidateCatalogCache(catalogId);
317293
maxComputeMetadataCacheMgr.invalidateCatalogCache(catalogId);
318-
paimonMetadataCacheMgr.invalidateCatalogCache(catalogId);
294+
PaimonMetadataCache paimonMetadataCache = paimonMetadataCacheMgr.getCache(catalogId);
295+
if (paimonMetadataCache != null) {
296+
paimonMetadataCache.invalidateCatalogCache(catalogId);
297+
}
319298
dorisExternalMetaCacheMgr.invalidateCatalogCache(catalogId);
320299
if (LOG.isDebugEnabled()) {
321300
LOG.debug("invalid catalog cache for {}", catalogId);
322301
}
323302
}
324303

325304
public void invalidSchemaCache(long catalogId) {
326-
synchronized (schemaCacheMap) {
327-
schemaCacheMap.remove(catalogId);
328-
}
305+
schemaCacheMgr.removeCache(catalogId);
329306
}
330307

331308
public void addPartitionsCache(long catalogId, HMSExternalTable table, List<String> partitionNames) {
332309
String dbName = ClusterNamespace.getNameFromFullName(table.getDbName());
333-
HiveMetaStoreCache metaCache = hiveCacheMap.get(catalogId);
310+
HiveMetaStoreCache metaCache = hiveMetaStoreCacheMgr.getCache(catalogId);
334311
if (metaCache != null) {
335312
List<Type> partitionColumnTypes;
336313
try {
@@ -348,7 +325,7 @@ public void addPartitionsCache(long catalogId, HMSExternalTable table, List<Stri
348325

349326
public void dropPartitionsCache(long catalogId, HMSExternalTable table, List<String> partitionNames) {
350327
String dbName = ClusterNamespace.getNameFromFullName(table.getDbName());
351-
HiveMetaStoreCache metaCache = hiveCacheMap.get(catalogId);
328+
HiveMetaStoreCache metaCache = hiveMetaStoreCacheMgr.getCache(catalogId);
352329
if (metaCache != null) {
353330
metaCache.dropPartitionsCache(table, partitionNames, true);
354331
}
@@ -358,7 +335,7 @@ public void dropPartitionsCache(long catalogId, HMSExternalTable table, List<Str
358335
}
359336

360337
public void invalidatePartitionsCache(ExternalTable dorisTable, List<String> partitionNames) {
361-
HiveMetaStoreCache metaCache = hiveCacheMap.get(dorisTable.getCatalog().getId());
338+
HiveMetaStoreCache metaCache = hiveMetaStoreCacheMgr.getCache(dorisTable.getCatalog().getId());
362339
if (metaCache != null) {
363340
for (String partitionName : partitionNames) {
364341
metaCache.invalidatePartitionCache(dorisTable, partitionName);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,14 @@
1919

2020
import org.apache.doris.common.CacheFactory;
2121
import org.apache.doris.common.Config;
22+
import org.apache.doris.datasource.metacache.CacheSpec;
2223
import org.apache.doris.metric.GaugeMetric;
2324
import org.apache.doris.metric.Metric;
2425
import org.apache.doris.metric.MetricLabel;
2526
import org.apache.doris.metric.MetricRepo;
2627

2728
import com.github.benmanes.caffeine.cache.LoadingCache;
2829
import lombok.Data;
29-
import org.apache.commons.lang3.math.NumberUtils;
3030
import org.apache.logging.log4j.LogManager;
3131
import org.apache.logging.log4j.Logger;
3232

@@ -49,13 +49,14 @@ public ExternalSchemaCache(ExternalCatalog catalog, ExecutorService executor) {
4949
}
5050

5151
private void init(ExecutorService executor) {
52-
long schemaCacheTtlSecond = NumberUtils.toLong(
53-
(catalog.getProperties().get(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND)), ExternalCatalog.CACHE_NO_TTL);
52+
CacheSpec cacheSpec = CacheSpec.fromTtlValue(
53+
catalog.getProperties().get(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND),
54+
Config.external_cache_expire_time_seconds_after_access,
55+
Config.max_external_schema_cache_num);
5456
CacheFactory schemaCacheFactory = new CacheFactory(
55-
OptionalLong.of(schemaCacheTtlSecond >= ExternalCatalog.CACHE_TTL_DISABLE_CACHE
56-
? schemaCacheTtlSecond : Config.external_cache_expire_time_seconds_after_access),
57+
CacheSpec.toExpireAfterAccess(cacheSpec.getTtlSecond()),
5758
OptionalLong.of(Config.external_cache_refresh_time_minutes * 60),
58-
Config.max_external_schema_cache_num,
59+
cacheSpec.getCapacity(),
5960
false,
6061
null);
6162
schemaCache = schemaCacheFactory.buildCache(this::loadSchema, executor);

0 commit comments

Comments
 (0)