Skip to content

Commit 73388f7

Browse files
committed
[fix](fe) Preserve Paimon cache semantics in MetaCache wrapper
### What problem does this PR solve? Issue Number: N/A Problem Summary: The Doris-owned Paimon MetaCache wrapper replaced the SDK CachingCatalog but did not preserve all of its cache, expiry, identifier, REST dispatch, and DDL invalidation semantics. This could leave stale metadata after partial DDL failures, bypass a configured memory budget through nested SDK caches, or route wrapped REST catalogs through the filesystem fallback. Preserve the user cache setting and access/write expiry, scope branch and system variants by their base table, unwrap REST catalogs for partition dispatch, and invalidate all shared metadata scopes after every possibly partial mutation. Mutable SDK table caches are omitted when an enclosing weight limit governs the catalog. ### Release note Fix Paimon metadata cache invalidation and configuration semantics while keeping REST catalog partition dispatch intact. ### Check List (For Author) - Test: Unit Test - Paimon focused tests: 31 passed - Paimon reactor install: 631 passed, 1 skipped - Full FE build: ./build.sh --fe - Behavior changed: Yes. Paimon cache configuration, expiry, invalidation, and memory governance now match the intended SDK semantics. - Does this need documentation: No
1 parent e7b387c commit 73388f7

7 files changed

Lines changed: 617 additions & 74 deletions

File tree

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogFactory.java‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -103,11 +103,19 @@ private PaimonCatalogFactory() {
103103
* plus each flavor's {@code appendCustomCatalogOptions()}.
104104
*/
105105
public static Options buildCatalogOptions(PaimonCatalogProperties catalogProperties) {
106-
return buildCatalogOptions(catalogProperties, false);
106+
Options options = assembleCatalogOptions(catalogProperties);
107+
// PaimonMetaCacheCatalog replaces the SDK CachingCatalog. The two cache layers cannot
108+
// coexist because a Doris invalidation cannot evict a frozen Table from the SDK wrapper.
109+
// Preserve the user's flag separately, but always disable the SDK wrapper itself.
110+
options.set(CatalogOptions.CACHE_ENABLED, false);
111+
return options;
107112
}
108113

109-
static Options buildCatalogOptions(
110-
PaimonCatalogProperties catalogProperties, boolean hasEnclosingMetaCacheWeightLimit) {
114+
static boolean isCatalogCacheEnabled(PaimonCatalogProperties catalogProperties) {
115+
return assembleCatalogOptions(catalogProperties).get(CatalogOptions.CACHE_ENABLED);
116+
}
117+
118+
private static Options assembleCatalogOptions(PaimonCatalogProperties catalogProperties) {
111119
Options options = new Options();
112120
Map<String, String> props = catalogProperties.getRaw();
113121
String flavor = catalogProperties.getFlavor();
@@ -141,13 +149,6 @@ static Options buildCatalogOptions(
141149
// filesystem: nothing custom.
142150
break;
143151
}
144-
// PaimonMetaCacheCatalog (Doris-owned) replaces the SDK CachingCatalog's table and database
145-
// caches with Doris's own CatalogMetaCache framework. The two caching layers cannot coexist:
146-
// if the SDK wraps itself in CachingCatalog (cache-enabled=true, the SDK default), a
147-
// PaimonMetaCacheCatalog miss falls through to CachingCatalog.getTable() which returns a
148-
// frozen Table — the stale-read bug DORIS-29032 would persist even after a Doris-side
149-
// REFRESH. Always disable the SDK cache so the catalog always loads fresh.
150-
options.set(CatalogOptions.CACHE_ENABLED, false);
151152
return options;
152153
}
153154

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogOps.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.paimon.catalog.Catalog;
2323
import org.apache.paimon.catalog.CatalogUtils;
2424
import org.apache.paimon.catalog.Database;
25+
import org.apache.paimon.catalog.DelegateCatalog;
2526
import org.apache.paimon.catalog.Identifier;
2627
import org.apache.paimon.partition.Partition;
2728
import org.apache.paimon.privilege.PrivilegedFileStoreTable;
@@ -310,17 +311,23 @@ public Table getTable(Identifier identifier) throws Catalog.TableNotExistExcepti
310311
@Override
311312
public List<Partition> listPartitions(Identifier identifier, Table table)
312313
throws Catalog.TableNotExistException {
313-
if (catalog instanceof RESTCatalog) {
314+
RESTCatalog restCatalog = restCatalog(catalog);
315+
if (restCatalog != null) {
314316
// REST owns partition visibility when its endpoint is implemented; the bridge
315317
// retains this effective relation copy only for the endpoint's filesystem fallback.
316318
return PaimonRestCatalogPartitions.listPartitions(
317-
(RESTCatalog) catalog, identifier, table);
319+
restCatalog, identifier, table);
318320
}
319321
// The supplied handle already contains catalog and relation policy. Reloading by identifier
320322
// would discard those copies before manifest enumeration reaches the final scan guard.
321323
return CatalogUtils.listPartitionsFromFileSystem(table);
322324
}
323325

326+
static RESTCatalog restCatalog(Catalog catalog) {
327+
Catalog rootCatalog = DelegateCatalog.rootCatalog(catalog);
328+
return rootCatalog instanceof RESTCatalog ? (RESTCatalog) rootCatalog : null;
329+
}
330+
324331
@Override
325332
public void createDatabase(String name, boolean ignoreIfExists, Map<String, String> properties)
326333
throws Catalog.DatabaseAlreadyExistException {

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -498,7 +498,7 @@ static boolean hasDlfCompatibleStorage(List<StorageProperties> storageProperties
498498
}
499499

500500
Options buildCatalogOptions() {
501-
return PaimonCatalogFactory.buildCatalogOptions(catalogProps, metaCache.hasEnclosingWeightLimit());
501+
return PaimonCatalogFactory.buildCatalogOptions(catalogProps);
502502
}
503503

504504
/**
@@ -546,9 +546,10 @@ private Catalog createCatalogFromContext(CatalogContext catalogContext, String f
546546
? createHmsCatalog(catalogContext, hmsAuth, catalogProps.getRaw(),
547547
storageHadoopConfig)
548548
: CatalogFactory.createCatalog(catalogContext);
549-
return new PaimonMetaCacheCatalog(catalog, metaCache,
549+
return PaimonMetaCacheCatalog.tryToCreate(catalog, metaCache,
550550
DEFAULT_TABLE_CACHE_CAPACITY, resolveTableCacheTtlSecond(catalogProps.getRaw()),
551-
catalogContext.options());
551+
catalogContext.options(), PaimonCatalogFactory.isCatalogCacheEnabled(catalogProps),
552+
metaCache.hasEnclosingWeightLimit());
552553
});
553554
} catch (Exception e) {
554555
throw new RuntimeException(failureMessage + " (flavor=" + flavor + "): " + e.getMessage(), e);

0 commit comments

Comments
 (0)