Skip to content

Commit 43a27b0

Browse files
committed
[fix](fe) Fence cold external refresh identities
### What problem does this PR solve? Issue Number: None Related PR: #68196 Problem Summary: On a mode-2 Paimon follower, a names refresh can rebind a logged Foo database to replacement FOO while a cold REFRESH replay still needs to retire SDK-only Foo handles and old Foo row counts. Resolve this ambiguous replay with an opening and closing catalog row-count fence around Paimon SDK retirement. A held table can also read after permanent DROP and recreate a row-count catalog generation; bound that registry so repeated overlapping drops cannot grow FE memory indefinitely. Finally, no-op DROP matching used equalsIgnoreCase, which equates I with dotless i even though catalog lookup's Locale.ROOT fold distinguishes them; use the same fold for resident identity matching. ### Release note Cold Paimon REFRESH replay and no-op database DROP now preserve the correct external metadata identity; late row-count reads after catalog DROP remain memory bounded. ### Check List (For Author) - Test: Unit Test: four related FE test classes, 153 tests passed; full ./build.sh --fe -j4 and Checkstyle passed - Behavior changed: Yes, ambiguous Paimon replay retires old SDK and row-count state, no-op DROP uses catalog name folding, and catalog generation state is bounded - Does this need documentation: No
1 parent d962085 commit 43a27b0

7 files changed

Lines changed: 167 additions & 16 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -258,7 +258,7 @@ public void replayRefreshTable(ExternalObjectLog log) {
258258
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(
259259
catalog.getId(), db.get().getId(), db.get().getFullName());
260260
}
261-
invalidatePaimonCatalogForUnresolvedReplay(catalog);
261+
invalidatePaimonCatalogForReplay(catalog);
262262
return;
263263
}
264264
if (!Strings.isNullOrEmpty(log.getNewTableName())) {
@@ -378,7 +378,7 @@ private void replayRefreshSafely(String operation, Runnable action) {
378378
}
379379
}
380380

381-
private void invalidatePaimonCatalogForUnresolvedReplay(ExternalCatalog catalog) {
381+
private void invalidatePaimonCatalogForReplay(ExternalCatalog catalog) {
382382
if (catalog instanceof PaimonExternalCatalog) {
383383
Env.getCurrentEnv().getExtMetaCacheMgr()
384384
.invalidateCatalogByEngine(catalog.getId(), PaimonExternalMetaCache.ENGINE);
@@ -392,14 +392,34 @@ private void invalidateColdReplayDatabase(ExternalCatalog catalog, ExternalObjec
392392
// known. Retire only that database's engine entries and independent row counts.
393393
LOG.debug("database object is cold when replaying refresh in catalog {}, db {}",
394394
catalog.getName(), identity.get().first);
395-
Env.getCurrentEnv().getExtMetaCacheMgr()
396-
.invalidateDb(catalog.getId(), identity.get().second, identity.get().first);
395+
// A mode-2 names refresh can rebind the logged Foo to a new FOO. The current
396+
// identity cannot fence an SDK-only Foo handle or Foo's independent row counts.
397+
boolean reboundPaimonName = catalog instanceof PaimonExternalCatalog
398+
&& catalog.getLowerCaseDatabaseNames() == 2
399+
&& !Strings.isNullOrEmpty(log.getDbName())
400+
&& !identity.get().first.equals(log.getDbName());
401+
ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr();
402+
if (reboundPaimonName) {
403+
cacheMgr.invalidateRowCountCache(catalog.getId());
404+
}
405+
try {
406+
cacheMgr.invalidateDb(catalog.getId(), identity.get().second, identity.get().first);
407+
} finally {
408+
if (reboundPaimonName) {
409+
try {
410+
invalidatePaimonCatalogForReplay(catalog);
411+
} finally {
412+
// Close the load window opened after the first fence.
413+
cacheMgr.invalidateRowCountCache(catalog.getId());
414+
}
415+
}
416+
}
397417
} else {
398418
// A lost mode-2 mapping cannot identify the database safely; widen the fence.
399419
LOG.warn("failed to resolve database identity when replaying refresh: {}",
400420
log.debugForRefreshDb());
401421
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(catalog.getId());
402-
invalidatePaimonCatalogForUnresolvedReplay(catalog);
422+
invalidatePaimonCatalogForReplay(catalog);
403423
}
404424
}
405425

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,12 @@ public class ExternalRowCountCache {
5252
private final ConcurrentHashMap<LoadKey, Set<LoadFence>> inFlightLoads = new ConcurrentHashMap<>();
5353
// A catalog refresh changes this generation in O(1). Old cache entries remain bounded by
5454
// Caffeine's size/expiry policy but are never addressable by a new metadata generation.
55-
private final ConcurrentHashMap<Long, AtomicLong> catalogGenerations = new ConcurrentHashMap<>();
55+
// A held table may issue a late read after permanent DROP releases its catalog generation.
56+
// Keep even those post-DROP admissions bounded; eviction forces a fresh unique generation.
57+
private final Cache<Long, AtomicLong> catalogGenerations;
5658
// Database generations are bounded independently of database CREATE/DROP churn. Eviction
5759
// merely forces a new, globally unique generation and makes the old entries unreachable.
58-
private final Cache<DatabaseKey, Long> dbGenerations = Caffeine.newBuilder()
59-
.maximumSize(Config.max_external_table_row_count_cache_num).build();
60+
private final Cache<DatabaseKey, Long> dbGenerations;
6061
// Generations are globally unique: two catalog IDs with the same table ID must not alias
6162
// when one catalog has retired its old cache generation.
6263
private final AtomicLong nextCatalogGeneration = new AtomicLong();
@@ -73,6 +74,13 @@ public ExternalRowCountCache(ExecutorService executor) {
7374
}
7475

7576
ExternalRowCountCache(ExecutorService executor, Ticker ticker, RowCountCacheLoader loader) {
77+
this(executor, ticker, loader, Config.max_external_table_row_count_cache_num);
78+
}
79+
80+
ExternalRowCountCache(ExecutorService executor, Ticker ticker, RowCountCacheLoader loader,
81+
long maxGenerationEntries) {
82+
catalogGenerations = Caffeine.newBuilder().maximumSize(maxGenerationEntries).build();
83+
dbGenerations = Caffeine.newBuilder().maximumSize(maxGenerationEntries).build();
7684
// 1. set expireAfterWrite to 1 day, avoid too many entries
7785
// 2. set refreshAfterWrite to 10min(default), so that the cache will be refreshed after 10min
7886
CacheFactory rowCountCacheFactory = new CacheFactory(
@@ -276,12 +284,12 @@ void refreshForTest(long catalogId, long dbId, long tableId) {
276284
}
277285

278286
private long currentCatalogGeneration(long catalogId) {
279-
return catalogGenerations.computeIfAbsent(catalogId,
287+
return catalogGenerations.get(catalogId,
280288
ignored -> new AtomicLong(nextCatalogGeneration.incrementAndGet())).get();
281289
}
282290

283291
private boolean isCatalogGenerationCurrent(long catalogId, long generation) {
284-
AtomicLong current = catalogGenerations.get(catalogId);
292+
AtomicLong current = catalogGenerations.getIfPresent(catalogId);
285293
return current != null && current.get() == generation;
286294
}
287295

@@ -296,7 +304,8 @@ private boolean isDbGenerationCurrent(long catalogId, long dbId, long generation
296304
}
297305

298306
int getCatalogGenerationCountForTest() {
299-
return catalogGenerations.size();
307+
catalogGenerations.cleanUp();
308+
return catalogGenerations.asMap().size();
300309
}
301310

302311
static Optional<Long> loadRowCount(RowCountKey rowCountKey, boolean fillMetaCache) {
@@ -380,7 +389,7 @@ public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId)
380389
long dbGeneration;
381390
publicationLock.readLock().lock();
382391
try {
383-
AtomicLong current = catalogGenerations.get(catalogId);
392+
AtomicLong current = catalogGenerations.getIfPresent(catalogId);
384393
if (current == null) {
385394
return -1;
386395
}
@@ -413,7 +422,7 @@ && isDbGenerationCurrent(catalogId, dbId, dbGeneration)) {
413422
void invalidateCatalog(long catalogId) {
414423
publicationLock.writeLock().lock();
415424
try {
416-
AtomicLong current = catalogGenerations.get(catalogId);
425+
AtomicLong current = catalogGenerations.getIfPresent(catalogId);
417426
if (current != null) {
418427
current.set(nextCatalogGeneration.incrementAndGet());
419428
}
@@ -426,7 +435,7 @@ void invalidateCatalog(long catalogId) {
426435
void releaseCatalog(long catalogId) {
427436
publicationLock.writeLock().lock();
428437
try {
429-
catalogGenerations.remove(catalogId);
438+
catalogGenerations.invalidate(catalogId);
430439
} finally {
431440
publicationLock.writeLock().unlock();
432441
}
@@ -448,7 +457,7 @@ void invalidateDb(long catalogId, long dbId) {
448457
void invalidateTable(long catalogId, long dbId, long tableId) {
449458
publicationLock.writeLock().lock();
450459
try {
451-
AtomicLong current = catalogGenerations.get(catalogId);
460+
AtomicLong current = catalogGenerations.getIfPresent(catalogId);
452461
if (current == null) {
453462
return;
454463
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.apache.logging.log4j.Logger;
3232

3333
import java.util.List;
34+
import java.util.Locale;
3435
import java.util.Map;
3536
import java.util.Objects;
3637
import java.util.Optional;
@@ -560,8 +561,9 @@ public Optional<String> getNameByIdIfPresent(long id) {
560561

561562
/** Snapshot resident object identities matching a DROP alias under the catalog's name mode. */
562563
public List<Pair<String, Long>> getCachedIdentitiesMatching(String name, boolean ignoreCase) {
564+
String foldedName = ignoreCase ? name.toLowerCase(Locale.ROOT) : name;
563565
return withMetaObjLifecycleReadLock(() -> idToName.entrySet().stream()
564-
.filter(entry -> ignoreCase ? entry.getValue().equalsIgnoreCase(name)
566+
.filter(entry -> ignoreCase ? entry.getValue().toLowerCase(Locale.ROOT).equals(foldedName)
565567
: entry.getValue().equals(name))
566568
.map(entry -> Pair.of(entry.getValue(), entry.getKey()))
567569
.collect(Collectors.toList()));

‎fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
import org.apache.doris.datasource.hive.HMSExternalTable;
2929
import org.apache.doris.datasource.hive.HiveExternalMetaCache;
3030
import org.apache.doris.datasource.lance.LanceExternalCatalog;
31+
import org.apache.doris.datasource.paimon.PaimonExternalCatalog;
32+
import org.apache.doris.datasource.paimon.PaimonExternalMetaCache;
3133
import org.apache.doris.persist.EditLog;
3234

3335
import org.junit.jupiter.api.Assertions;
@@ -74,6 +76,35 @@ void testColdKnownDatabaseReplayKeepsUnrelatedHotDatabase() {
7476
Mockito.verify(hotDb, Mockito.never()).resetMetaToUninitialized();
7577
}
7678

79+
@Test
80+
void testColdReboundPaimonReplayFencesOldRowCountsAroundSdkInvalidation() {
81+
long catalogId = 83L;
82+
long replacementDbId = 84L;
83+
PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class);
84+
Mockito.when(catalog.getId()).thenReturn(catalogId);
85+
Mockito.when(catalog.getLowerCaseDatabaseNames()).thenReturn(2);
86+
Mockito.when(catalog.getDbForReplay("Foo")).thenReturn(Optional.empty());
87+
Mockito.when(catalog.getDbIdentityForReplay("Foo", 0L))
88+
.thenReturn(Optional.of(Pair.of("FOO", replacementDbId)));
89+
CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
90+
Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId);
91+
ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
92+
Env env = Mockito.mock(Env.class);
93+
Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
94+
Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
95+
96+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
97+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
98+
new RefreshManager().replayRefreshDb(ExternalObjectLog.createForRefreshDb(catalogId, "Foo"));
99+
}
100+
101+
InOrder order = Mockito.inOrder(cacheMgr);
102+
order.verify(cacheMgr).invalidateRowCountCache(catalogId);
103+
order.verify(cacheMgr).invalidateDb(catalogId, replacementDbId, "FOO");
104+
order.verify(cacheMgr).invalidateCatalogByEngine(catalogId, PaimonExternalMetaCache.ENGINE);
105+
order.verify(cacheMgr).invalidateRowCountCache(catalogId);
106+
}
107+
77108
@Test
78109
void testColdDatabaseReplayInvalidatesCatalogRowCount() {
79110
long catalogId = 51L;

‎fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -967,10 +967,13 @@ void testNoOpDatabaseDropKeepsUnrelatedCachedDatabase() {
967967
ignored -> Collections.emptyList(), ignored -> Optional.empty(), (key, value, cause) -> { });
968968
long oldId = Util.genIdByName("testing_catalog", "Foo");
969969
long unrelatedId = Util.genIdByName("testing_catalog", "sales");
970+
long turkishId = Util.genIdByName("testing_catalog", "I");
970971
ExternalDatabase<?> oldDb = Mockito.mock(ExternalDatabase.class);
971972
ExternalDatabase<?> unrelatedDb = Mockito.mock(ExternalDatabase.class);
973+
ExternalDatabase<?> turkishDb = Mockito.mock(ExternalDatabase.class);
972974
metaCache.addObjForTest(oldId, "Foo", oldDb);
973975
metaCache.addObjForTest(unrelatedId, "sales", unrelatedDb);
976+
metaCache.addObjForTest(turkishId, "I", turkishDb);
974977
catalog.installMetaCache(metaCache);
975978
Env env = Mockito.mock(Env.class);
976979
ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class);
@@ -979,12 +982,17 @@ void testNoOpDatabaseDropKeepsUnrelatedCachedDatabase() {
979982
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
980983
catalog.retireCachedDatabaseForNoOp("absent");
981984
Assertions.assertSame(unrelatedDb, metaCache.tryGetMetaObj("sales").orElseThrow(AssertionError::new));
985+
// Locale.ROOT distinguishes I (i) from dotless i. A no-op DROP of the latter
986+
// must not retire the resident I database.
987+
catalog.retireCachedDatabaseForNoOp("ı");
988+
Assertions.assertSame(turkishDb, metaCache.tryGetMetaObj("I").orElseThrow(AssertionError::new));
982989
Mockito.verifyNoInteractions(cacheMgr);
983990

984991
// A lost mode-2 mapping still permits narrow retirement of a retained historical ID.
985992
catalog.retireCachedDatabaseForNoOp("foo");
986993
Assertions.assertFalse(metaCache.tryGetMetaObj("Foo").isPresent());
987994
Assertions.assertSame(unrelatedDb, metaCache.tryGetMetaObj("sales").orElseThrow(AssertionError::new));
995+
Assertions.assertSame(turkishDb, metaCache.tryGetMetaObj("I").orElseThrow(AssertionError::new));
988996
Mockito.verify(cacheMgr).invalidateDb(catalogId, oldId, "Foo");
989997
Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(catalogId);
990998
Mockito.verify(cacheMgr, Mockito.never()).invalidateRowCountCache(catalogId, unrelatedId);

‎fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,31 @@ protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey key) {
445445
}
446446
}
447447

448+
@Test
449+
public void testLateReadsAfterPermanentRemovalDoNotGrowCatalogGenerations() {
450+
java.util.Set<Long> activeCatalogs = new java.util.HashSet<>();
451+
ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() {
452+
@Override
453+
protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey key) {
454+
return activeCatalogs.contains(key.getCatalogId()) ? Optional.of(100L) : null;
455+
}
456+
};
457+
ExternalRowCountCache cache = new ExternalRowCountCache(
458+
MoreExecutors.newDirectExecutorService(), null, loader, 2);
459+
for (long catalogId = 1; catalogId <= 5; catalogId++) {
460+
activeCatalogs.add(catalogId);
461+
Assertions.assertEquals(100L, cache.getCachedRowCount(catalogId, 10, 100, false));
462+
activeCatalogs.remove(catalogId);
463+
cache.invalidateCatalog(catalogId);
464+
cache.releaseCatalog(catalogId);
465+
466+
// A statement retaining an old table can still enter the cache after DROP cleanup.
467+
Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT,
468+
cache.getCachedRowCount(catalogId, 10, 100, false));
469+
Assertions.assertTrue(cache.getCatalogGenerationCountForTest() <= 2);
470+
}
471+
}
472+
448473
@Test
449474
public void testInvalidateWhileRefreshIsRunningDoesNotRepublishStaleValue() throws Exception {
450475
AtomicInteger loadCount = new AtomicInteger();

‎fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2571,6 +2571,62 @@ public void testReplayRefreshInvalidatesSdkOnlyPaimonCache() throws Exception {
25712571
}
25722572
}
25732573

2574+
@Test
2575+
public void testMode2ColdReplayRefreshInvalidatesReboundSdkOnlyPaimonHandle() throws Exception {
2576+
java.io.File warehouse = temporaryFolder.newFolder("mode2_rebound_refresh_sdk_only");
2577+
Map<String, String> properties = new HashMap<>();
2578+
properties.put("type", "paimon");
2579+
properties.put(PaimonExternalCatalog.PAIMON_CATALOG_TYPE,
2580+
PaimonExternalCatalog.PAIMON_FILESYSTEM);
2581+
properties.put("warehouse", warehouse.toURI().toString());
2582+
properties.put(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2");
2583+
long catalogId = 120L;
2584+
PaimonExternalCatalog dorisCatalog = new PaimonExternalCatalog(
2585+
catalogId, "mode2_rebound_refresh_test", null, properties, "");
2586+
dorisCatalog.makeSureInitialized();
2587+
dorisCatalog.catalog.createDatabase("Foo", false);
2588+
Identifier oldIdentifier = Identifier.create("Foo", "table");
2589+
Schema schema = Schema.newBuilder().column("id", DataTypes.INT()).build();
2590+
dorisCatalog.catalog.createTable(oldIdentifier, schema, false);
2591+
2592+
Map<String, String> externalProperties = new HashMap<>(properties);
2593+
externalProperties.put("paimon.cache-enabled", "false");
2594+
PaimonExternalCatalog externalCatalog = new PaimonExternalCatalog(
2595+
121L, "external_paimon", null, externalProperties, "");
2596+
externalCatalog.makeSureInitialized();
2597+
2598+
CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
2599+
Env env = Mockito.mock(Env.class);
2600+
Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
2601+
Mockito.when(env.getAccessManager()).thenReturn(Mockito.mock(AccessControllerManager.class));
2602+
ExternalMetaCacheMgr cacheMgr = new ExternalMetaCacheMgr(true);
2603+
Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
2604+
Mockito.doReturn(dorisCatalog).when(catalogMgr).getCatalog(catalogId);
2605+
Mockito.doReturn(dorisCatalog).when(catalogMgr)
2606+
.getCatalogOrException(Mockito.eq(catalogId), Mockito.any());
2607+
NameMapping oldMapping = new NameMapping(catalogId, "Foo", "table", "Foo", "table");
2608+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
2609+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
2610+
dorisCatalog.getPaimonTable(oldMapping);
2611+
externalCatalog.catalog.dropTable(oldIdentifier, false);
2612+
externalCatalog.catalog.dropDatabase("Foo", false, false);
2613+
externalCatalog.catalog.createDatabase("FOO", false);
2614+
2615+
Field mappingField = ExternalCatalog.class.getDeclaredField("lowerCaseToDatabaseName");
2616+
mappingField.setAccessible(true);
2617+
Map<String, String> reboundNames = new HashMap<>();
2618+
reboundNames.put("foo", "FOO");
2619+
mappingField.set(dorisCatalog, reboundNames);
2620+
Assert.assertFalse(dorisCatalog.getDbForReplay("Foo").isPresent());
2621+
2622+
new RefreshManager().replayRefreshDb(ExternalObjectLog.createForRefreshDb(catalogId, "Foo"));
2623+
assertPaimonCatalogTableMissing(dorisCatalog, oldMapping);
2624+
} finally {
2625+
dorisCatalog.catalog.close();
2626+
externalCatalog.catalog.close();
2627+
}
2628+
}
2629+
25742630
@Test
25752631
public void testReplayDropInvalidatesSdkOnlyPaimonCache() throws Exception {
25762632
java.io.File warehouse = temporaryFolder.newFolder("replay_drop_sdk_only");

0 commit comments

Comments
 (0)