Skip to content

Commit 0c37f51

Browse files
committed
[fix](fe) Fence external metadata cache transitions
### What problem does this PR solve? Issue Number: None Related PR: #68196 Problem Summary: Metadata refresh and HMS events could expose new external objects while retaining row counts from an older incarnation, and some cold-table invalidations widened into expensive database or catalog scans. Fence row counts before and after fallible metadata transitions, use a constant-time catalog generation for whole-catalog retirement, resolve retained table IDs without loading cold objects, and normalize HMS database event names consistently with discovery and filters. This also fixes the FE UT case-insensitive event filter failure. ### Release note External table row counts and HMS database events now follow refreshed metadata more consistently. ### Check List (For Author) - Test: Unit Test: 15 related FE test classes, 239 tests passed; full FE build passed before the final filter adjustment - Behavior changed: Yes, external metadata refresh and event invalidation retire stale row counts; case-insensitive database filters match normalized HMS event names - Does this need documentation: No
1 parent de4f82f commit 0c37f51

12 files changed

Lines changed: 419 additions & 71 deletions

‎fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java‎

Lines changed: 41 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.doris.datasource.CatalogLog;
2626
import org.apache.doris.datasource.ExternalCatalog;
2727
import org.apache.doris.datasource.ExternalDatabase;
28+
import org.apache.doris.datasource.ExternalMetaCacheMgr;
2829
import org.apache.doris.datasource.ExternalObjectLog;
2930
import org.apache.doris.datasource.ExternalTable;
3031
import org.apache.doris.datasource.hive.HMSExternalCatalog;
@@ -96,7 +97,7 @@ public void handleRefreshDb(String catalogName, String dbName) throws DdlExcepti
9697
}
9798
DatabaseIf db = catalog.getDbOrDdlException(dbName);
9899
// Local DB-object eviction also resets metadata; only an explicit refresh retires access.
99-
invalidateLanceTableAccess(catalog);
100+
invalidateLanceTableAccess((ExternalDatabase) db);
100101
refreshDbInternal((ExternalDatabase) db);
101102

102103
ExternalObjectLog log = ExternalObjectLog.createForRefreshDb(catalog.getId(), db.getFullName());
@@ -111,7 +112,7 @@ public void replayRefreshDb(ExternalObjectLog log) {
111112
LOG.warn("failed to find catalog when replaying refresh db: {}", log.debugForRefreshDb());
112113
return;
113114
}
114-
invalidateLanceTableAccess(catalog);
115+
invalidateLanceTableAccess(catalog, log);
115116
Optional<ExternalDatabase<? extends ExternalTable>> db;
116117
if (!Strings.isNullOrEmpty(log.getDbName())) {
117118
db = catalog.getDbForReplay(log.getDbName());
@@ -127,14 +128,34 @@ public void replayRefreshDb(ExternalObjectLog log) {
127128
});
128129
}
129130

130-
private void invalidateLanceTableAccess(CatalogIf catalog) {
131-
// Access entries outlive the bounded DB/table object caches. Replay must invalidate by
132-
// catalog identity before its cache-only object lookup can return early, including ID logs.
133-
if (catalog instanceof LanceExternalCatalog) {
131+
private void invalidateLanceTableAccess(ExternalDatabase db) {
132+
if (db.getCatalog() instanceof LanceExternalCatalog) {
133+
ExternalCatalog catalog = (ExternalCatalog) db.getCatalog();
134+
Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(catalog.getId(), db.getId());
134135
((LanceExternalCatalog) catalog).invalidateTableAccessCache();
135136
}
136137
}
137138

139+
private void invalidateLanceTableAccess(ExternalCatalog catalog, ExternalObjectLog log) {
140+
if (!(catalog instanceof LanceExternalCatalog)) {
141+
return;
142+
}
143+
// Access entries outlive the bounded DB/table object caches. Fence the retained identity
144+
// before retiring access, even if the following cache-only object lookup misses.
145+
ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr();
146+
if (!Strings.isNullOrEmpty(log.getDbName()) && !Strings.isNullOrEmpty(log.getTableName())) {
147+
cacheMgr.invalidateRowCountCache(catalog.getId(), log.getDbName(), log.getTableName());
148+
} else {
149+
Optional<Pair<String, Long>> identity = catalog.getDbIdentityForReplay(log.getDbName(), log.getDbId());
150+
if (identity.isPresent()) {
151+
cacheMgr.invalidateRowCountCache(catalog.getId(), identity.get().second);
152+
} else {
153+
cacheMgr.invalidateRowCountCache(catalog.getId());
154+
}
155+
}
156+
((LanceExternalCatalog) catalog).invalidateTableAccessCache();
157+
}
158+
138159
private void refreshDbInternal(ExternalDatabase db) {
139160
db.resetMetaToUninitialized();
140161
LOG.info("refresh database {} in catalog {}", db.getFullName(), db.getCatalog().getName());
@@ -198,7 +219,7 @@ public void replayRefreshTable(ExternalObjectLog log) {
198219
LOG.warn("failed to find catalog when replaying refresh table: {}", log.debugForRefreshTable());
199220
return;
200221
}
201-
invalidateLanceTableAccess(catalog);
222+
invalidateLanceTableAccess(catalog, log);
202223
Optional<ExternalDatabase<? extends ExternalTable>> db;
203224
if (!Strings.isNullOrEmpty(log.getDbName())) {
204225
db = catalog.getDbForReplay(log.getDbName());
@@ -404,6 +425,17 @@ public void refreshPartitions(String catalogName, String dbName, String tableNam
404425
// before any database/table reload can fail and make the ignored-not-found path return.
405426
Env.getCurrentEnv().getExtMetaCacheMgr()
406427
.invalidateRowCountCache(catalog.getId(), dbName, tableName);
428+
try {
429+
refreshPartitionsAfterFence(catalog, dbName, tableName, partitionNames, updateTime,
430+
ignoreIfNotExists);
431+
} finally {
432+
Env.getCurrentEnv().getExtMetaCacheMgr()
433+
.invalidateRowCountCache(catalog.getId(), dbName, tableName);
434+
}
435+
}
436+
437+
private void refreshPartitionsAfterFence(CatalogIf catalog, String dbName, String tableName,
438+
List<String> partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException {
407439
DatabaseIf db = catalog.getDbNullable(dbName);
408440
if (db == null) {
409441
Env.getCurrentEnv().getExtMetaCacheMgr()
@@ -426,15 +458,8 @@ public void refreshPartitions(String catalogName, String dbName, String tableNam
426458

427459
ExternalTable externalTable = (ExternalTable) table;
428460
HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr().hive(externalTable.getCatalog().getId());
429-
try {
430-
for (String partitionName : partitionNames) {
431-
cache.invalidatePartitionCache(externalTable, partitionName);
432-
}
433-
} finally {
434-
// Close the admission window opened by the pre-fence above, even when the selective
435-
// invalidation fails partway.
436-
Env.getCurrentEnv().getExtMetaCacheMgr()
437-
.invalidateRowCountCache(catalog.getId(), dbName, tableName);
461+
for (String partitionName : partitionNames) {
462+
cache.invalidatePartitionCache(externalTable, partitionName);
438463
}
439464
((HMSExternalTable) table).setUpdateTime(updateTime);
440465
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1090,7 +1090,7 @@ public void registerExternalDatabaseFromEvent(String dbName, String catalogName)
10901090

10911091
HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog;
10921092
long metadataLoadEpoch = hmsCatalog.acquireMetadataLoadEpoch();
1093-
long dbId = Util.genIdByName(catalogName, dbName);
1093+
long dbId = Util.genIdByName(catalogName, hmsCatalog.localDatabaseNameFromRemote(dbName));
10941094
// -1L means it will be dropped later, ignore
10951095
if (dbId == ExternalMetaIdMgr.META_ID_FOR_NOT_EXISTS) {
10961096
return;
@@ -1119,6 +1119,17 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl
11191119
// before any database/table reload can fail and make the ignored-not-found path return.
11201120
Env.getCurrentEnv().getExtMetaCacheMgr()
11211121
.invalidateRowCountCache(catalog.getId(), dbName, tableName);
1122+
try {
1123+
addExternalPartitionsAfterFence(catalog, dbName, tableName, partitionNames, updateTime,
1124+
ignoreIfNotExists);
1125+
} finally {
1126+
Env.getCurrentEnv().getExtMetaCacheMgr()
1127+
.invalidateRowCountCache(catalog.getId(), dbName, tableName);
1128+
}
1129+
}
1130+
1131+
private void addExternalPartitionsAfterFence(CatalogIf catalog, String dbName, String tableName,
1132+
List<String> partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException {
11221133
DatabaseIf db = catalog.getDbNullable(dbName);
11231134
if (db == null) {
11241135
Env.getCurrentEnv().getExtMetaCacheMgr()
@@ -1152,14 +1163,7 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl
11521163
return;
11531164
}
11541165
HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr().hive(catalog.getId());
1155-
try {
1156-
cache.addPartitionsCache(hmsTable.getOrBuildNameMapping(), partitionNames, partitionColumnTypes);
1157-
} finally {
1158-
// Close the admission window opened by the pre-fence above, even when the selective
1159-
// update fails partway.
1160-
Env.getCurrentEnv().getExtMetaCacheMgr()
1161-
.invalidateRowCountCache(catalog.getId(), dbName, tableName);
1162-
}
1166+
cache.addPartitionsCache(hmsTable.getOrBuildNameMapping(), partitionNames, partitionColumnTypes);
11631167
hmsTable.setUpdateTime(updateTime);
11641168
}
11651169

@@ -1181,6 +1185,17 @@ public void dropExternalPartitions(String catalogName, String dbName, String tab
11811185
// before any database/table reload can fail and make the ignored-not-found path return.
11821186
Env.getCurrentEnv().getExtMetaCacheMgr()
11831187
.invalidateRowCountCache(catalog.getId(), dbName, tableName);
1188+
try {
1189+
dropExternalPartitionsAfterFence(catalog, dbName, tableName, partitionNames, updateTime,
1190+
ignoreIfNotExists);
1191+
} finally {
1192+
Env.getCurrentEnv().getExtMetaCacheMgr()
1193+
.invalidateRowCountCache(catalog.getId(), dbName, tableName);
1194+
}
1195+
}
1196+
1197+
private void dropExternalPartitionsAfterFence(CatalogIf catalog, String dbName, String tableName,
1198+
List<String> partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException {
11841199
DatabaseIf db = catalog.getDbNullable(dbName);
11851200
if (db == null) {
11861201
Env.getCurrentEnv().getExtMetaCacheMgr()
@@ -1202,15 +1217,8 @@ public void dropExternalPartitions(String catalogName, String dbName, String tab
12021217
}
12031218

12041219
HMSExternalTable hmsTable = (HMSExternalTable) table;
1205-
try {
1206-
Env.getCurrentEnv().getExtMetaCacheMgr().hive(catalog.getId())
1207-
.dropPartitionsCache(hmsTable, partitionNames, true);
1208-
} finally {
1209-
// Close the admission window opened by the pre-fence above, even when the selective
1210-
// update fails partway.
1211-
Env.getCurrentEnv().getExtMetaCacheMgr()
1212-
.invalidateRowCountCache(catalog.getId(), dbName, tableName);
1213-
}
1220+
Env.getCurrentEnv().getExtMetaCacheMgr().hive(catalog.getId())
1221+
.dropPartitionsCache(hmsTable, partitionNames, true);
12141222
hmsTable.setUpdateTime(updateTime);
12151223
}
12161224

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -581,20 +581,11 @@ private List<Pair<String, String>> getFilteredDatabaseNames() {
581581

582582
allDatabases = allDatabases.stream()
583583
.filter(dbName -> isDatabaseAllowedByFilter(
584-
dbName, includeDatabaseMap, excludeDatabaseMap, false))
584+
dbName, includeDatabaseMap, excludeDatabaseMap, getLowerCaseDatabaseNames() != 0))
585585
.collect(Collectors.toList());
586586

587587
for (String remoteDbName : allDatabases) {
588-
String localDbName = fromRemoteDatabaseName(remoteDbName);
589-
// Apply lower_case_database_names mode to local name
590-
int dbNameMode = getLowerCaseDatabaseNames();
591-
if (dbNameMode == 1) {
592-
localDbName = localDbName.toLowerCase();
593-
} else if (dbNameMode == 2) {
594-
// Mode 2: preserve original remote case for display
595-
localDbName = remoteDbName;
596-
}
597-
remoteToLocalPairs.add(Pair.of(remoteDbName, localDbName));
588+
remoteToLocalPairs.add(Pair.of(remoteDbName, localDatabaseNameFromRemote(remoteDbName)));
598589
}
599590

600591
// Check for conflicts when lower_case_meta_names = true or lower_case_database_names = 2
@@ -627,8 +618,20 @@ private List<Pair<String, String>> getFilteredDatabaseNames() {
627618
return remoteToLocalPairs;
628619
}
629620

621+
/** Use the same local identity for database discovery and HMS create events. */
622+
protected final String localDatabaseNameFromRemote(String remoteDbName) {
623+
String localDbName = fromRemoteDatabaseName(remoteDbName);
624+
int mode = getLowerCaseDatabaseNames();
625+
if (mode == 1) {
626+
return localDbName.toLowerCase(Locale.ROOT);
627+
}
628+
// Mode 2 preserves remote spelling for display and deterministic database IDs.
629+
return mode == 2 ? remoteDbName : localDbName;
630+
}
631+
630632
protected boolean isDatabaseAllowedByFilter(String dbName) {
631-
return isDatabaseAllowedByFilter(dbName, getIncludeDatabaseMap(), getExcludeDatabaseMap(), false);
633+
return isDatabaseAllowedByFilter(dbName, getIncludeDatabaseMap(), getExcludeDatabaseMap(),
634+
getLowerCaseDatabaseNames() != 0);
632635
}
633636

634637
private boolean isDatabaseAllowedByFilter(String dbName, Map<String, Boolean> includeDatabaseMap,
@@ -1016,7 +1019,7 @@ protected ExternalDatabase<? extends ExternalTable> buildDbForInit(String remote
10161019
long dbId, InitCatalogLog.Type logType, boolean checkExists) {
10171020
// Step 1: Map local database name if not already provided
10181021
if (localDbName == null && remoteDbName != null) {
1019-
localDbName = fromRemoteDatabaseName(remoteDbName);
1022+
localDbName = localDatabaseNameFromRemote(remoteDbName);
10201023
}
10211024

10221025
// Step 2:

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -417,6 +417,13 @@ public boolean hasLocalTableName(String tableName) {
417417
return getLocalTableName(tableName, true) != null;
418418
}
419419

420+
/** Resolve a cold table's deterministic ID without loading its evicted object. */
421+
public OptionalLong getTableIdForReplay(String tableName) {
422+
String localName = getLocalTableName(tableName, true);
423+
return localName == null ? OptionalLong.empty()
424+
: OptionalLong.of(Util.genIdByName(extCatalog.getName(), name, localName));
425+
}
426+
420427
@Override
421428
public void readLock() {
422429
this.rwLock.readLock().lock();

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -620,6 +620,10 @@ private Runnable resolveTableRowCountFence(long catalogId, String dbName,
620620
return () -> rowCountCache.invalidateTable(tableCatalogId, tableDbId, tableId);
621621
}
622622
long dbId = db.get().getId();
623+
OptionalLong tableId = db.get().getTableIdForReplay(tableName);
624+
if (tableId.isPresent()) {
625+
return () -> rowCountCache.invalidateTable(catalogId, dbId, tableId.getAsLong());
626+
}
623627
return () -> rowCountCache.invalidateDb(catalogId, dbId);
624628
}
625629
Optional<Pair<String, Long>> identity = getDbIdentityForReplay(catalogId, dbName);

0 commit comments

Comments
 (0)