Skip to content

Commit e235389

Browse files
committed
[fix](fe) Fence catalog removal and controller publication
### What problem does this PR solve? Issue Number: N/A Related PR: #69665 Problem Summary: DROP and rename can detach an authorization controller while the old catalog remains published, allowing a concurrent lookup to install a controller that escapes cleanup. In-flight authorization publication can also follow the reused catalog object's new name after rename. Unpublish the mappings before detachment and retain the original name throughout authorization publication and rollback. Removal also waited for network-bound initialization while holding the global catalog lock. Acquire the catalog monitor outside that lock, use a nonblocking manager-lock attempt with identity revalidation, and release the monitor before waiting on manager contention. Preserve synchronous retirement, captured-instance asynchronous close, journal ordering, post-lock cleanup, and monitored write-lock timing. ### Release note Prevent orphaned authorization controllers during catalog removal and keep unrelated catalog mutations responsive while another catalog initializes. ### Check List (For Author) - Test: Unit Test via run-fe-ut.sh across CatalogRemovalConcurrencyTest, CatalogMgrTest, CatalogVarbinaryMigrationTest, ExternalCatalogReplayCleanupTest, ExternalCatalogDeadlockTest, and AccessControllerManagerTest (49 tests); final FE Checkstyle (0 violations); git diff --check. Original-code counterchecks reproduced 10 expected failures among the first 11 new cases. - Behavior changed: Yes. DROP and rename wait for initialization outside the global catalog lock and fence old-name authorization publication. - Does this need documentation: No.
1 parent ba0f1a1 commit e235389

4 files changed

Lines changed: 507 additions & 89 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/common/lock/MonitoredReentrantReadWriteLock.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,14 @@ public void lock() {
112112
}
113113
}
114114

115+
@Override
116+
public boolean tryLock() {
117+
long startTime = System.nanoTime();
118+
boolean acquired = super.tryLock();
119+
monitor.afterTryLock(acquired, startTime);
120+
return acquired;
121+
}
122+
115123
/**
116124
* Releases the write lock.
117125
* Records the time when the lock is released and logs the duration.

‎fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java‎

Lines changed: 103 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@
6565
import java.util.TreeMap;
6666
import java.util.concurrent.ConcurrentMap;
6767
import java.util.function.Function;
68+
import java.util.function.Supplier;
6869
import java.util.stream.Collectors;
6970

7071
/**
@@ -125,14 +126,67 @@ private RemovedCatalog removeCatalog(long catalogId) {
125126
return null;
126127
}
127128
String catalogName = catalog.getName();
128-
// Detach the old runtime before the mapping can be reused (not later from a cleanup callback).
129-
catalog.onClose();
130-
Env.getCurrentEnv().getRefreshManager().removeFromRefreshMap(catalogId);
129+
// Authorization lookups do not take our lock. Unpublish first so they cannot install another
130+
// controller after onClose() detaches the old one. Keep the write lock until retirement finishes.
131131
idToCatalog.remove(catalogId);
132132
nameToCatalog.remove(catalogName);
133+
catalog.onClose();
134+
Env.getCurrentEnv().getRefreshManager().removeFromRefreshMap(catalogId);
133135
return new RemovedCatalog(catalog, catalogName);
134136
}
135137

138+
@FunctionalInterface
139+
private interface CatalogRemovalAction<E extends Exception> {
140+
void run(CatalogIf catalog) throws E;
141+
}
142+
143+
private <E extends Exception> void withCatalogForRemoval(Supplier<CatalogIf> lookup,
144+
CatalogRemovalAction<E> action) throws E {
145+
while (true) {
146+
CatalogIf snapshot = lookup.get();
147+
if (snapshot == null) {
148+
writeLock();
149+
try {
150+
if (lookup.get() != null) {
151+
continue;
152+
}
153+
action.run(null);
154+
return;
155+
} finally {
156+
writeUnlock();
157+
}
158+
}
159+
// Initialization can perform remote IO under this monitor. Wait without the manager lock.
160+
synchronized (snapshot) {
161+
// Other mutations still acquire manager -> catalog. Never wait for the manager while
162+
// holding the catalog monitor: release it and join the manager's wait queue instead.
163+
if (lock.writeLock().tryLock()) {
164+
try {
165+
if (lookup.get() != snapshot) {
166+
continue;
167+
}
168+
action.run(snapshot);
169+
return;
170+
} finally {
171+
writeUnlock();
172+
}
173+
}
174+
}
175+
writeLock();
176+
writeUnlock();
177+
}
178+
}
179+
180+
/** Records removal even if journaling fails, and cleans up after both locks have been released. */
181+
private final class CatalogRemoval implements AutoCloseable {
182+
private RemovedCatalog removed;
183+
184+
@Override
185+
public void close() {
186+
cleanupRemovedCatalog(removed);
187+
}
188+
}
189+
136190
private void cleanupRemovedCatalog(RemovedCatalog removedCatalog) {
137191
if (removedCatalog == null) {
138192
return;
@@ -302,58 +356,49 @@ public void createCatalog(CreateCatalogCommand cmd) throws UserException {
302356
* Remove the catalog instance by name and write the meta log.
303357
*/
304358
public void dropCatalog(String catalogName, boolean ifExists) throws UserException {
305-
RemovedCatalog removedCatalog = null;
306-
writeLock();
307-
try {
308-
if (ifExists && !nameToCatalog.containsKey(catalogName)) {
309-
LOG.warn("Non catalog {} is found.", catalogName);
310-
return;
311-
}
312-
CatalogIf<DatabaseIf<TableIf>> catalog = nameToCatalog.get(catalogName);
313-
if (catalog == null) {
314-
throw new DdlException("No catalog found with name: " + catalogName);
359+
try (CatalogRemoval removal = new CatalogRemoval()) {
360+
this.<UserException>withCatalogForRemoval(() -> nameToCatalog.get(catalogName), catalog -> {
361+
if (catalog == null) {
362+
if (ifExists) {
363+
LOG.warn("Non catalog {} is found.", catalogName);
364+
return;
365+
}
366+
throw new DdlException("No catalog found with name: " + catalogName);
367+
}
368+
CatalogLog log = new CatalogLog();
369+
log.setCatalogId(catalog.getId());
370+
removal.removed = removeCatalog(log.getCatalogId());
371+
Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_DROP_CATALOG, log);
372+
});
373+
if (removal.removed != null) {
374+
LOG.info("finished to drop catalog {}:{}", removal.removed.catalogName, removal.removed.catalogId);
315375
}
316-
CatalogLog log = new CatalogLog();
317-
log.setCatalogId(catalog.getId());
318-
removedCatalog = removeCatalog(log.getCatalogId());
319-
Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_DROP_CATALOG, log);
320-
} finally {
321-
writeUnlock();
322-
cleanupRemovedCatalog(removedCatalog);
323-
}
324-
if (removedCatalog == null) {
325-
return;
326376
}
327-
LOG.info("finished to drop catalog {}:{}", removedCatalog.catalogName, removedCatalog.catalogId);
328377
}
329378

330379
/**
331380
* Modify the catalog name into a new one and write the meta log.
332381
*/
333382
public void alterCatalogName(String catalogName, String newCatalogName) throws UserException {
334-
RemovedCatalog removedCatalog = null;
335-
writeLock();
336-
try {
337-
CatalogIf catalog = nameToCatalog.get(catalogName);
338-
if (catalog == null) {
339-
throw new DdlException("No catalog found with name: " + catalogName);
340-
}
341-
if (nameToCatalog.get(newCatalogName) != null) {
342-
throw new DdlException("Catalog with name " + newCatalogName + " already exist");
343-
}
344-
ConnectContext ctx = ConnectContext.get();
345-
String lastDb = ctx == null ? null : ctx.getLastDBOfCatalog(catalogName);
346-
removedCatalog = renameCatalog(catalog, newCatalogName);
347-
CatalogLog log = new CatalogLog();
348-
log.setCatalogId(catalog.getId());
349-
log.setNewCatalogName(newCatalogName);
350-
Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_NAME, log);
351-
if (ctx != null && lastDb != null) {
352-
ctx.addLastDBOfCatalog(newCatalogName, lastDb);
353-
}
354-
} finally {
355-
writeUnlock();
356-
cleanupRemovedCatalog(removedCatalog);
383+
try (CatalogRemoval removal = new CatalogRemoval()) {
384+
this.<UserException>withCatalogForRemoval(() -> nameToCatalog.get(catalogName), catalog -> {
385+
if (catalog == null) {
386+
throw new DdlException("No catalog found with name: " + catalogName);
387+
}
388+
if (nameToCatalog.get(newCatalogName) != null) {
389+
throw new DdlException("Catalog with name " + newCatalogName + " already exist");
390+
}
391+
ConnectContext ctx = ConnectContext.get();
392+
String lastDb = ctx == null ? null : ctx.getLastDBOfCatalog(catalogName);
393+
removal.removed = renameCatalog(catalog, newCatalogName);
394+
CatalogLog log = new CatalogLog();
395+
log.setCatalogId(catalog.getId());
396+
log.setNewCatalogName(newCatalogName);
397+
Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_NAME, log);
398+
if (ctx != null && lastDb != null) {
399+
ctx.addLastDBOfCatalog(newCatalogName, lastDb);
400+
}
401+
});
357402
}
358403
}
359404

@@ -642,31 +687,24 @@ private void createCatalogInternal(CatalogIf catalog, boolean isReplay) throws D
642687
* Reply for drop catalog event.
643688
*/
644689
public void replayDropCatalog(CatalogLog log) {
645-
RemovedCatalog removedCatalog = null;
646-
writeLock();
647-
try {
648-
removedCatalog = removeCatalog(log.getCatalogId());
649-
} finally {
650-
writeUnlock();
651-
cleanupRemovedCatalog(removedCatalog);
690+
try (CatalogRemoval removal = new CatalogRemoval()) {
691+
withCatalogForRemoval(() -> idToCatalog.get(log.getCatalogId()), catalog -> {
692+
removal.removed = removeCatalog(log.getCatalogId());
693+
});
652694
}
653695
}
654696

655697
/**
656698
* Reply for alter catalog name event.
657699
*/
658700
public void replayAlterCatalogName(CatalogLog log) {
659-
RemovedCatalog removedCatalog = null;
660-
writeLock();
661-
try {
662-
CatalogIf catalog = idToCatalog.get(log.getCatalogId());
663-
if (catalog == null) {
664-
throw new IllegalStateException("No catalog found with id: " + log.getCatalogId());
665-
}
666-
removedCatalog = renameCatalog(catalog, log.getNewCatalogName());
667-
} finally {
668-
writeUnlock();
669-
cleanupRemovedCatalog(removedCatalog);
701+
try (CatalogRemoval removal = new CatalogRemoval()) {
702+
withCatalogForRemoval(() -> idToCatalog.get(log.getCatalogId()), catalog -> {
703+
if (catalog == null) {
704+
throw new IllegalStateException("No catalog found with id: " + log.getCatalogId());
705+
}
706+
removal.removed = renameCatalog(catalog, log.getNewCatalogName());
707+
});
670708
}
671709
}
672710

‎fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/AccessControllerManager.java‎

Lines changed: 26 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -718,7 +718,7 @@ public AuthorizationPlugin getAccessControllerOrDefault(String ctl) {
718718
if (entry != null && entry.catalogId == catalog.getId()) {
719719
return entry.accessController;
720720
}
721-
lazyLoadCtlAccessController((ExternalCatalog) catalog);
721+
lazyLoadCtlAccessController((ExternalCatalog) catalog, ctl);
722722
entry = ctlToCtlAccessController.get(ctl);
723723
if (entry != null && entry.catalogId == catalog.getId()) {
724724
return entry.accessController;
@@ -728,52 +728,52 @@ public AuthorizationPlugin getAccessControllerOrDefault(String ctl) {
728728
return defaultAccessController;
729729
}
730730

731-
private void lazyLoadCtlAccessController(ExternalCatalog catalog) {
731+
private void lazyLoadCtlAccessController(ExternalCatalog catalog, String catalogName) {
732732
if (authorizesNothing) {
733733
// Same reason the constructor loaded nothing: a checkpoint Env replays a CREATE CATALOG but never
734734
// authorizes against it, and the source it would build here starts threads nothing stops.
735735
return;
736736
}
737737
CatalogAccessControllerEntry staleEntry = null;
738738
synchronized (this) {
739-
if (!isCurrentCatalog(catalog)) {
739+
if (!isCurrentCatalog(catalog, catalogName)) {
740740
return;
741741
}
742-
CatalogAccessControllerEntry entry = ctlToCtlAccessController.get(catalog.getName());
742+
CatalogAccessControllerEntry entry = ctlToCtlAccessController.get(catalogName);
743743
if (entry != null && entry.catalogId == catalog.getId()) {
744744
return;
745745
}
746-
if (entry != null && ctlToCtlAccessController.remove(catalog.getName(), entry)) {
746+
if (entry != null && ctlToCtlAccessController.remove(catalogName, entry)) {
747747
staleEntry = entry;
748748
}
749749
}
750-
closeEntry(catalog.getName(), staleEntry);
750+
closeEntry(catalogName, staleEntry);
751751

752752
catalog.initAccessController(false);
753753

754754
CatalogAccessControllerEntry displaced = null;
755755
boolean stillCurrent;
756756
synchronized (this) {
757-
stillCurrent = isCurrentCatalog(catalog);
757+
stillCurrent = isCurrentCatalog(catalog, catalogName);
758758
if (stillCurrent) {
759-
CatalogAccessControllerEntry entry = ctlToCtlAccessController.get(catalog.getName());
759+
CatalogAccessControllerEntry entry = ctlToCtlAccessController.get(catalogName);
760760
if (entry == null || entry.catalogId != catalog.getId()) {
761-
displaced = ctlToCtlAccessController.put(catalog.getName(),
761+
displaced = ctlToCtlAccessController.put(catalogName,
762762
new CatalogAccessControllerEntry(catalog.getId(), defaultAccessController, false));
763763
}
764764
}
765765
}
766-
closeEntry(catalog.getName(), displaced);
766+
closeEntry(catalogName, displaced);
767767
if (!stillCurrent) {
768768
// A DROP can complete while initAccessController() is constructing the plugin. The custom
769769
// publication path performs the same post-publication check; this also covers the fallback path.
770-
removeAccessController(catalog.getName(), catalog.getId());
770+
removeAccessController(catalogName, catalog.getId());
771771
return;
772772
}
773773
// If DROP won immediately after the synchronized publication, its onClose() removes this id. If DROP
774774
// already completed before publication, this final identity check removes the orphan ourselves.
775-
if (!isCurrentCatalog(catalog)) {
776-
removeAccessController(catalog.getName(), catalog.getId());
775+
if (!isCurrentCatalog(catalog, catalogName)) {
776+
removeAccessController(catalogName, catalog.getId());
777777
}
778778
}
779779

@@ -783,6 +783,8 @@ public boolean checkIfAccessControllerExist(String ctl) {
783783

784784
public void createAccessController(ExternalCatalog catalog, String acFactoryClassName, Map<String, String> prop,
785785
boolean isDryRun) {
786+
// Rename reuses the catalog object. Keep publication and rollback tied to the original name.
787+
String catalogName = catalog.getName();
786788
String pluginIdentifier = getPluginIdentifierForAccessController(acFactoryClassName);
787789
AuthorizationPlugin accessController = create(pluginIdentifier, prop);
788790
if (isDryRun) {
@@ -794,36 +796,36 @@ public void createAccessController(ExternalCatalog catalog, String acFactoryClas
794796
// source shares one plugin per service and keeps an unread one for a grace period, so that the
795797
// CREATE this validated for does not immediately rebuild what it just tore down - but a
796798
// validation nothing follows does stop polling once that period is up.
797-
closeAccessController(catalog.getName(), accessController);
799+
closeAccessController(catalogName, accessController);
798800
return;
799801
}
800802

801803
CatalogAccessControllerEntry displaced = null;
802804
boolean installed = false;
803805
synchronized (this) {
804-
if (isCurrentCatalog(catalog)) {
805-
CatalogAccessControllerEntry current = ctlToCtlAccessController.get(catalog.getName());
806+
if (isCurrentCatalog(catalog, catalogName)) {
807+
CatalogAccessControllerEntry current = ctlToCtlAccessController.get(catalogName);
806808
if (current == null || current.catalogId != catalog.getId()) {
807-
displaced = ctlToCtlAccessController.put(catalog.getName(),
809+
displaced = ctlToCtlAccessController.put(catalogName,
808810
new CatalogAccessControllerEntry(catalog.getId(), accessController, true));
809811
installed = true;
810812
}
811813
}
812814
}
813-
closeEntry(catalog.getName(), displaced);
815+
closeEntry(catalogName, displaced);
814816
if (!installed) {
815-
closeAccessController(catalog.getName(), accessController);
817+
closeAccessController(catalogName, accessController);
816818
return;
817819
}
818820
LOG.info("create access controller {} for catalog {}:{}",
819-
acFactoryClassName, catalog.getName(), catalog.getId());
820-
if (!isCurrentCatalog(catalog)) {
821-
removeAccessController(catalog.getName(), catalog.getId());
821+
acFactoryClassName, catalogName, catalog.getId());
822+
if (!isCurrentCatalog(catalog, catalogName)) {
823+
removeAccessController(catalogName, catalog.getId());
822824
}
823825
}
824826

825-
private boolean isCurrentCatalog(ExternalCatalog catalog) {
826-
CatalogIf currentCatalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalog.getName());
827+
private boolean isCurrentCatalog(ExternalCatalog catalog, String catalogName) {
828+
CatalogIf currentCatalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName);
827829
return currentCatalog == catalog && currentCatalog.getId() == catalog.getId();
828830
}
829831

0 commit comments

Comments
 (0)