Skip to content

Commit d962085

Browse files
committed
[fix](fe) Bound external row-count database invalidation
### What problem does this PR solve? Issue Number: None Related PR: #68196 Problem Summary: REFRESH DATABASE fenced external row counts by scanning every cached entry and in-flight load twice under the global publication write lock, delaying unrelated catalog planning. An invalidated async load also completed with null, and the synchronous reader dereferenced it as Optional, emitting an unexpected NPE warning during ordinary refresh/read overlap. Give each database a globally unique, bounded cache generation so invalidation is constant time and old loads cannot publish or return stale counts. Treat a null load result as an expected, non-cacheable UNKNOWN row count. ### Release note External database refresh no longer scans unrelated row-count caches or logs an NPE for an invalidated load. ### Check List (For Author) - Test: Unit Test: ExternalRowCountCacheTest, 17 tests passed; full ./build.sh --fe -j4 and Checkstyle passed - Behavior changed: Yes, database row-count invalidation uses a bounded generation fence and invalidated reads return UNKNOWN without an exception log - Does this need documentation: No
1 parent 04fa4af commit d962085

2 files changed

Lines changed: 156 additions & 21 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java‎

Lines changed: 81 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@
2626

2727
import com.github.benmanes.caffeine.cache.AsyncCacheLoader;
2828
import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
29+
import com.github.benmanes.caffeine.cache.Cache;
30+
import com.github.benmanes.caffeine.cache.Caffeine;
2931
import com.github.benmanes.caffeine.cache.Ticker;
3032
import lombok.Getter;
3133
import org.apache.logging.log4j.LogManager;
@@ -51,6 +53,10 @@ public class ExternalRowCountCache {
5153
// A catalog refresh changes this generation in O(1). Old cache entries remain bounded by
5254
// Caffeine's size/expiry policy but are never addressable by a new metadata generation.
5355
private final ConcurrentHashMap<Long, AtomicLong> catalogGenerations = new ConcurrentHashMap<>();
56+
// Database generations are bounded independently of database CREATE/DROP churn. Eviction
57+
// merely forces a new, globally unique generation and makes the old entries unreachable.
58+
private final Cache<DatabaseKey, Long> dbGenerations = Caffeine.newBuilder()
59+
.maximumSize(Config.max_external_table_row_count_cache_num).build();
5460
// Generations are globally unique: two catalog IDs with the same table ID must not alias
5561
// when one catalog has retired its old cache generation.
5662
private final AtomicLong nextCatalogGeneration = new AtomicLong();
@@ -85,16 +91,18 @@ public static class RowCountKey {
8591
private final long dbId;
8692
private final long tableId;
8793
private final long catalogGeneration;
94+
private final long dbGeneration;
8895

8996
public RowCountKey(long catalogId, long dbId, long tableId) {
90-
this(catalogId, dbId, tableId, 0);
97+
this(catalogId, dbId, tableId, 0, 0);
9198
}
9299

93-
private RowCountKey(long catalogId, long dbId, long tableId, long catalogGeneration) {
100+
private RowCountKey(long catalogId, long dbId, long tableId, long catalogGeneration, long dbGeneration) {
94101
this.catalogId = catalogId;
95102
this.dbId = dbId;
96103
this.tableId = tableId;
97104
this.catalogGeneration = catalogGeneration;
105+
this.dbGeneration = dbGeneration;
98106
}
99107

100108
@Override
@@ -106,12 +114,14 @@ public boolean equals(Object obj) {
106114
return false;
107115
}
108116
RowCountKey other = (RowCountKey) obj;
109-
return other.tableId == tableId && other.catalogGeneration == catalogGeneration;
117+
return other.tableId == tableId && other.catalogGeneration == catalogGeneration
118+
&& other.dbGeneration == dbGeneration;
110119
}
111120

112121
@Override
113122
public int hashCode() {
114-
return 31 * Long.hashCode(tableId) + Long.hashCode(catalogGeneration);
123+
int hash = 31 * Long.hashCode(tableId) + Long.hashCode(catalogGeneration);
124+
return 31 * hash + Long.hashCode(dbGeneration);
115125
}
116126
}
117127

@@ -171,6 +181,33 @@ public int hashCode() {
171181
}
172182
}
173183

184+
private static final class DatabaseKey {
185+
private final long catalogId;
186+
private final long dbId;
187+
188+
private DatabaseKey(long catalogId, long dbId) {
189+
this.catalogId = catalogId;
190+
this.dbId = dbId;
191+
}
192+
193+
@Override
194+
public boolean equals(Object obj) {
195+
if (this == obj) {
196+
return true;
197+
}
198+
if (!(obj instanceof DatabaseKey)) {
199+
return false;
200+
}
201+
DatabaseKey other = (DatabaseKey) obj;
202+
return catalogId == other.catalogId && dbId == other.dbId;
203+
}
204+
205+
@Override
206+
public int hashCode() {
207+
return 31 * Long.hashCode(catalogId) + Long.hashCode(dbId);
208+
}
209+
}
210+
174211
private CompletableFuture<Optional<Long>> loadWithInvalidationFence(
175212
RowCountKey key, Executor executor, Supplier<Optional<Long>> loader) {
176213
LoadFence fence = new LoadFence();
@@ -235,7 +272,7 @@ int getInFlightLoadCountForTest() {
235272

236273
void refreshForTest(long catalogId, long dbId, long tableId) {
237274
rowCountCache.synchronous().refresh(new RowCountKey(
238-
catalogId, dbId, tableId, currentCatalogGeneration(catalogId)));
275+
catalogId, dbId, tableId, currentCatalogGeneration(catalogId), currentDbGeneration(catalogId, dbId)));
239276
}
240277

241278
private long currentCatalogGeneration(long catalogId) {
@@ -248,6 +285,16 @@ private boolean isCatalogGenerationCurrent(long catalogId, long generation) {
248285
return current != null && current.get() == generation;
249286
}
250287

288+
private long currentDbGeneration(long catalogId, long dbId) {
289+
return dbGenerations.get(new DatabaseKey(catalogId, dbId),
290+
ignored -> nextCatalogGeneration.incrementAndGet());
291+
}
292+
293+
private boolean isDbGenerationCurrent(long catalogId, long dbId, long generation) {
294+
Long current = dbGenerations.getIfPresent(new DatabaseKey(catalogId, dbId));
295+
return current != null && current == generation;
296+
}
297+
251298
int getCatalogGenerationCountForTest() {
252299
return catalogGenerations.size();
253300
}
@@ -271,9 +318,7 @@ static Optional<Long> loadRowCount(RowCountKey rowCountKey, boolean fillMetaCach
271318
// so we can't try to load the row count until the cache expire.
272319
// Throw an exception here will cause too much stack log in fe.out.
273320
// So we return null when exception happen.
274-
// Null may raise NPE in caller, but that is expected.
275-
// We catch that NPE and return a default value -1 without keep the value in cache,
276-
// so we can trigger the load function to fetch row count again next time in this exception case.
321+
// The caller treats null as an uncached UNKNOWN result, so the next read can retry.
277322
return null;
278323
}
279324
}
@@ -288,10 +333,12 @@ public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean f
288333
try {
289334
CompletableFuture<Optional<Long>> f;
290335
long generation;
336+
long dbGeneration;
291337
publicationLock.readLock().lock();
292338
try {
293339
generation = currentCatalogGeneration(catalogId);
294-
RowCountKey key = new RowCountKey(catalogId, dbId, tableId, generation);
340+
dbGeneration = currentDbGeneration(catalogId, dbId);
341+
RowCountKey key = new RowCountKey(catalogId, dbId, tableId, generation, dbGeneration);
295342
f = fillMetaCache
296343
? rowCountCache.get(key, (rowCountKey, executor) -> loadWithInvalidationFence(
297344
rowCountKey, executor, () -> loadRowCount(rowCountKey, true)))
@@ -303,12 +350,14 @@ public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean f
303350
if (ConnectContext.get() == null
304351
|| ConnectContext.get().getSessionVariable().fetchHiveRowCountSync) {
305352
Optional<Long> value = f.get();
306-
return isCatalogGenerationCurrent(catalogId, generation)
353+
return value != null && isCatalogGenerationCurrent(catalogId, generation)
354+
&& isDbGenerationCurrent(catalogId, dbId, dbGeneration)
307355
? value.orElse(TableIf.UNKNOWN_ROW_COUNT) : TableIf.UNKNOWN_ROW_COUNT;
308356
} else {
309357
if (f.isDone()) {
310358
Optional<Long> value = f.get();
311-
return isCatalogGenerationCurrent(catalogId, generation)
359+
return value != null && isCatalogGenerationCurrent(catalogId, generation)
360+
&& isDbGenerationCurrent(catalogId, dbId, dbGeneration)
312361
? value.orElse(TableIf.UNKNOWN_ROW_COUNT) : TableIf.UNKNOWN_ROW_COUNT;
313362
}
314363
LOG.info("Row count for table {}.{}.{} is still processing.", catalogId, dbId, tableId);
@@ -328,22 +377,30 @@ public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId)
328377
try {
329378
CompletableFuture<Optional<Long>> f;
330379
long generation;
380+
long dbGeneration;
331381
publicationLock.readLock().lock();
332382
try {
333383
AtomicLong current = catalogGenerations.get(catalogId);
334384
if (current == null) {
335385
return -1;
336386
}
337387
generation = current.get();
338-
RowCountKey key = new RowCountKey(catalogId, dbId, tableId, generation);
388+
Long currentDb = dbGenerations.getIfPresent(new DatabaseKey(catalogId, dbId));
389+
if (currentDb == null) {
390+
return -1;
391+
}
392+
dbGeneration = currentDb;
393+
RowCountKey key = new RowCountKey(catalogId, dbId, tableId, generation, dbGeneration);
339394
f = rowCountCache.getIfPresent(key);
340395
} finally {
341396
publicationLock.readLock().unlock();
342397
}
343398
if (f == null) {
344399
return -1;
345-
} else if (f.isDone() && isCatalogGenerationCurrent(catalogId, generation)) {
346-
return f.get().orElse(-1L);
400+
} else if (f.isDone() && isCatalogGenerationCurrent(catalogId, generation)
401+
&& isDbGenerationCurrent(catalogId, dbId, dbGeneration)) {
402+
Optional<Long> value = f.get();
403+
return value == null ? -1L : value.orElse(-1L);
347404
}
348405
} catch (Exception e) {
349406
LOG.warn("Unexpected exception while returning row count if present", e);
@@ -378,12 +435,11 @@ void releaseCatalog(long catalogId) {
378435
void invalidateDb(long catalogId, long dbId) {
379436
publicationLock.writeLock().lock();
380437
try {
381-
inFlightLoads.forEach((key, fences) -> {
382-
if (key.catalogId == catalogId && key.dbId == dbId) {
383-
fences.forEach(fence -> fence.invalidated = true);
384-
}
385-
});
386-
rowCountCache.asMap().keySet().removeIf(key -> key.catalogId == catalogId && key.dbId == dbId);
438+
DatabaseKey key = new DatabaseKey(catalogId, dbId);
439+
Long current = dbGenerations.getIfPresent(key);
440+
if (current != null) {
441+
dbGenerations.put(key, nextCatalogGeneration.incrementAndGet());
442+
}
387443
} finally {
388444
publicationLock.writeLock().unlock();
389445
}
@@ -396,7 +452,11 @@ void invalidateTable(long catalogId, long dbId, long tableId) {
396452
if (current == null) {
397453
return;
398454
}
399-
RowCountKey key = new RowCountKey(catalogId, dbId, tableId, current.get());
455+
Long currentDb = dbGenerations.getIfPresent(new DatabaseKey(catalogId, dbId));
456+
if (currentDb == null) {
457+
return;
458+
}
459+
RowCountKey key = new RowCountKey(catalogId, dbId, tableId, current.get(), currentDb);
400460
Set<LoadFence> fences = inFlightLoads.get(new LoadKey(key));
401461
if (fences != null) {
402462
fences.forEach(fence -> fence.invalidated = true);

‎fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,81 @@ protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey key) {
333333
}
334334
}
335335

336+
@Test
337+
public void testDatabaseGenerationFenceKeepsOtherDatabasesReadable() throws Exception {
338+
AtomicInteger loadCount = new AtomicInteger();
339+
CountDownLatch oldLoadStarted = new CountDownLatch(1);
340+
CountDownLatch releaseOldLoad = new CountDownLatch(1);
341+
ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() {
342+
@Override
343+
protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey key) {
344+
if (key.getDbId() == 10 && loadCount.incrementAndGet() == 1) {
345+
oldLoadStarted.countDown();
346+
Uninterruptibles.awaitUninterruptibly(releaseOldLoad);
347+
return Optional.of(100L);
348+
}
349+
return Optional.of(key.getDbId() * 100L + loadCount.get());
350+
}
351+
};
352+
ExecutorService cacheExecutor = Executors.newFixedThreadPool(2);
353+
ExecutorService readerExecutor = Executors.newSingleThreadExecutor();
354+
try {
355+
ExternalRowCountCache cache = new ExternalRowCountCache(cacheExecutor, null, loader);
356+
Assertions.assertEquals(2000L, cache.getCachedRowCount(1, 20, 100, false));
357+
Future<Long> oldRead = readerExecutor.submit(() -> cache.getCachedRowCount(1, 10, 100, false));
358+
Assertions.assertTrue(oldLoadStarted.await(10, TimeUnit.SECONDS));
359+
360+
cache.invalidateDb(1, 10);
361+
Assertions.assertEquals(2000L, cache.getCachedRowCountIfPresent(1, 20, 100));
362+
Assertions.assertEquals(1002L, cache.getCachedRowCount(1, 10, 100, false));
363+
releaseOldLoad.countDown();
364+
365+
Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, oldRead.get(10, TimeUnit.SECONDS));
366+
Assertions.assertEquals(1002L, cache.getCachedRowCountIfPresent(1, 10, 100));
367+
Assertions.assertEquals(2000L, cache.getCachedRowCountIfPresent(1, 20, 100));
368+
} finally {
369+
releaseOldLoad.countDown();
370+
readerExecutor.shutdownNow();
371+
cacheExecutor.shutdownNow();
372+
}
373+
}
374+
375+
@Test
376+
public void testInvalidatedNullLoadReturnsUnknownWithoutReusingIt() throws Exception {
377+
AtomicInteger loadCount = new AtomicInteger();
378+
CountDownLatch loadStarted = new CountDownLatch(1);
379+
CountDownLatch releaseLoad = new CountDownLatch(1);
380+
ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() {
381+
@Override
382+
protected Optional<Long> doLoad(ExternalRowCountCache.RowCountKey key) {
383+
if (loadCount.incrementAndGet() == 1) {
384+
loadStarted.countDown();
385+
Uninterruptibles.awaitUninterruptibly(releaseLoad);
386+
}
387+
return Optional.of(100L);
388+
}
389+
};
390+
ExecutorService cacheExecutor = Executors.newSingleThreadExecutor();
391+
ExecutorService readerExecutor = Executors.newSingleThreadExecutor();
392+
try {
393+
ExternalRowCountCache cache = new ExternalRowCountCache(cacheExecutor, null, loader);
394+
Future<Long> oldRead = readerExecutor.submit(() -> cache.getCachedRowCount(1, 10, 100, false));
395+
Assertions.assertTrue(loadStarted.await(10, TimeUnit.SECONDS));
396+
397+
cache.invalidateTable(1, 10, 100);
398+
releaseLoad.countDown();
399+
400+
Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, oldRead.get(10, TimeUnit.SECONDS));
401+
Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cache.getCachedRowCountIfPresent(1, 10, 100));
402+
Assertions.assertEquals(100L, cache.getCachedRowCount(1, 10, 100, false));
403+
Assertions.assertEquals(2, loadCount.get());
404+
} finally {
405+
releaseLoad.countDown();
406+
readerExecutor.shutdownNow();
407+
cacheExecutor.shutdownNow();
408+
}
409+
}
410+
336411
@Test
337412
public void testPermanentCatalogRemovalReleasesGenerationWithoutRevivingOldReader() throws Exception {
338413
CountDownLatch loadStarted = new CountDownLatch(1);

0 commit comments

Comments
 (0)