2626
2727import com .github .benmanes .caffeine .cache .AsyncCacheLoader ;
2828import com .github .benmanes .caffeine .cache .AsyncLoadingCache ;
29+ import com .github .benmanes .caffeine .cache .Cache ;
30+ import com .github .benmanes .caffeine .cache .Caffeine ;
2931import com .github .benmanes .caffeine .cache .Ticker ;
3032import lombok .Getter ;
3133import org .apache .logging .log4j .LogManager ;
@@ -51,6 +53,10 @@ public class ExternalRowCountCache {
5153 // A catalog refresh changes this generation in O(1). Old cache entries remain bounded by
5254 // Caffeine's size/expiry policy but are never addressable by a new metadata generation.
5355 private final ConcurrentHashMap <Long , AtomicLong > catalogGenerations = new ConcurrentHashMap <>();
56+ // Database generations are bounded independently of database CREATE/DROP churn. Eviction
57+ // merely forces a new, globally unique generation and makes the old entries unreachable.
58+ private final Cache <DatabaseKey , Long > dbGenerations = Caffeine .newBuilder ()
59+ .maximumSize (Config .max_external_table_row_count_cache_num ).build ();
5460 // Generations are globally unique: two catalog IDs with the same table ID must not alias
5561 // when one catalog has retired its old cache generation.
5662 private final AtomicLong nextCatalogGeneration = new AtomicLong ();
@@ -85,16 +91,18 @@ public static class RowCountKey {
8591 private final long dbId ;
8692 private final long tableId ;
8793 private final long catalogGeneration ;
94+ private final long dbGeneration ;
8895
8996 public RowCountKey (long catalogId , long dbId , long tableId ) {
90- this (catalogId , dbId , tableId , 0 );
97+ this (catalogId , dbId , tableId , 0 , 0 );
9198 }
9299
93- private RowCountKey (long catalogId , long dbId , long tableId , long catalogGeneration ) {
100+ private RowCountKey (long catalogId , long dbId , long tableId , long catalogGeneration , long dbGeneration ) {
94101 this .catalogId = catalogId ;
95102 this .dbId = dbId ;
96103 this .tableId = tableId ;
97104 this .catalogGeneration = catalogGeneration ;
105+ this .dbGeneration = dbGeneration ;
98106 }
99107
100108 @ Override
@@ -106,12 +114,14 @@ public boolean equals(Object obj) {
106114 return false ;
107115 }
108116 RowCountKey other = (RowCountKey ) obj ;
109- return other .tableId == tableId && other .catalogGeneration == catalogGeneration ;
117+ return other .tableId == tableId && other .catalogGeneration == catalogGeneration
118+ && other .dbGeneration == dbGeneration ;
110119 }
111120
112121 @ Override
113122 public int hashCode () {
114- return 31 * Long .hashCode (tableId ) + Long .hashCode (catalogGeneration );
123+ int hash = 31 * Long .hashCode (tableId ) + Long .hashCode (catalogGeneration );
124+ return 31 * hash + Long .hashCode (dbGeneration );
115125 }
116126 }
117127
@@ -171,6 +181,33 @@ public int hashCode() {
171181 }
172182 }
173183
184+ private static final class DatabaseKey {
185+ private final long catalogId ;
186+ private final long dbId ;
187+
188+ private DatabaseKey (long catalogId , long dbId ) {
189+ this .catalogId = catalogId ;
190+ this .dbId = dbId ;
191+ }
192+
193+ @ Override
194+ public boolean equals (Object obj ) {
195+ if (this == obj ) {
196+ return true ;
197+ }
198+ if (!(obj instanceof DatabaseKey )) {
199+ return false ;
200+ }
201+ DatabaseKey other = (DatabaseKey ) obj ;
202+ return catalogId == other .catalogId && dbId == other .dbId ;
203+ }
204+
205+ @ Override
206+ public int hashCode () {
207+ return 31 * Long .hashCode (catalogId ) + Long .hashCode (dbId );
208+ }
209+ }
210+
174211 private CompletableFuture <Optional <Long >> loadWithInvalidationFence (
175212 RowCountKey key , Executor executor , Supplier <Optional <Long >> loader ) {
176213 LoadFence fence = new LoadFence ();
@@ -235,7 +272,7 @@ int getInFlightLoadCountForTest() {
235272
236273 void refreshForTest (long catalogId , long dbId , long tableId ) {
237274 rowCountCache .synchronous ().refresh (new RowCountKey (
238- catalogId , dbId , tableId , currentCatalogGeneration (catalogId )));
275+ catalogId , dbId , tableId , currentCatalogGeneration (catalogId ), currentDbGeneration ( catalogId , dbId ) ));
239276 }
240277
241278 private long currentCatalogGeneration (long catalogId ) {
@@ -248,6 +285,16 @@ private boolean isCatalogGenerationCurrent(long catalogId, long generation) {
248285 return current != null && current .get () == generation ;
249286 }
250287
288+ private long currentDbGeneration (long catalogId , long dbId ) {
289+ return dbGenerations .get (new DatabaseKey (catalogId , dbId ),
290+ ignored -> nextCatalogGeneration .incrementAndGet ());
291+ }
292+
293+ private boolean isDbGenerationCurrent (long catalogId , long dbId , long generation ) {
294+ Long current = dbGenerations .getIfPresent (new DatabaseKey (catalogId , dbId ));
295+ return current != null && current == generation ;
296+ }
297+
251298 int getCatalogGenerationCountForTest () {
252299 return catalogGenerations .size ();
253300 }
@@ -271,9 +318,7 @@ static Optional<Long> loadRowCount(RowCountKey rowCountKey, boolean fillMetaCach
271318 // so we can't try to load the row count until the cache expire.
272319 // Throw an exception here will cause too much stack log in fe.out.
273320 // So we return null when exception happen.
274- // Null may raise NPE in caller, but that is expected.
275- // We catch that NPE and return a default value -1 without keep the value in cache,
276- // so we can trigger the load function to fetch row count again next time in this exception case.
321+ // The caller treats null as an uncached UNKNOWN result, so the next read can retry.
277322 return null ;
278323 }
279324 }
@@ -288,10 +333,12 @@ public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean f
288333 try {
289334 CompletableFuture <Optional <Long >> f ;
290335 long generation ;
336+ long dbGeneration ;
291337 publicationLock .readLock ().lock ();
292338 try {
293339 generation = currentCatalogGeneration (catalogId );
294- RowCountKey key = new RowCountKey (catalogId , dbId , tableId , generation );
340+ dbGeneration = currentDbGeneration (catalogId , dbId );
341+ RowCountKey key = new RowCountKey (catalogId , dbId , tableId , generation , dbGeneration );
295342 f = fillMetaCache
296343 ? rowCountCache .get (key , (rowCountKey , executor ) -> loadWithInvalidationFence (
297344 rowCountKey , executor , () -> loadRowCount (rowCountKey , true )))
@@ -303,12 +350,14 @@ public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean f
303350 if (ConnectContext .get () == null
304351 || ConnectContext .get ().getSessionVariable ().fetchHiveRowCountSync ) {
305352 Optional <Long > value = f .get ();
306- return isCatalogGenerationCurrent (catalogId , generation )
353+ return value != null && isCatalogGenerationCurrent (catalogId , generation )
354+ && isDbGenerationCurrent (catalogId , dbId , dbGeneration )
307355 ? value .orElse (TableIf .UNKNOWN_ROW_COUNT ) : TableIf .UNKNOWN_ROW_COUNT ;
308356 } else {
309357 if (f .isDone ()) {
310358 Optional <Long > value = f .get ();
311- return isCatalogGenerationCurrent (catalogId , generation )
359+ return value != null && isCatalogGenerationCurrent (catalogId , generation )
360+ && isDbGenerationCurrent (catalogId , dbId , dbGeneration )
312361 ? value .orElse (TableIf .UNKNOWN_ROW_COUNT ) : TableIf .UNKNOWN_ROW_COUNT ;
313362 }
314363 LOG .info ("Row count for table {}.{}.{} is still processing." , catalogId , dbId , tableId );
@@ -328,22 +377,30 @@ public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId)
328377 try {
329378 CompletableFuture <Optional <Long >> f ;
330379 long generation ;
380+ long dbGeneration ;
331381 publicationLock .readLock ().lock ();
332382 try {
333383 AtomicLong current = catalogGenerations .get (catalogId );
334384 if (current == null ) {
335385 return -1 ;
336386 }
337387 generation = current .get ();
338- RowCountKey key = new RowCountKey (catalogId , dbId , tableId , generation );
388+ Long currentDb = dbGenerations .getIfPresent (new DatabaseKey (catalogId , dbId ));
389+ if (currentDb == null ) {
390+ return -1 ;
391+ }
392+ dbGeneration = currentDb ;
393+ RowCountKey key = new RowCountKey (catalogId , dbId , tableId , generation , dbGeneration );
339394 f = rowCountCache .getIfPresent (key );
340395 } finally {
341396 publicationLock .readLock ().unlock ();
342397 }
343398 if (f == null ) {
344399 return -1 ;
345- } else if (f .isDone () && isCatalogGenerationCurrent (catalogId , generation )) {
346- return f .get ().orElse (-1L );
400+ } else if (f .isDone () && isCatalogGenerationCurrent (catalogId , generation )
401+ && isDbGenerationCurrent (catalogId , dbId , dbGeneration )) {
402+ Optional <Long > value = f .get ();
403+ return value == null ? -1L : value .orElse (-1L );
347404 }
348405 } catch (Exception e ) {
349406 LOG .warn ("Unexpected exception while returning row count if present" , e );
@@ -378,12 +435,11 @@ void releaseCatalog(long catalogId) {
378435 void invalidateDb (long catalogId , long dbId ) {
379436 publicationLock .writeLock ().lock ();
380437 try {
381- inFlightLoads .forEach ((key , fences ) -> {
382- if (key .catalogId == catalogId && key .dbId == dbId ) {
383- fences .forEach (fence -> fence .invalidated = true );
384- }
385- });
386- rowCountCache .asMap ().keySet ().removeIf (key -> key .catalogId == catalogId && key .dbId == dbId );
438+ DatabaseKey key = new DatabaseKey (catalogId , dbId );
439+ Long current = dbGenerations .getIfPresent (key );
440+ if (current != null ) {
441+ dbGenerations .put (key , nextCatalogGeneration .incrementAndGet ());
442+ }
387443 } finally {
388444 publicationLock .writeLock ().unlock ();
389445 }
@@ -396,7 +452,11 @@ void invalidateTable(long catalogId, long dbId, long tableId) {
396452 if (current == null ) {
397453 return ;
398454 }
399- RowCountKey key = new RowCountKey (catalogId , dbId , tableId , current .get ());
455+ Long currentDb = dbGenerations .getIfPresent (new DatabaseKey (catalogId , dbId ));
456+ if (currentDb == null ) {
457+ return ;
458+ }
459+ RowCountKey key = new RowCountKey (catalogId , dbId , tableId , current .get (), currentDb );
400460 Set <LoadFence > fences = inFlightLoads .get (new LoadKey (key ));
401461 if (fences != null ) {
402462 fences .forEach (fence -> fence .invalidated = true );
0 commit comments