Skip to content

Commit 90540d3

Browse files
authored
[fix](hive) invalidate stale file cache on partition refresh when partition cache misses (apache#65334)
Problem Summary: Partition-level refresh / invalidation (`HiveExternalMetaCache.invalidatePartitionCache`) returned early when the target partition was absent from the partition metadata cache — evicted under `max_hive_partition_cache_num` pressure, or simply never loaded — and left that partition's file listing cache untouched. The partition cache (`partition`) and the file listing cache (`file`) are independent caches with independent size limits (`max_hive_partition_cache_num` vs `max_external_file_cache_num`) and evict separately, so the partition entry can be evicted while its file listing survives. In that state a partition-level refresh cannot rebuild the exact `FileCacheKey` (it needs the partition path / input format that only the cached `HivePartition` carries), so the old code gave up and the stale file listing remained. A later query would reload the partition metadata and re-hit the same stale file listing — i.e. "the partition exists, but Doris keeps seeing the old files/data". Fix: when the exact `FileCacheKey` cannot be rebuilt, fall back to a predicate-based invalidation on the file cache keyed by `table id + partition values`, so the stale listing for that partition is cleared without over-invalidating other partitions. This mirrors table-level invalidation, which already invalidates the file cache by predicate. The partition invalidation coordinator is also decoupled from `ExternalTable` to `NameMapping` (mirroring the existing `invalidateTableCache(NameMapping)`) so the path can be unit tested directly. This behavior has existed since partition-level invalidation was first introduced in apache#15702 (the file cache was only invalidated inside the `if (partition != null)` branch), and was later carried into the current `HiveExternalMetaCache` by the apache#60937 cache-framework refactor. ### Release note Fix a Hive external catalog consistency issue where a partition-level refresh could leave a stale file listing cached when the partition metadata cache had been evicted, causing queries to keep seeing outdated files for that partition.
1 parent bd10160 commit 90540d3

2 files changed

Lines changed: 85 additions & 18 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java‎

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -582,7 +582,11 @@ public void invalidateTableCache(NameMapping nameMapping) {
582582
}
583583

584584
public void invalidatePartitionCache(ExternalTable dorisTable, String partitionName) {
585-
partitionCacheCoordinator.invalidatePartitionCache(dorisTable, partitionName);
585+
invalidatePartitionCache(dorisTable.getOrBuildNameMapping(), partitionName);
586+
}
587+
588+
public void invalidatePartitionCache(NameMapping nameMapping, String partitionName) {
589+
partitionCacheCoordinator.invalidatePartitionCache(nameMapping, partitionName);
586590
}
587591

588592
/**
@@ -614,33 +618,34 @@ public void dropPartitionsCache(ExternalTable dorisTable,
614618
}
615619

616620
private final class PartitionCacheCoordinator {
617-
private void invalidatePartitionCache(ExternalTable dorisTable, String partitionName) {
618-
NameMapping nameMapping = dorisTable.getOrBuildNameMapping();
621+
private void invalidatePartitionCache(NameMapping nameMapping, String partitionName) {
619622
long catalogId = nameMapping.getCtlId();
620623

621-
MetaCacheEntry<PartitionValueCacheKey, HivePartitionValues> partitionValuesEntry =
622-
partitionValuesEntryIfInitialized(catalogId);
623624
MetaCacheEntry<PartitionCacheKey, HivePartition> partitionEntry = partitionEntryIfInitialized(catalogId);
624625
MetaCacheEntry<FileCacheKey, FileCacheValue> fileEntry = fileEntryIfInitialized(catalogId);
625-
if (partitionValuesEntry == null || partitionEntry == null || fileEntry == null) {
626+
if (partitionEntry == null || fileEntry == null) {
626627
return;
627628
}
628629

629630
long tableId = Util.genIdByName(nameMapping.getLocalDbName(), nameMapping.getLocalTblName());
630-
PartitionValueCacheKey key = new PartitionValueCacheKey(nameMapping, null);
631-
HivePartitionValues partitionValues = partitionValuesEntry.getIfPresent(key);
632-
if (partitionValues == null) {
633-
return;
634-
}
635-
636-
List<String> values = partitionValues.nameToPartitionValues.get(partitionName);
637-
if (values == null) {
638-
return;
639-
}
631+
// Derive the partition values directly from the partition name (the partition-values cache is
632+
// populated the same way, via HiveUtil.toPartitionValues) instead of reading them from that
633+
// cache, so file cache invalidation still runs when the table's partition-values cache entry has
634+
// been evicted while its file listings are still cached.
635+
List<String> values = HiveUtil.toPartitionValues(partitionName);
640636

641637
PartitionCacheKey partKey = new PartitionCacheKey(nameMapping, values);
642638
HivePartition partition = partitionEntry.getIfPresent(partKey);
643639
if (partition == null) {
640+
// Partition metadata cache miss: the exact FileCacheKey cannot be rebuilt here because it
641+
// needs the partition path and input format carried by HivePartition. Invalidate this
642+
// table's cached file listings for the partition by (table id + partition values). Scoping
643+
// by table id is intentional: matching partition values alone would also drop other tables'
644+
// listings that merely share the same partition value names (e.g. dt=...) at a different
645+
// location, forcing needless re-listing. The exact-key path below (taken when the partition
646+
// is cached) already clears a listing regardless of which table id populated it.
647+
fileEntry.invalidateIf(k -> k.isSameTable(tableId) && Objects.equals(k.getPartitionValues(), values));
648+
partitionEntry.invalidateKey(partKey);
644649
return;
645650
}
646651

@@ -685,7 +690,7 @@ private void refreshAffectedPartitionsCache(HMSExternalTable table,
685690
List<String> modifiedPartNames,
686691
List<String> newPartNames) {
687692
for (String partitionName : modifiedPartNames) {
688-
invalidatePartitionCache(table, partitionName);
693+
invalidatePartitionCache(table.getOrBuildNameMapping(), partitionName);
689694
}
690695

691696
List<String> mergedPartNames = Lists.newArrayList(modifiedPartNames);
@@ -772,7 +777,7 @@ private void dropPartitionsCache(ExternalTable dorisTable,
772777
nameToPartitionValues.remove(partitionName);
773778

774779
if (invalidPartitionCache) {
775-
invalidatePartitionCache(dorisTable, partitionName);
780+
invalidatePartitionCache(nameMapping, partitionName);
776781
}
777782
}
778783

‎fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,68 @@ public void testInvalidateTableCache() {
8686
Assertions.assertEquals(0, entrySize(partitionValuesCache));
8787
}
8888

89+
@Test
90+
public void testInvalidatePartitionCacheClearsStaleFileCacheOnPartitionMiss() {
91+
ThreadPoolExecutor executor = ThreadPoolManager.newDaemonFixedThreadPool(
92+
1, 1, "refresh", 1, false);
93+
ThreadPoolExecutor listExecutor = ThreadPoolManager.newDaemonFixedThreadPool(
94+
1, 1, "file", 1, false);
95+
try {
96+
HiveExternalMetaCache cache = new HiveExternalMetaCache(executor, listExecutor);
97+
cache.initCatalog(0, new HashMap<>());
98+
99+
MetaCacheEntry<HiveExternalMetaCache.FileCacheKey, HiveExternalMetaCache.FileCacheValue> fileCache =
100+
cache.entry(0, HiveExternalMetaCache.ENTRY_FILE,
101+
HiveExternalMetaCache.FileCacheKey.class,
102+
HiveExternalMetaCache.FileCacheValue.class);
103+
104+
String dbName = "db";
105+
String tbName = "tb";
106+
NameMapping nameMapping = NameMapping.createForTest(dbName, tbName);
107+
long catalogId = nameMapping.getCtlId();
108+
long tableId = Util.genIdByName(dbName, tbName);
109+
long otherTableId = Util.genIdByName(dbName, "tb2");
110+
111+
String targetPartName = "dt=2024-01-01";
112+
List<String> targetValues = Collections.singletonList("2024-01-01");
113+
String otherPartName = "dt=2024-01-02";
114+
List<String> otherValues = Collections.singletonList("2024-01-02");
115+
116+
// Neither the `partition` cache nor the `partition_values` cache is populated for this table,
117+
// simulating entries that were evicted or never loaded. invalidatePartitionCache must still
118+
// clear the stale file listing: it derives the partition values from the partition name and
119+
// cannot rebuild the exact FileCacheKey (which needs the partition path / input format).
120+
HiveExternalMetaCache.FileCacheKey targetFileKey = new HiveExternalMetaCache.FileCacheKey(
121+
catalogId, tableId, "/wh/db/tb/" + targetPartName, "orc", targetValues);
122+
// Same table, a different partition -> must be kept.
123+
HiveExternalMetaCache.FileCacheKey otherPartFileKey = new HiveExternalMetaCache.FileCacheKey(
124+
catalogId, tableId, "/wh/db/tb/" + otherPartName, "orc", otherValues);
125+
// A different table that merely shares the same partition value names at a different location
126+
// -> must be kept (the fallback is intentionally scoped by table id, not by values alone).
127+
HiveExternalMetaCache.FileCacheKey otherTableFileKey = new HiveExternalMetaCache.FileCacheKey(
128+
catalogId, otherTableId, "/wh/db/tb2/" + targetPartName, "orc", targetValues);
129+
fileCache.put(targetFileKey, new HiveExternalMetaCache.FileCacheValue());
130+
fileCache.put(otherPartFileKey, new HiveExternalMetaCache.FileCacheValue());
131+
fileCache.put(otherTableFileKey, new HiveExternalMetaCache.FileCacheValue());
132+
Assertions.assertEquals(3, entrySize(fileCache));
133+
134+
// Partition-level refresh for the target partition. Even though its `partition` cache entry
135+
// is missing, the stale file listing for that partition must still be invalidated.
136+
cache.invalidatePartitionCache(nameMapping, targetPartName);
137+
138+
Assertions.assertNull(fileCache.getIfPresent(targetFileKey),
139+
"stale file cache for the refreshed partition must be cleared even on partition cache miss");
140+
Assertions.assertNotNull(fileCache.getIfPresent(otherPartFileKey),
141+
"file cache for other partitions of the same table must NOT be affected");
142+
Assertions.assertNotNull(fileCache.getIfPresent(otherTableFileKey),
143+
"file cache for other tables sharing the same partition values must NOT be affected");
144+
Assertions.assertEquals(2, entrySize(fileCache));
145+
} finally {
146+
executor.shutdownNow();
147+
listExecutor.shutdownNow();
148+
}
149+
}
150+
89151
@Test
90152
public void testDefaultSpecsFollowConfig() {
91153
ThreadPoolExecutor executor = ThreadPoolManager.newDaemonFixedThreadPool(

0 commit comments

Comments
 (0)