Skip to content

Commit ac33193

Browse files
committed
[fix](cloud) Invalidate caches after visible commit retries
### What problem does this PR solve? Issue Number: close apache#67099 Related PR: apache#67813 Problem Summary: A cloud commit retry may return a VISIBLE transaction without version results. Invalidate the affected table and partition caches and notify peer FEs through the existing version RPC, so subsequent reads fetch fresh versions without making a durable commit fail on a version-service outage. Protect batch cache publication with version write locks and use in-memory epochs to prevent older RPC completions from clearing invalidation. ### Release note Prevent stale reads and false commit failures after cloud commit retries. ### Check List (For Author) - Test: Unit Test - 55 FE tests passed via run-fe-ut.sh (CloudGlobalTransactionMgrTest, CloudPartitionTest, OlapTableTest); FE Checkstyle: 0 violations; git diff --check passed. - Behavior changed: Yes. Missing-version VISIBLE responses invalidate caches; successful commit callbacks and return semantics are preserved. - Does this need documentation: No
1 parent a22b311 commit ac33193

7 files changed

Lines changed: 409 additions & 87 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,10 @@ public enum OlapTableState {
248248
private volatile long lastTableVersionCachedTimeMs = 0;
249249
private volatile long cachedTableVersion = -1;
250250

251+
// Commit notifications cannot clear invalidation: their versions may precede the missing commit result.
252+
private final AtomicLong tableVersionCacheEpoch = new AtomicLong();
253+
private final AtomicLong refreshedTableVersionCacheEpoch = new AtomicLong();
254+
251255
private ReadWriteLock versionLock = Config.isCloudMode() ? new ReentrantReadWriteLock(true) : null;
252256

253257
public OlapTable() {
@@ -3591,7 +3595,7 @@ public long getNextVersion() {
35913595
@VisibleForTesting
35923596
protected boolean isCachedTableVersionExpired() {
35933597
// -1 means no cache yet, need to fetch from MS
3594-
if (cachedTableVersion == -1) {
3598+
if (cachedTableVersion == -1 || tableVersionCacheEpoch.get() != refreshedTableVersionCacheEpoch.get()) {
35953599
return true;
35963600
}
35973601
ConnectContext ctx = ConnectContext.get();
@@ -3605,13 +3609,18 @@ protected boolean isCachedTableVersionExpired() {
36053609

36063610
public boolean isCachedTableVersionExpired(long expirationMs) {
36073611
// -1 means no cache yet, need to fetch from MS
3608-
if (cachedTableVersion == -1 || expirationMs <= 0) {
3612+
if (cachedTableVersion == -1 || expirationMs <= 0
3613+
|| tableVersionCacheEpoch.get() != refreshedTableVersionCacheEpoch.get()) {
36093614
return true;
36103615
}
36113616
return System.currentTimeMillis() - lastTableVersionCachedTimeMs > expirationMs;
36123617
}
36133618

3614-
public void setCachedTableVersion(long version) {
3619+
public void invalidateCachedTableVersion() {
3620+
tableVersionCacheEpoch.incrementAndGet();
3621+
}
3622+
3623+
public synchronized void setCachedTableVersion(long version) {
36153624
if (version >= cachedTableVersion) {
36163625
cachedTableVersion = version;
36173626
lastTableVersionCachedTimeMs = System.currentTimeMillis();
@@ -3632,6 +3641,7 @@ public long getVisibleVersion() throws RpcException {
36323641
return getCachedTableVersion();
36333642
}
36343643

3644+
long cacheEpoch = tableVersionCacheEpoch.get();
36353645
// get version rpc
36363646
Cloud.GetVersionRequest request = Cloud.GetVersionRequest.newBuilder()
36373647
.setRequestIp(FrontendOptions.getLocalHostAddressCached())
@@ -3658,6 +3668,7 @@ public long getVisibleVersion() throws RpcException {
36583668
}
36593669
// update cache
36603670
setCachedTableVersion(version);
3671+
refreshedTableVersionCacheEpoch.accumulateAndGet(cacheEpoch, Math::max);
36613672
return version;
36623673
} catch (RpcException e) {
36633674
LOG.warn("get version from meta service failed", e);
@@ -3722,9 +3733,11 @@ public static List<Long> getVisibleVersionInBatch(List<OlapTable> tables) {
37223733
private static List<Long> getVisibleVersionInBatchFromMs(List<OlapTable> tables) {
37233734
List<Long> dbIds = new ArrayList<>(tables.size());
37243735
List<Long> tableIds = new ArrayList<>(tables.size());
3736+
List<Long> cacheEpochs = new ArrayList<>(tables.size());
37253737
for (OlapTable table : tables) {
37263738
dbIds.add(table.getDatabase().getId());
37273739
tableIds.add(table.getId());
3740+
cacheEpochs.add(table.tableVersionCacheEpoch.get());
37283741
}
37293742

37303743
List<Long> versions = getVisibleVersionFromMeta(dbIds, tableIds);
@@ -3733,6 +3746,7 @@ private static List<Long> getVisibleVersionInBatchFromMs(List<OlapTable> tables)
37333746
Preconditions.checkState(tables.size() == versions.size());
37343747
for (int i = 0; i < tables.size(); i++) {
37353748
tables.get(i).setCachedTableVersion(versions.get(i));
3749+
tables.get(i).refreshedTableVersionCacheEpoch.accumulateAndGet(cacheEpochs.get(i), Math::max);
37363750
}
37373751
return versions;
37383752
}

‎fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudFEVersionSynchronizer.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@
4949

5050
public class CloudFEVersionSynchronizer {
5151
private static final Logger LOG = LogManager.getLogger(CloudFEVersionSynchronizer.class);
52+
// Reuse the version RPC for whole-table invalidation when a commit response has no versions.
53+
// Older FEs ignore this value and continue relying on their periodic version synchronization.
54+
private static final long INVALIDATE_VERSION_CACHE = -1;
5255

5356
private static final ExecutorService SYNC_VERSION_THREAD_POOL = Executors.newFixedThreadPool(
5457
Config.cloud_sync_version_task_threads_num,
@@ -57,6 +60,54 @@ public class CloudFEVersionSynchronizer {
5760
public CloudFEVersionSynchronizer() {
5861
}
5962

63+
public void invalidateVersionCaches(long dbId, List<Long> tableIds) {
64+
Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
65+
if (db == null) {
66+
return; // The database may have been dropped since the original commit.
67+
}
68+
List<OlapTable> tables = new ArrayList<>();
69+
for (long tableId : tableIds) {
70+
Table table = db.getTableNullable(tableId);
71+
if (table != null && table.isManagedTable()) {
72+
tables.add((OlapTable) table);
73+
}
74+
}
75+
invalidateVersionCaches(tables);
76+
List<Pair<OlapTable, Long>> tableVersions = tables.stream()
77+
.map(table -> Pair.of(table, INVALIDATE_VERSION_CACHE)).collect(Collectors.toList());
78+
pushVersionAsync(dbId, tableVersions, Collections.emptyMap());
79+
}
80+
81+
private void invalidateVersionCaches(List<OlapTable> tables) {
82+
tables.sort(Comparator.comparingLong(OlapTable::getId));
83+
for (OlapTable table : tables) {
84+
table.readLock();
85+
}
86+
try {
87+
for (OlapTable table : tables) {
88+
table.versionWriteLock();
89+
}
90+
try {
91+
for (OlapTable table : tables) {
92+
table.invalidateCachedTableVersion();
93+
for (Partition partition : table.getAllPartitions()) {
94+
((CloudPartition) partition).invalidateCachedVisibleVersion();
95+
}
96+
}
97+
} finally {
98+
for (int i = tables.size() - 1; i >= 0; i--) {
99+
tables.get(i).versionWriteUnlock();
100+
}
101+
}
102+
} finally {
103+
for (int i = tables.size() - 1; i >= 0; i--) {
104+
tables.get(i).readUnlock();
105+
}
106+
}
107+
LOG.info("Invalidated cloud version caches for tables {}",
108+
tables.stream().map(OlapTable::getId).collect(Collectors.toList()));
109+
}
110+
60111
// master FE send sync version rpc to other FEs
61112
public void pushVersionAsync(long dbId, OlapTable table, long version) {
62113
if (!Config.cloud_enable_version_syncer) {
@@ -154,17 +205,25 @@ public void syncVersionAsync(TFrontendSyncCloudVersionRequest request) {
154205
}
155206
// only update table version
156207
if (request.getPartitionVersionInfos().isEmpty()) {
208+
List<OlapTable> invalidatedTables = new ArrayList<>();
157209
request.getTableVersionInfos().forEach(tableVersionInfo -> {
158210
Table table = db.getTableNullable(tableVersionInfo.getTableId());
159211
if (table == null || !table.isManagedTable()) {
160212
return;
161213
}
162214
OlapTable olapTable = (OlapTable) table;
215+
if (tableVersionInfo.getVersion() == INVALIDATE_VERSION_CACHE) {
216+
invalidatedTables.add(olapTable);
217+
return;
218+
}
163219
olapTable.setCachedTableVersion(tableVersionInfo.getVersion());
164220
if (LOG.isDebugEnabled()) {
165221
LOG.debug("Update tableId: {}, version: {}", olapTable.getId(), tableVersionInfo.getVersion());
166222
}
167223
});
224+
if (!invalidatedTables.isEmpty()) {
225+
invalidateVersionCaches(invalidatedTables);
226+
}
168227
return;
169228
}
170229
// update partition and table version

‎fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java‎

Lines changed: 36 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import java.util.HashMap;
4949
import java.util.List;
5050
import java.util.Map;
51+
import java.util.concurrent.atomic.AtomicLong;
5152
import java.util.concurrent.locks.ReentrantLock;
5253
import java.util.stream.Collectors;
5354

@@ -66,6 +67,10 @@ public class CloudPartition extends Partition {
6667
// This value is set when get the version from meta-service, 0 means version is not cached yet
6768
private volatile long lastVersionCachedTimeMs = 0;
6869

70+
// Only an MS read started after invalidation may make this cache valid again.
71+
private final AtomicLong versionCacheEpoch = new AtomicLong();
72+
private final AtomicLong refreshedVersionCacheEpoch = new AtomicLong();
73+
6974
private ReentrantLock lock = new ReentrantLock(true);
7075

7176
public CloudPartition(long id, String name, MaterializedIndex baseIndex,
@@ -127,9 +132,13 @@ public long getCachedVisibleVersion() {
127132
return super.getVisibleVersion();
128133
}
129134

135+
public void invalidateCachedVisibleVersion() {
136+
versionCacheEpoch.incrementAndGet();
137+
}
138+
130139
@VisibleForTesting
131140
protected boolean isCachedVersionExpired() {
132-
if (lastVersionCachedTimeMs == 0) {
141+
if (lastVersionCachedTimeMs == 0 || versionCacheEpoch.get() != refreshedVersionCacheEpoch.get()) {
133142
return true;
134143
}
135144
ConnectContext ctx = ConnectContext.get();
@@ -155,6 +164,7 @@ public long getVisibleVersion() {
155164
}
156165

157166
private long getVisibleVersionFromMs(boolean waitForPendingTxns) {
167+
long cacheEpoch = versionCacheEpoch.get();
158168
if (LOG.isDebugEnabled()) {
159169
LOG.debug("getVisibleVersionFromMs use CloudPartition {}, waitForPendingTxns: {}",
160170
super.getName(), waitForPendingTxns);
@@ -183,6 +193,7 @@ private long getVisibleVersionFromMs(boolean waitForPendingTxns) {
183193
version, tso, super.getId());
184194
}
185195
setCachedVisibleVersion(version, mTime, tso);
196+
refreshedVersionCacheEpoch.accumulateAndGet(cacheEpoch, Math::max);
186197
return version;
187198
} else {
188199
assert resp.getStatus().getCode() == MetaServiceCode.VERSION_NOT_FOUND;
@@ -193,6 +204,7 @@ private long getVisibleVersionFromMs(boolean waitForPendingTxns) {
193204
LOG.debug("get version from meta service, version: {}, partition: {}", version, super.getId());
194205
}
195206
setCachedVisibleVersion(version, mTime);
207+
refreshedVersionCacheEpoch.accumulateAndGet(cacheEpoch, Math::max);
196208
return version;
197209
} catch (RpcException e) {
198210
throw new RuntimeException("get version from meta service failed");
@@ -248,10 +260,12 @@ public static List<Long> getSnapshotVisibleVersionFromMs(
248260
List<Long> partitionIds = new ArrayList<>();
249261
List<Long> versionUpdateTimesMs = new ArrayList<>();
250262
List<Long> commitTsos = new ArrayList<>();
263+
List<Long> cacheEpochs = new ArrayList<>();
251264
for (CloudPartition partition : partitions) {
252265
dbIds.add(partition.getDbId());
253266
tableIds.add(partition.getTableId());
254267
partitionIds.add(partition.getId());
268+
cacheEpochs.add(partition.versionCacheEpoch.get());
255269
}
256270

257271
List<Long> versions = getSnapshotVisibleVersion(
@@ -260,15 +274,27 @@ public static List<Long> getSnapshotVisibleVersionFromMs(
260274
// Cache visible version, see hasData() for details.
261275
int size = versions.size();
262276
boolean hasCommitTsos = commitTsos.size() == size;
263-
for (int i = 0; i < size; ++i) {
264-
Long version = versions.get(i);
265-
if (version > Partition.PARTITION_INIT_VERSION) {
266-
// For compatibility, the existing partitions may not have mtime
267-
long mTime = versions.size() == versionUpdateTimesMs.size() ? versionUpdateTimesMs.get(i) : 0;
268-
long tso = hasCommitTsos ? commitTsos.get(i) : -1;
269-
partitions.get(i).setCachedVisibleVersion(versions.get(i), mTime, tso);
270-
} else { // No data has been written to this partition
271-
partitions.get(i).setCachedVisibleVersion(Partition.PARTITION_INIT_VERSION, System.currentTimeMillis());
277+
List<OlapTable> tables = getTables(partitions);
278+
for (OlapTable table : tables) {
279+
table.versionWriteLock();
280+
}
281+
try {
282+
for (int i = 0; i < size; ++i) {
283+
Long version = versions.get(i);
284+
if (version > Partition.PARTITION_INIT_VERSION) {
285+
// For compatibility, the existing partitions may not have mtime
286+
long mTime = versions.size() == versionUpdateTimesMs.size() ? versionUpdateTimesMs.get(i) : 0;
287+
long tso = hasCommitTsos ? commitTsos.get(i) : -1;
288+
partitions.get(i).setCachedVisibleVersion(versions.get(i), mTime, tso);
289+
} else { // No data has been written to this partition
290+
partitions.get(i).setCachedVisibleVersion(Partition.PARTITION_INIT_VERSION,
291+
System.currentTimeMillis());
292+
}
293+
partitions.get(i).refreshedVersionCacheEpoch.accumulateAndGet(cacheEpochs.get(i), Math::max);
294+
}
295+
} finally {
296+
for (int i = tables.size() - 1; i >= 0; i--) {
297+
tables.get(i).versionWriteUnlock();
272298
}
273299
}
274300

0 commit comments

Comments
 (0)