Skip to content

Commit 3dbe4d3

Browse files
authored
[fix](cloud) Invalidate version caches on visible commit retries (#67813)
### What problem does this PR solve? Issue Number: close #67099 Related PR: None Problem Summary: After a cloud commit response is lost, a retry can return a VISIBLE transaction without version results. FE can then keep using stale partition versions and the table version used to validate SQL caches. Invalidate the affected table and partition version caches, including the MoW already-visible shortcut, and notify other FEs through the existing version synchronization RPC. Subsequent reads fetch fresh versions from MS. Commit processing performs no extra MS version query, so a version-service outage does not turn an already committed transaction into a commit error or require replaying its callbacks. Publish batch cache updates under the tables' version write locks. Invalidation epochs prevent old in-flight MS responses and delayed commit notifications from restoring stale caches. The synchronization RPC uses table version -1 for invalidation; older FEs ignore it and retain their existing periodic synchronization. Peer notification keeps its existing asynchronous behavior and configuration. ### Release note Prevent stale reads and false commit failures after cloud commit retries. ### Check List (For Author) - Test: Unit Test - 58 FE tests passed in both ordinary and JaCoCo coverage modes via run-fe-ut.sh (CloudGlobalTransactionMgrTest, CloudPartitionTest, OlapTableTest, VersionHelperTest); FE Checkstyle: 0 violations; PR diff whitespace check passed. - Tests cover invalidation and batch-publication locks, table/partition cache recovery, old RPC completions, successful commit callbacks during a version outage, and FE notification serialization/receiver handling. - No live cluster deployment or SQL regression run. - Behavior changed: Yes. VISIBLE retries without versions invalidate local and peer caches; subsequent reads retrieve versions without changing the successful commit result. - Does this need documentation: No
1 parent 7b9693d commit 3dbe4d3

5 files changed

Lines changed: 520 additions & 14 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,10 @@ public enum OlapTableState {
248248
private volatile long lastTableVersionCachedTimeMs = 0;
249249
private volatile long cachedTableVersion = -1;
250250

251+
// Commit notifications cannot clear invalidation: their versions may precede the missing commit result.
252+
private final AtomicLong tableVersionCacheEpoch = new AtomicLong();
253+
private final AtomicLong refreshedTableVersionCacheEpoch = new AtomicLong();
254+
251255
private ReadWriteLock versionLock = Config.isCloudMode() ? new ReentrantReadWriteLock(true) : null;
252256

253257
public OlapTable() {
@@ -3591,7 +3595,7 @@ public long getNextVersion() {
35913595
@VisibleForTesting
35923596
protected boolean isCachedTableVersionExpired() {
35933597
// -1 means no cache yet, need to fetch from MS
3594-
if (cachedTableVersion == -1) {
3598+
if (cachedTableVersion == -1 || tableVersionCacheEpoch.get() != refreshedTableVersionCacheEpoch.get()) {
35953599
return true;
35963600
}
35973601
ConnectContext ctx = ConnectContext.get();
@@ -3605,13 +3609,18 @@ protected boolean isCachedTableVersionExpired() {
36053609

36063610
public boolean isCachedTableVersionExpired(long expirationMs) {
36073611
// -1 means no cache yet, need to fetch from MS
3608-
if (cachedTableVersion == -1 || expirationMs <= 0) {
3612+
if (cachedTableVersion == -1 || expirationMs <= 0
3613+
|| tableVersionCacheEpoch.get() != refreshedTableVersionCacheEpoch.get()) {
36093614
return true;
36103615
}
36113616
return System.currentTimeMillis() - lastTableVersionCachedTimeMs > expirationMs;
36123617
}
36133618

3614-
public void setCachedTableVersion(long version) {
3619+
public void invalidateCachedTableVersion() {
3620+
tableVersionCacheEpoch.incrementAndGet();
3621+
}
3622+
3623+
public synchronized void setCachedTableVersion(long version) {
36153624
if (version >= cachedTableVersion) {
36163625
cachedTableVersion = version;
36173626
lastTableVersionCachedTimeMs = System.currentTimeMillis();
@@ -3632,6 +3641,7 @@ public long getVisibleVersion() throws RpcException {
36323641
return getCachedTableVersion();
36333642
}
36343643

3644+
long cacheEpoch = tableVersionCacheEpoch.get();
36353645
// get version rpc
36363646
Cloud.GetVersionRequest request = Cloud.GetVersionRequest.newBuilder()
36373647
.setRequestIp(FrontendOptions.getLocalHostAddressCached())
@@ -3658,6 +3668,7 @@ public long getVisibleVersion() throws RpcException {
36583668
}
36593669
// update cache
36603670
setCachedTableVersion(version);
3671+
refreshedTableVersionCacheEpoch.accumulateAndGet(cacheEpoch, Math::max);
36613672
return version;
36623673
} catch (RpcException e) {
36633674
LOG.warn("get version from meta service failed", e);
@@ -3722,9 +3733,11 @@ public static List<Long> getVisibleVersionInBatch(List<OlapTable> tables) {
37223733
private static List<Long> getVisibleVersionInBatchFromMs(List<OlapTable> tables) {
37233734
List<Long> dbIds = new ArrayList<>(tables.size());
37243735
List<Long> tableIds = new ArrayList<>(tables.size());
3736+
List<Long> cacheEpochs = new ArrayList<>(tables.size());
37253737
for (OlapTable table : tables) {
37263738
dbIds.add(table.getDatabase().getId());
37273739
tableIds.add(table.getId());
3740+
cacheEpochs.add(table.tableVersionCacheEpoch.get());
37283741
}
37293742

37303743
List<Long> versions = getVisibleVersionFromMeta(dbIds, tableIds);
@@ -3733,6 +3746,7 @@ private static List<Long> getVisibleVersionInBatchFromMs(List<OlapTable> tables)
37333746
Preconditions.checkState(tables.size() == versions.size());
37343747
for (int i = 0; i < tables.size(); i++) {
37353748
tables.get(i).setCachedTableVersion(versions.get(i));
3749+
tables.get(i).refreshedTableVersionCacheEpoch.accumulateAndGet(cacheEpochs.get(i), Math::max);
37363750
}
37373751
return versions;
37383752
}

‎fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudFEVersionSynchronizer.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@
4949

5050
public class CloudFEVersionSynchronizer {
5151
private static final Logger LOG = LogManager.getLogger(CloudFEVersionSynchronizer.class);
52+
// Reuse the version RPC for whole-table invalidation when a commit response has no versions.
53+
// Older FEs ignore this value and continue relying on their periodic version synchronization.
54+
private static final long INVALIDATE_VERSION_CACHE = -1;
5255

5356
private static final ExecutorService SYNC_VERSION_THREAD_POOL = Executors.newFixedThreadPool(
5457
Config.cloud_sync_version_task_threads_num,
@@ -57,6 +60,54 @@ public class CloudFEVersionSynchronizer {
5760
public CloudFEVersionSynchronizer() {
5861
}
5962

63+
public void invalidateVersionCaches(long dbId, List<Long> tableIds) {
64+
Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId);
65+
if (db == null) {
66+
return; // The database may have been dropped since the original commit.
67+
}
68+
List<OlapTable> tables = new ArrayList<>();
69+
for (long tableId : tableIds) {
70+
Table table = db.getTableNullable(tableId);
71+
if (table != null && table.isManagedTable()) {
72+
tables.add((OlapTable) table);
73+
}
74+
}
75+
invalidateVersionCaches(tables);
76+
List<Pair<OlapTable, Long>> tableVersions = tables.stream()
77+
.map(table -> Pair.of(table, INVALIDATE_VERSION_CACHE)).collect(Collectors.toList());
78+
pushVersionAsync(dbId, tableVersions, Collections.emptyMap());
79+
}
80+
81+
private void invalidateVersionCaches(List<OlapTable> tables) {
82+
tables.sort(Comparator.comparingLong(OlapTable::getId));
83+
for (OlapTable table : tables) {
84+
table.readLock();
85+
}
86+
try {
87+
for (OlapTable table : tables) {
88+
table.versionWriteLock();
89+
}
90+
try {
91+
for (OlapTable table : tables) {
92+
table.invalidateCachedTableVersion();
93+
for (Partition partition : table.getAllPartitions()) {
94+
((CloudPartition) partition).invalidateCachedVisibleVersion();
95+
}
96+
}
97+
} finally {
98+
for (int i = tables.size() - 1; i >= 0; i--) {
99+
tables.get(i).versionWriteUnlock();
100+
}
101+
}
102+
} finally {
103+
for (int i = tables.size() - 1; i >= 0; i--) {
104+
tables.get(i).readUnlock();
105+
}
106+
}
107+
LOG.info("Invalidated cloud version caches for tables {}",
108+
tables.stream().map(OlapTable::getId).collect(Collectors.toList()));
109+
}
110+
60111
// master FE send sync version rpc to other FEs
61112
public void pushVersionAsync(long dbId, OlapTable table, long version) {
62113
if (!Config.cloud_enable_version_syncer) {
@@ -154,17 +205,25 @@ public void syncVersionAsync(TFrontendSyncCloudVersionRequest request) {
154205
}
155206
// only update table version
156207
if (request.getPartitionVersionInfos().isEmpty()) {
208+
List<OlapTable> invalidatedTables = new ArrayList<>();
157209
request.getTableVersionInfos().forEach(tableVersionInfo -> {
158210
Table table = db.getTableNullable(tableVersionInfo.getTableId());
159211
if (table == null || !table.isManagedTable()) {
160212
return;
161213
}
162214
OlapTable olapTable = (OlapTable) table;
215+
if (tableVersionInfo.getVersion() == INVALIDATE_VERSION_CACHE) {
216+
invalidatedTables.add(olapTable);
217+
return;
218+
}
163219
olapTable.setCachedTableVersion(tableVersionInfo.getVersion());
164220
if (LOG.isDebugEnabled()) {
165221
LOG.debug("Update tableId: {}, version: {}", olapTable.getId(), tableVersionInfo.getVersion());
166222
}
167223
});
224+
if (!invalidatedTables.isEmpty()) {
225+
invalidateVersionCaches(invalidatedTables);
226+
}
168227
return;
169228
}
170229
// update partition and table version

‎fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudPartition.java‎

Lines changed: 36 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import java.util.HashMap;
4949
import java.util.List;
5050
import java.util.Map;
51+
import java.util.concurrent.atomic.AtomicLong;
5152
import java.util.concurrent.locks.ReentrantLock;
5253
import java.util.stream.Collectors;
5354

@@ -66,6 +67,10 @@ public class CloudPartition extends Partition {
6667
// This value is set when get the version from meta-service, 0 means version is not cached yet
6768
private volatile long lastVersionCachedTimeMs = 0;
6869

70+
// Only an MS read started after invalidation may make this cache valid again.
71+
private final AtomicLong versionCacheEpoch = new AtomicLong();
72+
private final AtomicLong refreshedVersionCacheEpoch = new AtomicLong();
73+
6974
private ReentrantLock lock = new ReentrantLock(true);
7075

7176
public CloudPartition(long id, String name, MaterializedIndex baseIndex,
@@ -127,9 +132,13 @@ public long getCachedVisibleVersion() {
127132
return super.getVisibleVersion();
128133
}
129134

135+
public void invalidateCachedVisibleVersion() {
136+
versionCacheEpoch.incrementAndGet();
137+
}
138+
130139
@VisibleForTesting
131140
protected boolean isCachedVersionExpired() {
132-
if (lastVersionCachedTimeMs == 0) {
141+
if (lastVersionCachedTimeMs == 0 || versionCacheEpoch.get() != refreshedVersionCacheEpoch.get()) {
133142
return true;
134143
}
135144
ConnectContext ctx = ConnectContext.get();
@@ -155,6 +164,7 @@ public long getVisibleVersion() {
155164
}
156165

157166
private long getVisibleVersionFromMs(boolean waitForPendingTxns) {
167+
long cacheEpoch = versionCacheEpoch.get();
158168
if (LOG.isDebugEnabled()) {
159169
LOG.debug("getVisibleVersionFromMs use CloudPartition {}, waitForPendingTxns: {}",
160170
super.getName(), waitForPendingTxns);
@@ -183,6 +193,7 @@ private long getVisibleVersionFromMs(boolean waitForPendingTxns) {
183193
version, tso, super.getId());
184194
}
185195
setCachedVisibleVersion(version, mTime, tso);
196+
refreshedVersionCacheEpoch.accumulateAndGet(cacheEpoch, Math::max);
186197
return version;
187198
} else {
188199
assert resp.getStatus().getCode() == MetaServiceCode.VERSION_NOT_FOUND;
@@ -193,6 +204,7 @@ private long getVisibleVersionFromMs(boolean waitForPendingTxns) {
193204
LOG.debug("get version from meta service, version: {}, partition: {}", version, super.getId());
194205
}
195206
setCachedVisibleVersion(version, mTime);
207+
refreshedVersionCacheEpoch.accumulateAndGet(cacheEpoch, Math::max);
196208
return version;
197209
} catch (RpcException e) {
198210
throw new RuntimeException("get version from meta service failed");
@@ -254,10 +266,12 @@ public static List<Long> getSnapshotVisibleVersionFromMs(
254266
List<Long> partitionIds = new ArrayList<>();
255267
List<Long> versionUpdateTimesMs = new ArrayList<>();
256268
List<Long> commitTsos = new ArrayList<>();
269+
List<Long> cacheEpochs = new ArrayList<>();
257270
for (CloudPartition partition : partitions) {
258271
dbIds.add(partition.getDbId());
259272
tableIds.add(partition.getTableId());
260273
partitionIds.add(partition.getId());
274+
cacheEpochs.add(partition.versionCacheEpoch.get());
261275
}
262276

263277
List<Long> versions = getSnapshotVisibleVersion(
@@ -266,15 +280,27 @@ public static List<Long> getSnapshotVisibleVersionFromMs(
266280
// Cache visible version, see hasData() for details.
267281
int size = versions.size();
268282
boolean hasCommitTsos = commitTsos.size() == size;
269-
for (int i = 0; i < size; ++i) {
270-
Long version = versions.get(i);
271-
if (version > Partition.PARTITION_INIT_VERSION) {
272-
// For compatibility, the existing partitions may not have mtime
273-
long mTime = versions.size() == versionUpdateTimesMs.size() ? versionUpdateTimesMs.get(i) : 0;
274-
long tso = hasCommitTsos ? commitTsos.get(i) : -1;
275-
partitions.get(i).setCachedVisibleVersion(versions.get(i), mTime, tso);
276-
} else { // No data has been written to this partition
277-
partitions.get(i).setCachedVisibleVersion(Partition.PARTITION_INIT_VERSION, System.currentTimeMillis());
283+
List<OlapTable> tables = getTables(partitions);
284+
for (OlapTable table : tables) {
285+
table.versionWriteLock();
286+
}
287+
try {
288+
for (int i = 0; i < size; ++i) {
289+
Long version = versions.get(i);
290+
if (version > Partition.PARTITION_INIT_VERSION) {
291+
// For compatibility, the existing partitions may not have mtime
292+
long mTime = versions.size() == versionUpdateTimesMs.size() ? versionUpdateTimesMs.get(i) : 0;
293+
long tso = hasCommitTsos ? commitTsos.get(i) : -1;
294+
partitions.get(i).setCachedVisibleVersion(versions.get(i), mTime, tso);
295+
} else { // No data has been written to this partition
296+
partitions.get(i).setCachedVisibleVersion(Partition.PARTITION_INIT_VERSION,
297+
System.currentTimeMillis());
298+
}
299+
partitions.get(i).refreshedVersionCacheEpoch.accumulateAndGet(cacheEpochs.get(i), Math::max);
300+
}
301+
} finally {
302+
for (int i = tables.size() - 1; i >= 0; i--) {
303+
tables.get(i).versionWriteUnlock();
278304
}
279305
}
280306

‎fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,10 @@ private boolean checkTransactionStateBeforeCommit(long dbId, long transactionId)
491491
+ "] is already aborted. abort reason: " + transactionState.getReason());
492492
} else if (transactionState.getTransactionStatus() == TransactionStatus.COMMITTED
493493
|| transactionState.getTransactionStatus() == TransactionStatus.VISIBLE) {
494+
if (transactionState.getTransactionStatus() == TransactionStatus.VISIBLE) {
495+
((CloudEnv) Env.getCurrentEnv()).getCloudFEVersionSynchronizer()
496+
.invalidateVersionCaches(dbId, transactionState.getTableIdList());
497+
}
494498
LOG.info("txn={}, status={} not need to calculate delete bitmap again, just return ",
495499
transactionId,
496500
transactionState.getTransactionStatus().toString());
@@ -597,6 +601,13 @@ private Map<Long, List<Long>> updateVersion(CommitTxnResponse commitTxnResponse)
597601
? commitTxnResponse.getTxnInfo().getCommitTso() : -1;
598602
int totalPartitionNum = commitTxnResponse.getPartitionIdsList().size();
599603
if (totalPartitionNum == 0 && commitTxnResponse.getTableStatsList().isEmpty()) {
604+
// A visible retry can contain only txn_info. Invalidate locally without another commit-time RPC;
605+
// subsequent reads fetch fresh versions, including the table version used to validate SQL caches.
606+
if (commitTxnResponse.getTxnInfo().getStatus() == TxnStatusPB.TXN_STATUS_VISIBLE
607+
&& !(commitTxnResponse.getIsLazyCommit() && commitTxnResponse.getIsLazyCommitIncomplete())) {
608+
((CloudEnv) Env.getCurrentEnv()).getCloudFEVersionSynchronizer()
609+
.invalidateVersionCaches(dbId, commitTxnResponse.getTxnInfo().getTableIdsList());
610+
}
600611
return Collections.emptyMap();
601612
}
602613
Env env = Env.getCurrentEnv();

0 commit comments

Comments
 (0)