Skip to content

Commit 300d532

Browse files
[fix](table stream) Preserve table stream offsets during cleanup (apache#67533)
### What problem does this PR solve? Issue Number: close apache#67094
1 parent 6fe0b19 commit 300d532

3 files changed

Lines changed: 272 additions & 37 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/stream/TableStreamManager.java‎

Lines changed: 41 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,9 @@
3232
import org.apache.doris.common.io.Text;
3333
import org.apache.doris.common.io.Writable;
3434
import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock;
35+
import org.apache.doris.common.util.DebugPointUtil;
3536
import org.apache.doris.common.util.MasterDaemon;
37+
import org.apache.doris.persist.EditLog.EditLogItem;
3638
import org.apache.doris.persist.TableStreamCleanupInfo;
3739
import org.apache.doris.persist.gson.GsonPostProcessable;
3840
import org.apache.doris.persist.gson.GsonUtils;
@@ -57,6 +59,7 @@
5759
import java.util.Optional;
5860
import java.util.Set;
5961
import java.util.concurrent.TimeUnit;
62+
import java.util.concurrent.locks.LockSupport;
6063

6164
public class TableStreamManager extends MasterDaemon implements Writable, GsonPostProcessable {
6265
private static final Logger LOG = LogManager.getLogger(TableStreamManager.class);
@@ -166,7 +169,7 @@ public List<Cloud.TableStreamIdentityPB> getCloudTableStreamsForBaseTable(
166169
public void cleanupStalePartitionOffsets() {
167170
List<Long> staleDbIds = new ArrayList<>();
168171
List<Pair<Long, Long>> staleStreamIds = new ArrayList<>();
169-
List<TableStreamCleanupInfo.PartitionOffsetPruneEntry> pruneEntries = new ArrayList<>();
172+
List<EditLogItem> editLogItems = new ArrayList<>();
170173
for (Map.Entry<Long, Set<Long>> entry : copyDbStreamMap().entrySet()) {
171174
Optional<Database> db = Env.getCurrentInternalCatalog().getDb(entry.getKey());
172175
if (!db.isPresent()) {
@@ -183,18 +186,18 @@ public void cleanupStalePartitionOffsets() {
183186
staleStreamIds.add(Pair.of(db.get().getId(), tableId));
184187
continue;
185188
}
186-
cleanupStalePartitionOffsets((OlapTableStream) table.get()).ifPresent(pruneEntries::add);
189+
cleanupStalePartitionOffsets((OlapTableStream) table.get()).ifPresent(editLogItems::add);
187190
}
188191
}
189192
removeStaleDbAndStream(staleDbIds, staleStreamIds);
190-
if (!pruneEntries.isEmpty() || !staleDbIds.isEmpty() || !staleStreamIds.isEmpty()) {
191-
Env.getCurrentEnv().getEditLog().logTableStreamCleanup(
192-
new TableStreamCleanupInfo(pruneEntries, staleDbIds, staleStreamIds));
193+
if (!staleDbIds.isEmpty() || !staleStreamIds.isEmpty()) {
194+
editLogItems.add(Env.getCurrentEnv().getEditLog().logTableStreamCleanup(
195+
new TableStreamCleanupInfo(Collections.emptyList(), staleDbIds, staleStreamIds)));
193196
}
197+
editLogItems.forEach(EditLogItem::await);
194198
}
195199

196-
private Optional<TableStreamCleanupInfo.PartitionOffsetPruneEntry> cleanupStalePartitionOffsets(
197-
OlapTableStream stream) {
200+
private Optional<EditLogItem> cleanupStalePartitionOffsets(OlapTableStream stream) {
198201
if (!stream.tryReadLock(Table.TRY_LOCK_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
199202
if (LOG.isDebugEnabled()) {
200203
LOG.debug("skip cleaning stream {} because stream read lock is busy", stream.getName());
@@ -215,51 +218,54 @@ private Optional<TableStreamCleanupInfo.PartitionOffsetPruneEntry> cleanupStaleP
215218
stream.readUnlock();
216219
}
217220
// stream read lock is released
218-
// base table read lock is held
219221
if (!baseTable.tryReadLock(Table.TRY_LOCK_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
220222
if (LOG.isDebugEnabled()) {
221223
LOG.debug("skip cleaning stream {} because base table {} read lock is busy",
222224
stream.getName(), baseTable.getName());
223225
}
224226
return Optional.empty();
225227
}
226-
Set<Long> validPartitionIds;
228+
Set<Long> stalePartitionIds;
229+
EditLogItem editLogItem;
227230
try {
228231
if (baseTable.isDropped) {
229232
return Optional.empty();
230233
}
231-
validPartitionIds = new HashSet<>(baseTable.getPartitionIds());
232-
} finally {
233-
baseTable.readUnlock();
234-
}
235-
// base table read lock is released
236-
// stream write lock is held
237-
if (!stream.tryWriteLock(Table.TRY_LOCK_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
238-
if (LOG.isDebugEnabled()) {
239-
LOG.debug("skip cleaning stream {} because stream write lock is busy", stream.getName());
234+
Set<Long> validPartitionIds = new HashSet<>(baseTable.getPartitionIds());
235+
while (DebugPointUtil.getDebugParamOrDefault(
236+
"TableStreamManager.cleanupStalePartitionOffsets.blockAfterPartitionSnapshot", -1L)
237+
== stream.getId()) {
238+
LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(10));
240239
}
241-
return Optional.empty();
242-
}
243-
Set<Long> stalePartitionIds;
244-
try {
245-
if (stream.isDisabled() || stream.isStale()) {
240+
if (!stream.tryWriteLockIfExist(Table.TRY_LOCK_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
241+
if (LOG.isDebugEnabled()) {
242+
LOG.debug("skip cleaning stream {} because it is busy or dropped", stream.getName());
243+
}
246244
return Optional.empty();
247245
}
248-
stalePartitionIds = stream.unprotectedCollectStalePartitionOffsetIds(validPartitionIds);
249-
if (stalePartitionIds.isEmpty()) {
250-
return Optional.empty();
246+
try {
247+
if (stream.isDisabled() || stream.isStale()) {
248+
return Optional.empty();
249+
}
250+
stalePartitionIds = stream.unprotectedCollectStalePartitionOffsetIds(validPartitionIds);
251+
if (stalePartitionIds.isEmpty()) {
252+
return Optional.empty();
253+
}
254+
stream.unprotectedPrunePartitionOffsets(stalePartitionIds);
255+
TableStreamCleanupInfo.PartitionOffsetPruneEntry pruneEntry =
256+
new TableStreamCleanupInfo.PartitionOffsetPruneEntry(
257+
stream.getDatabase().getId(), stream.getId(), stalePartitionIds);
258+
editLogItem = Env.getCurrentEnv().getEditLog().logTableStreamCleanup(
259+
new TableStreamCleanupInfo(Collections.singletonList(pruneEntry)));
260+
} finally {
261+
stream.writeUnlock();
251262
}
252-
stream.unprotectedPrunePartitionOffsets(stalePartitionIds);
253263
} finally {
254-
stream.writeUnlock();
255-
}
256-
// stream write lock is released
257-
if (stalePartitionIds.size() > 0) {
258-
LOG.info("cleaned {} stale partition offset entries from stream {}.{} ({})",
259-
stalePartitionIds.size(), stream.getDatabase().getFullName(), stream.getName(), stream.getId());
264+
baseTable.readUnlock();
260265
}
261-
return Optional.of(new TableStreamCleanupInfo.PartitionOffsetPruneEntry(
262-
stream.getDatabase().getId(), stream.getId(), stalePartitionIds));
266+
LOG.info("cleaned {} stale partition offset entries from stream {}.{} ({})",
267+
stalePartitionIds.size(), stream.getDatabase().getFullName(), stream.getName(), stream.getId());
268+
return Optional.of(editLogItem);
263269
}
264270

265271
public void replayTableStreamCleanup(TableStreamCleanupInfo info) {

‎fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2308,8 +2308,8 @@ public void logDynamicPartition(ModifyTablePropertyOperationLog info) {
23082308
logModifyTableProperty(OperationType.OP_DYNAMIC_PARTITION, info);
23092309
}
23102310

2311-
public void logTableStreamCleanup(TableStreamCleanupInfo info) {
2312-
logEdit(OperationType.OP_TABLE_STREAM_CLEANUP, info);
2311+
public EditLogItem logTableStreamCleanup(TableStreamCleanupInfo info) {
2312+
return submitEdit(OperationType.OP_TABLE_STREAM_CLEANUP, info);
23132313
}
23142314

23152315
public long logModifyReplicationNum(ModifyTablePropertyOperationLog info) {

0 commit comments

Comments
 (0)