3232import org .apache .doris .common .io .Text ;
3333import org .apache .doris .common .io .Writable ;
3434import org .apache .doris .common .lock .MonitoredReentrantReadWriteLock ;
35+ import org .apache .doris .common .util .DebugPointUtil ;
3536import org .apache .doris .common .util .MasterDaemon ;
37+ import org .apache .doris .persist .EditLog .EditLogItem ;
3638import org .apache .doris .persist .TableStreamCleanupInfo ;
3739import org .apache .doris .persist .gson .GsonPostProcessable ;
3840import org .apache .doris .persist .gson .GsonUtils ;
5759import java .util .Optional ;
5860import java .util .Set ;
5961import java .util .concurrent .TimeUnit ;
62+ import java .util .concurrent .locks .LockSupport ;
6063
6164public class TableStreamManager extends MasterDaemon implements Writable , GsonPostProcessable {
6265 private static final Logger LOG = LogManager .getLogger (TableStreamManager .class );
@@ -166,7 +169,7 @@ public List<Cloud.TableStreamIdentityPB> getCloudTableStreamsForBaseTable(
166169 public void cleanupStalePartitionOffsets () {
167170 List <Long > staleDbIds = new ArrayList <>();
168171 List <Pair <Long , Long >> staleStreamIds = new ArrayList <>();
169- List <TableStreamCleanupInfo . PartitionOffsetPruneEntry > pruneEntries = new ArrayList <>();
172+ List <EditLogItem > editLogItems = new ArrayList <>();
170173 for (Map .Entry <Long , Set <Long >> entry : copyDbStreamMap ().entrySet ()) {
171174 Optional <Database > db = Env .getCurrentInternalCatalog ().getDb (entry .getKey ());
172175 if (!db .isPresent ()) {
@@ -183,18 +186,18 @@ public void cleanupStalePartitionOffsets() {
183186 staleStreamIds .add (Pair .of (db .get ().getId (), tableId ));
184187 continue ;
185188 }
186- cleanupStalePartitionOffsets ((OlapTableStream ) table .get ()).ifPresent (pruneEntries ::add );
189+ cleanupStalePartitionOffsets ((OlapTableStream ) table .get ()).ifPresent (editLogItems ::add );
187190 }
188191 }
189192 removeStaleDbAndStream (staleDbIds , staleStreamIds );
190- if (!pruneEntries . isEmpty () || ! staleDbIds .isEmpty () || !staleStreamIds .isEmpty ()) {
191- Env .getCurrentEnv ().getEditLog ().logTableStreamCleanup (
192- new TableStreamCleanupInfo (pruneEntries , staleDbIds , staleStreamIds ));
193+ if (!staleDbIds .isEmpty () || !staleStreamIds .isEmpty ()) {
194+ editLogItems . add ( Env .getCurrentEnv ().getEditLog ().logTableStreamCleanup (
195+ new TableStreamCleanupInfo (Collections . emptyList () , staleDbIds , staleStreamIds ) ));
193196 }
197+ editLogItems .forEach (EditLogItem ::await );
194198 }
195199
196- private Optional <TableStreamCleanupInfo .PartitionOffsetPruneEntry > cleanupStalePartitionOffsets (
197- OlapTableStream stream ) {
200+ private Optional <EditLogItem > cleanupStalePartitionOffsets (OlapTableStream stream ) {
198201 if (!stream .tryReadLock (Table .TRY_LOCK_TIMEOUT_MS , TimeUnit .MILLISECONDS )) {
199202 if (LOG .isDebugEnabled ()) {
200203 LOG .debug ("skip cleaning stream {} because stream read lock is busy" , stream .getName ());
@@ -215,51 +218,54 @@ private Optional<TableStreamCleanupInfo.PartitionOffsetPruneEntry> cleanupStaleP
215218 stream .readUnlock ();
216219 }
217220 // stream read lock is released
218- // base table read lock is held
219221 if (!baseTable .tryReadLock (Table .TRY_LOCK_TIMEOUT_MS , TimeUnit .MILLISECONDS )) {
220222 if (LOG .isDebugEnabled ()) {
221223 LOG .debug ("skip cleaning stream {} because base table {} read lock is busy" ,
222224 stream .getName (), baseTable .getName ());
223225 }
224226 return Optional .empty ();
225227 }
226- Set <Long > validPartitionIds ;
228+ Set <Long > stalePartitionIds ;
229+ EditLogItem editLogItem ;
227230 try {
228231 if (baseTable .isDropped ) {
229232 return Optional .empty ();
230233 }
231- validPartitionIds = new HashSet <>(baseTable .getPartitionIds ());
232- } finally {
233- baseTable .readUnlock ();
234- }
235- // base table read lock is released
236- // stream write lock is held
237- if (!stream .tryWriteLock (Table .TRY_LOCK_TIMEOUT_MS , TimeUnit .MILLISECONDS )) {
238- if (LOG .isDebugEnabled ()) {
239- LOG .debug ("skip cleaning stream {} because stream write lock is busy" , stream .getName ());
234+ Set <Long > validPartitionIds = new HashSet <>(baseTable .getPartitionIds ());
235+ while (DebugPointUtil .getDebugParamOrDefault (
236+ "TableStreamManager.cleanupStalePartitionOffsets.blockAfterPartitionSnapshot" , -1L )
237+ == stream .getId ()) {
238+ LockSupport .parkNanos (TimeUnit .MILLISECONDS .toNanos (10 ));
240239 }
241- return Optional .empty ();
242- }
243- Set <Long > stalePartitionIds ;
244- try {
245- if (stream .isDisabled () || stream .isStale ()) {
240+ if (!stream .tryWriteLockIfExist (Table .TRY_LOCK_TIMEOUT_MS , TimeUnit .MILLISECONDS )) {
241+ if (LOG .isDebugEnabled ()) {
242+ LOG .debug ("skip cleaning stream {} because it is busy or dropped" , stream .getName ());
243+ }
246244 return Optional .empty ();
247245 }
248- stalePartitionIds = stream .unprotectedCollectStalePartitionOffsetIds (validPartitionIds );
249- if (stalePartitionIds .isEmpty ()) {
250- return Optional .empty ();
246+ try {
247+ if (stream .isDisabled () || stream .isStale ()) {
248+ return Optional .empty ();
249+ }
250+ stalePartitionIds = stream .unprotectedCollectStalePartitionOffsetIds (validPartitionIds );
251+ if (stalePartitionIds .isEmpty ()) {
252+ return Optional .empty ();
253+ }
254+ stream .unprotectedPrunePartitionOffsets (stalePartitionIds );
255+ TableStreamCleanupInfo .PartitionOffsetPruneEntry pruneEntry =
256+ new TableStreamCleanupInfo .PartitionOffsetPruneEntry (
257+ stream .getDatabase ().getId (), stream .getId (), stalePartitionIds );
258+ editLogItem = Env .getCurrentEnv ().getEditLog ().logTableStreamCleanup (
259+ new TableStreamCleanupInfo (Collections .singletonList (pruneEntry )));
260+ } finally {
261+ stream .writeUnlock ();
251262 }
252- stream .unprotectedPrunePartitionOffsets (stalePartitionIds );
253263 } finally {
254- stream .writeUnlock ();
255- }
256- // stream write lock is released
257- if (stalePartitionIds .size () > 0 ) {
258- LOG .info ("cleaned {} stale partition offset entries from stream {}.{} ({})" ,
259- stalePartitionIds .size (), stream .getDatabase ().getFullName (), stream .getName (), stream .getId ());
264+ baseTable .readUnlock ();
260265 }
261- return Optional .of (new TableStreamCleanupInfo .PartitionOffsetPruneEntry (
262- stream .getDatabase ().getId (), stream .getId (), stalePartitionIds ));
266+ LOG .info ("cleaned {} stale partition offset entries from stream {}.{} ({})" ,
267+ stalePartitionIds .size (), stream .getDatabase ().getFullName (), stream .getName (), stream .getId ());
268+ return Optional .of (editLogItem );
263269 }
264270
265271 public void replayTableStreamCleanup (TableStreamCleanupInfo info ) {
0 commit comments