Skip to content

Commit 6cb029c

Browse files
morningmanclaude
andcommitted
[fix](ivm) Keep the keep-alive TSOs of a stream in a concurrent map
### What problem does this PR solve? Issue Number: None Related PR: #69831 Problem Summary: `OlapTableStream.partitionKeepAlive` was a `HashMap` that the keep-alive daemon writes under the stream write lock. The delta scan of an IVM refresh reads it without the stream lock: `IvmDeltaRewriteState` looks the stream up and creates its `OlapTableStreamWrapper` after the statement has locked its tables, so the stream is never among them, and the base table read lock the daemon also holds in shared-nothing mode does not exclude the refresh, which holds it too. A put that adds a partition and resizes the map can make a concurrent get miss the keep-alive TSO of another partition. The wrapper keeps the miss for the whole refresh, and when that partition's offset is older than the TTL the read is refused: an AUTO refresh falls back to COMPLETE and a strict INCREMENTAL refresh fails, although a fresh keep-alive TSO was recorded. No row is read wrong. The field is now declared as a `ConcurrentMap`, so a get never misses an entry while another one is added. Declared as such, GSON reads it back from an image as a concurrent map too, as it does `Database.name2Function`. ### Release note None ### Check List (For Author) - Test: Unit Test - `TableStreamManagerKeepAliveTest`: a keep-alive TSO written to an image reads back, into a concurrent map; the cases that set the field directly set a concurrent map. The race itself has no deterministic test. - Behavior changed: No - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 754db89 commit 6cb029c

2 files changed

Lines changed: 33 additions & 8 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/stream/OlapTableStream.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232

3333
import com.google.common.base.Preconditions;
3434
import com.google.common.collect.ImmutableList;
35+
import com.google.common.collect.Maps;
3536
import com.google.gson.annotations.SerializedName;
3637

3738
import java.io.DataInput;
@@ -43,6 +44,7 @@
4344
import java.util.List;
4445
import java.util.Map;
4546
import java.util.Set;
47+
import java.util.concurrent.ConcurrentMap;
4648
import java.util.function.Predicate;
4749

4850
public class OlapTableStream extends BaseTableStream {
@@ -59,14 +61,16 @@ public class OlapTableStream extends BaseTableStream {
5961
// The keep-alive TSO K of a partition, with the offset C it was taken against: the partition is known to hold
6062
// no change in (C, K], so the changelog a read from C needs is only the part after K. It counts only for a
6163
// read from C, so an offset that moved, or one that is not what it was taken against (a cloud offset lives in
62-
// MetaService), leaves it out. Only the partitions of an IVM stream have one (IvmStreamKeepAliveDaemon).
64+
// MetaService), leaves it out. Only the partitions of an IVM stream have one (IvmStreamKeepAliveDaemon). It is
65+
// written under the stream write lock but read without it: an IVM refresh reads the stream through a scan it
66+
// creates after the statement has locked its tables (OlapTableStreamWrapper), so it is a concurrent map.
6367
@SerializedName("pka")
64-
private Map<Long, PartitionKeepAlive> partitionKeepAlive;
68+
private ConcurrentMap<Long, PartitionKeepAlive> partitionKeepAlive;
6569

6670
// for persist; an image written before the keep-alive TSO has no "pka"
6771
public OlapTableStream() {
6872
super();
69-
this.partitionKeepAlive = new HashMap<>();
73+
this.partitionKeepAlive = Maps.newConcurrentMap();
7074
}
7175

7276
public OlapTableStream(long id, String streamName, TableIf baseTable) {
@@ -75,7 +79,7 @@ public OlapTableStream(long id, String streamName, TableIf baseTable) {
7579
this.partitionOffset = new HashMap<>();
7680
this.partitionConsumptionTime = new HashMap<>();
7781
this.historicalPartitionTSO = new HashMap<>();
78-
this.partitionKeepAlive = new HashMap<>();
82+
this.partitionKeepAlive = Maps.newConcurrentMap();
7983
}
8084

8185
public OlapTableStream(String streamName, TableIf baseTable) {

‎fe/fe-core/src/test/java/org/apache/doris/catalog/stream/TableStreamManagerKeepAliveTest.java‎

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@
5353
import java.util.Map;
5454
import java.util.Optional;
5555
import java.util.Set;
56+
import java.util.concurrent.ConcurrentHashMap;
57+
import java.util.concurrent.ConcurrentMap;
5658

5759
public class TableStreamManagerKeepAliveTest extends TestWithFeService {
5860
private static final long TTL_SECONDS = 10;
@@ -89,7 +91,7 @@ public void testKeepAliveRecordsTsoForIdlePartitionAndReplaysIt() throws Excepti
8991
Assertions.assertEquals(1, logged.size());
9092
TableStreamCleanupInfo replayed = TableStreamCleanupInfo.read(roundTrip(logged.get(0)));
9193
Deencapsulation.setField(context.stream, "partitionKeepAlive",
92-
new HashMap<Long, OlapTableStream.PartitionKeepAlive>());
94+
new ConcurrentHashMap<Long, OlapTableStream.PartitionKeepAlive>());
9395
Env.getCurrentEnv().getTableStreamManager().replayTableStreamCleanup(replayed);
9496
Assertions.assertEquals(keepAliveTso, keepAliveTso(context, context.partition("p1").getId()));
9597
Assertions.assertNull(keepAliveTso(context, context.partition("p2").getId()));
@@ -140,8 +142,9 @@ public void testKeepAliveRenewsOnlyOnceTheMarginIsBelowHalfTheTtl() throws Excep
140142
Assertions.assertEquals(freshKeepAliveTso, keepAliveTso(context, partitionId));
141143

142144
// Six seconds old: less than half the TTL is left, and it is renewed.
143-
Deencapsulation.setField(context.stream, "partitionKeepAlive", new HashMap<>(Collections.singletonMap(
144-
partitionId, new OlapTableStream.PartitionKeepAlive(offset, tsoSecondsAgo(6)))));
145+
Deencapsulation.setField(context.stream, "partitionKeepAlive", new ConcurrentHashMap<>(
146+
Collections.singletonMap(partitionId,
147+
new OlapTableStream.PartitionKeepAlive(offset, tsoSecondsAgo(6)))));
145148
long renewed = tsoSecondsAgo(0);
146149
keepAlive(context, renewed, new ArrayList<>());
147150
Assertions.assertEquals(renewed, keepAliveTso(context, partitionId));
@@ -208,6 +211,24 @@ public void testStateWrittenBeforeKeepAliveTsoReads() throws Exception {
208211
Assertions.assertTrue(oldInfo.getKeepAliveEntries().isEmpty());
209212
}
210213

214+
@Test
215+
public void testKeepAliveTsoReadsBackFromTheImage() throws Exception {
216+
StreamContext context = createStreamContext("image");
217+
long partitionId = context.partition("p1").getId();
218+
long offset = tsoSecondsAgo(60);
219+
setIdle(context, "p1", offset);
220+
long keepAliveTso = tsoSecondsAgo(1);
221+
keepAliveDirectly(context, partitionId, offset, keepAliveTso);
222+
223+
OlapTableStream read = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(context.stream), OlapTableStream.class);
224+
OlapTableStream.PartitionKeepAlive keepAlive = read.getKeepAlive(partitionId);
225+
Assertions.assertNotNull(keepAlive);
226+
Assertions.assertEquals(offset, keepAlive.getOffset());
227+
Assertions.assertEquals(keepAliveTso, keepAlive.getKeepAliveTso());
228+
// An IVM refresh reads it without the stream lock, so it stays a concurrent map once read from an image.
229+
Assertions.assertInstanceOf(ConcurrentMap.class, Deencapsulation.getField(read, "partitionKeepAlive"));
230+
}
231+
211232
@Test
212233
public void testKeepAliveTakenAgainstAnotherOffsetDoesNotCount() throws Exception {
213234
StreamContext context = createStreamContext("moved_offset");
@@ -287,7 +308,7 @@ public void testCloudKeepAliveRecordsCommittedTsoForPartitionAtItsEnd() throws E
287308
Assertions.assertEquals(1, logged.size());
288309
TableStreamCleanupInfo replayed = TableStreamCleanupInfo.read(roundTrip(logged.get(0)));
289310
Deencapsulation.setField(context.stream, "partitionKeepAlive",
290-
new HashMap<Long, OlapTableStream.PartitionKeepAlive>());
311+
new ConcurrentHashMap<Long, OlapTableStream.PartitionKeepAlive>());
291312
Env.getCurrentEnv().getTableStreamManager().replayTableStreamCleanup(replayed);
292313
assertKeepAlive(context, p1, offset, committedTso);
293314
Assertions.assertNull(context.stream.getKeepAlive(p2));

0 commit comments

Comments
 (0)