Skip to content

Commit cf84aba

Browse files
morningmanclaude
andcommitted
[fix](ivm) Wait for the TSO service before a keep-alive round
### What problem does this PR solve? Issue Number: None Related PR: #69831 Problem Summary: `startMasterOnlyDaemonThreads()` starts `IvmStreamKeepAliveDaemon` before the TSO service, and both wait for `Env.isReady` before their first cycle. The TSO service calibrates itself in that first cycle, which writes the next allocation window to the edit log and, in cloud mode, advances MetaService's TSO fence, so the first keep-alive check after an FE restart or a master change can come before it. The daemon set `lastRoundMs` before running the round, so such a round was spent for nothing: in shared-nothing mode `getTSO()` threw "TSO timestamp is not calibrated" at the first stream with a partition due for renewal, which ended the round, and in cloud mode the round was skipped because the TSO status was not initialized. The next round then came `ivm_stream_keepalive_interval_second` (600 seconds by default) later. With a `changelog.ttl_seconds` close to twice the interval, which the config description allows, an idle offset could expire before that round: an AUTO refresh fell back to COMPLETE and a strict INCREMENTAL refresh failed. No row was read wrong. A round now waits until the TSO service is initialized, without setting `lastRoundMs`, so the first check after the TSO service is ready starts it. ### Release note None ### Check List (For Author) - Test: Unit Test - `IvmStreamKeepAliveDaemonTest`: no round starts while the TSO service is not initialized, the first check after it is starts one, and the next round waits for the interval. - Behavior changed: No - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 19b8e52 commit cf84aba

2 files changed

Lines changed: 80 additions & 0 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmStreamKeepAliveDaemon.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,12 @@ protected void runAfterCatalogReady() {
6363
if (!Config.enable_table_stream || !Config.enable_ivm_stream_keepalive) {
6464
return;
6565
}
66+
// The TSO service calibrates itself in its own first round once this FE is master, which can come after a
67+
// check of this daemon. A round started before that would have no keep-alive TSO to take, and the next
68+
// round would come a whole interval later, so a round waits until the TSO service is ready.
69+
if (!Env.getCurrentEnv().getTSOService().getStatusSnapshot().isInitialized()) {
70+
return;
71+
}
6672
long now = System.currentTimeMillis();
6773
if (now - lastRoundMs < Config.ivm_stream_keepalive_interval_second * 1000L) {
6874
return;
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.mtmv.ivm;
19+
20+
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.common.Config;
22+
import org.apache.doris.tso.TSOService;
23+
24+
import org.junit.jupiter.api.Assertions;
25+
import org.junit.jupiter.api.Test;
26+
import org.mockito.MockedStatic;
27+
import org.mockito.Mockito;
28+
29+
import java.util.concurrent.atomic.AtomicInteger;
30+
31+
public class IvmStreamKeepAliveDaemonTest {
32+
33+
@Test
34+
public void testRoundWaitsForTheTsoServiceWithoutWaitingAnInterval() {
35+
boolean enableTableStream = Config.enable_table_stream;
36+
boolean enableKeepAlive = Config.enable_ivm_stream_keepalive;
37+
int interval = Config.ivm_stream_keepalive_interval_second;
38+
AtomicInteger rounds = new AtomicInteger();
39+
IvmStreamKeepAliveDaemon daemon = new IvmStreamKeepAliveDaemon() {
40+
@Override
41+
public void keepAliveIdlePartitions() {
42+
rounds.incrementAndGet();
43+
}
44+
};
45+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
46+
Config.enable_table_stream = true;
47+
Config.enable_ivm_stream_keepalive = true;
48+
Config.ivm_stream_keepalive_interval_second = 600;
49+
Env env = Mockito.mock(Env.class);
50+
TSOService tsoService = Mockito.mock(TSOService.class);
51+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
52+
Mockito.when(env.getTSOService()).thenReturn(tsoService);
53+
54+
// A new master checks before its TSO service has calibrated itself: no round is started.
55+
Mockito.when(tsoService.getStatusSnapshot()).thenReturn(new TSOService.TSOStatusSnapshot(false, 0, 0));
56+
daemon.runAfterCatalogReady();
57+
daemon.runAfterCatalogReady();
58+
Assertions.assertEquals(0, rounds.get());
59+
60+
// The first check after the TSO service is ready starts the round, not one an interval later.
61+
Mockito.when(tsoService.getStatusSnapshot()).thenReturn(new TSOService.TSOStatusSnapshot(true, 1, 0));
62+
daemon.runAfterCatalogReady();
63+
Assertions.assertEquals(1, rounds.get());
64+
65+
// The next round waits for the interval.
66+
daemon.runAfterCatalogReady();
67+
Assertions.assertEquals(1, rounds.get());
68+
} finally {
69+
Config.enable_table_stream = enableTableStream;
70+
Config.enable_ivm_stream_keepalive = enableKeepAlive;
71+
Config.ivm_stream_keepalive_interval_second = interval;
72+
}
73+
}
74+
}

0 commit comments

Comments
 (0)