Skip to content

Commit f0affa9

Browse files
authored
[fix](cloud) Release warm-up destination on initialization failure (apache#67924)
A cloud warm-up job registers its destination compute group before initializing tablet batches. If initialization throws, the outer `run()` handler only logs the exception: the job stays `PENDING` and keeps the destination registration. Other ONCE/PERIODIC jobs targeting that group cannot start. A later successful retry can recover the original job, but repeated initialization failures can block the group indefinitely because the warm-up timeout only applies to `RUNNING` jobs. Catch initialization failures before transitioning to `RUNNING` and reuse `cancel(..., false)` to persist the error and release the destination registration. ONCE jobs become `CANCELLED`; PERIODIC jobs remain `PENDING` and retry at their existing interval. Initialization has not submitted work to BEs, so this path does not send cleanup RPCs. Successful initialization retains the destination registration as before. ### Release note Release the destination compute group when cloud warm-up initialization fails, allowing subsequent warm-up jobs to proceed. Report the initialization error and preserve periodic retry scheduling.
1 parent ca30be1 commit f0affa9

2 files changed

Lines changed: 121 additions & 17 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java‎

Lines changed: 25 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -932,23 +932,31 @@ private void runPendingJob() throws DdlException {
932932
return;
933933
}
934934

935-
// Todo: nothing to prepare yet
936-
this.setJobDone = false;
937-
this.lastBatchId = -1;
938-
this.startTimeMs = System.currentTimeMillis();
939-
// reset clients to ensure we have the latest BE info
940-
this.beToThriftAddress = null;
941-
this.beToClient = null;
942-
this.beToAddr = null;
943-
MetricRepo.updateClusterWarmUpJobLatestStartTime(String.valueOf(jobId), srcClusterName,
944-
dstClusterName, startTimeMs);
945-
this.fetchBeToTabletIdBatches();
946-
long totalTablets = beToTabletIdBatches.values().stream()
947-
.flatMap(List::stream)
948-
.mapToLong(List::size)
949-
.sum();
950-
MetricRepo.increaseClusterWarmUpJobRequestedTablets(dstClusterName, totalTablets);
951-
MetricRepo.increaseClusterWarmUpJobExecCount(dstClusterName);
935+
long totalTablets;
936+
try {
937+
this.setJobDone = false;
938+
this.lastBatchId = -1;
939+
this.startTimeMs = System.currentTimeMillis();
940+
// reset clients to ensure we have the latest BE info
941+
this.beToThriftAddress = null;
942+
this.beToClient = null;
943+
this.beToAddr = null;
944+
MetricRepo.updateClusterWarmUpJobLatestStartTime(String.valueOf(jobId), srcClusterName,
945+
dstClusterName, startTimeMs);
946+
this.fetchBeToTabletIdBatches();
947+
totalTablets = beToTabletIdBatches.values().stream()
948+
.flatMap(List::stream)
949+
.mapToLong(List::size)
950+
.sum();
951+
MetricRepo.increaseClusterWarmUpJobRequestedTablets(dstClusterName, totalTablets);
952+
MetricRepo.increaseClusterWarmUpJobExecCount(dstClusterName);
953+
} catch (Exception e) {
954+
LOG.warn("failed to initialize cloud warm up job {}", jobId, e);
955+
// No BE job has started. Reuse cancellation to release the destination registration
956+
// and preserve periodic jobs for their next scheduled attempt.
957+
cancel("Failed to initialize warm up job: " + e.getMessage(), false);
958+
return;
959+
}
952960
this.jobState = JobState.RUNNING;
953961
Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(this);
954962
LOG.info("warmup-lock state-transition jobId={} srcCluster={} dstCluster={} syncMode={} jobType={} "

‎fe/fe-core/src/test/java/org/apache/doris/cloud/CloudWarmUpJobTest.java‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@
4141
import org.junit.jupiter.api.Assertions;
4242
import org.junit.jupiter.api.BeforeEach;
4343
import org.junit.jupiter.api.Test;
44+
import org.junit.jupiter.params.ParameterizedTest;
45+
import org.junit.jupiter.params.provider.EnumSource;
4446
import org.mockito.ArgumentCaptor;
4547
import org.mockito.MockedStatic;
4648
import org.mockito.Mockito;
@@ -56,6 +58,7 @@
5658
import java.util.LinkedHashMap;
5759
import java.util.List;
5860
import java.util.Map;
61+
import java.util.concurrent.ThreadPoolExecutor;
5962
import java.util.concurrent.atomic.AtomicReference;
6063

6164
public class CloudWarmUpJobTest {
@@ -186,6 +189,88 @@ public void testPendingRetryKeepsErrMsgWhenJobStarts() throws Exception {
186189
Mockito.verify(editLog).logModifyCloudWarmUpJob(job);
187190
}
188191

192+
@ParameterizedTest
193+
@EnumSource(value = SyncMode.class, names = {"ONCE", "PERIODIC"})
194+
public void testPendingInitializationFailureReleasesDestinationLock(SyncMode syncMode) throws Exception {
195+
CloudWarmUpJob job = Mockito.spy(createPendingJob(204L, syncMode));
196+
CloudWarmUpJob nextJob = createPendingJob(205L, SyncMode.ONCE);
197+
CloudEnv cloudEnv = Mockito.mock(CloudEnv.class);
198+
CacheHotspotManager manager = new CacheHotspotManager(Mockito.mock(CloudSystemInfoService.class),
199+
Mockito.mock(ThreadPoolExecutor.class));
200+
EditLog editLog = Mockito.mock(EditLog.class);
201+
Mockito.when(cloudEnv.getCacheHotspotMgr()).thenReturn(manager);
202+
Mockito.when(cloudEnv.getEditLog()).thenReturn(editLog);
203+
Mockito.doAnswer(invocation -> {
204+
Assertions.assertFalse(manager.tryRegisterRunningJob(nextJob));
205+
throw new IllegalStateException("initialization failed");
206+
}).when(job).fetchBeToTabletIdBatches();
207+
208+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
209+
mockedEnv.when(Env::getCurrentEnv).thenReturn(cloudEnv);
210+
job.run();
211+
212+
Assertions.assertTrue(manager.tryRegisterRunningJob(nextJob));
213+
Assertions.assertEquals(syncMode == SyncMode.ONCE ? JobState.CANCELLED : JobState.PENDING,
214+
job.getJobState());
215+
Assertions.assertEquals("Failed to initialize warm up job: initialization failed", job.getErrMsg());
216+
Assertions.assertTrue(job.getStartTimeMs() > 0);
217+
Assertions.assertTrue(job.getFinishedTimeMs() >= job.getStartTimeMs());
218+
Assertions.assertEquals(syncMode == SyncMode.PERIODIC, job.shouldWait());
219+
Mockito.verify(editLog).logModifyCloudWarmUpJob(job);
220+
221+
CloudWarmUpJob persistedJob = copyBySerialization(job);
222+
Assertions.assertEquals(job.getJobState(), persistedJob.getJobState());
223+
Assertions.assertEquals(job.getErrMsg(), persistedJob.getErrMsg());
224+
Assertions.assertEquals(job.getStartTimeMs(), persistedJob.getStartTimeMs());
225+
Assertions.assertEquals(job.getFinishedTimeMs(), persistedJob.getFinishedTimeMs());
226+
227+
nextJob.run();
228+
Assertions.assertEquals(JobState.RUNNING, nextJob.getJobState());
229+
Mockito.verifyNoInteractions(mockBackendPool);
230+
231+
if (syncMode == SyncMode.PERIODIC) {
232+
manager.notifyJobStop(nextJob);
233+
Mockito.doCallRealMethod().when(job).fetchBeToTabletIdBatches();
234+
setStartTimeMs(job, System.currentTimeMillis() - 61_000L);
235+
Assertions.assertFalse(job.shouldWait());
236+
job.run();
237+
Assertions.assertEquals(JobState.RUNNING, job.getJobState());
238+
Assertions.assertFalse(manager.tryRegisterRunningJob(nextJob));
239+
} else {
240+
job.run();
241+
Mockito.verify(job).fetchBeToTabletIdBatches();
242+
Assertions.assertEquals(JobState.CANCELLED, job.getJobState());
243+
}
244+
}
245+
}
246+
247+
@ParameterizedTest
248+
@EnumSource(value = SyncMode.class, names = {"ONCE", "PERIODIC"})
249+
public void testPendingInitializationKeepsDestinationLockOnSuccess(SyncMode syncMode) {
250+
CloudWarmUpJob job = createPendingJob(206L, syncMode);
251+
CloudWarmUpJob nextJob = Mockito.spy(createPendingJob(207L, SyncMode.ONCE));
252+
CloudEnv cloudEnv = Mockito.mock(CloudEnv.class);
253+
CacheHotspotManager manager = new CacheHotspotManager(Mockito.mock(CloudSystemInfoService.class),
254+
Mockito.mock(ThreadPoolExecutor.class));
255+
EditLog editLog = Mockito.mock(EditLog.class);
256+
Mockito.when(cloudEnv.getCacheHotspotMgr()).thenReturn(manager);
257+
Mockito.when(cloudEnv.getEditLog()).thenReturn(editLog);
258+
259+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
260+
mockedEnv.when(Env::getCurrentEnv).thenReturn(cloudEnv);
261+
job.run();
262+
Assertions.assertEquals(JobState.RUNNING, job.getJobState());
263+
264+
nextJob.run();
265+
Assertions.assertEquals(JobState.PENDING, nextJob.getJobState());
266+
Assertions.assertEquals(-1L, nextJob.getStartTimeMs());
267+
Assertions.assertFalse(manager.tryRegisterRunningJob(nextJob));
268+
Mockito.verify(nextJob, Mockito.never()).fetchBeToTabletIdBatches();
269+
Mockito.verify(editLog, Mockito.never()).logModifyCloudWarmUpJob(nextJob);
270+
Mockito.verify(editLog).logModifyCloudWarmUpJob(job);
271+
}
272+
}
273+
189274
@Test
190275
public void testEventDrivenSuccessfulRetryClearsErrMsg() throws Exception {
191276
CloudSystemInfoService cloudSystemInfoService = Mockito.mock(CloudSystemInfoService.class);
@@ -325,6 +410,17 @@ public void testRunningRetryClearsErrMsgWhenJobFinishes() throws Exception {
325410
Mockito.verify(mockBackendPool).returnObject(address, client);
326411
}
327412

413+
private CloudWarmUpJob createPendingJob(long jobId, SyncMode syncMode) {
414+
return new CloudWarmUpJob.Builder()
415+
.setJobId(jobId)
416+
.setSrcClusterName("source_cluster")
417+
.setDstClusterName("target_cluster")
418+
.setJobType(JobType.CLUSTER)
419+
.setSyncMode(syncMode)
420+
.setSyncInterval(60L)
421+
.build();
422+
}
423+
328424
private CloudWarmUpJob createRunningJob(long jobId, TNetworkAddress firstAddress,
329425
TNetworkAddress secondAddress) {
330426
CloudWarmUpJob job = new CloudWarmUpJob.Builder()

0 commit comments

Comments
 (0)