Skip to content

Commit e76a778

Browse files
committed
[fix](fe) Persist routine load job before scheduling
### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Routine load creation published a mutable job to the scheduler before serializing its create journal. The scheduler could change the job from NEED_SCHEDULE to RUNNING before serialization, while runtime task membership was not persisted. Replaying that record restored a RUNNING job without tasks. Persist the initial NEED_SCHEDULE job before publishing it so the create journal always contains a stable, reconstructable state. ### Release note Routine load creation now persists job metadata before the job becomes schedulable. ### Check List (For Author) - Test: No unit test executed per local review request; targeted FE Checkstyle passed. Added Kafka and Kinesis create-journal ordering coverage. - Behavior changed: Yes. A new routine load job becomes visible to the scheduler only after its create journal is durable. - Does this need documentation: No.
1 parent 303d121 commit e76a778

2 files changed

Lines changed: 148 additions & 1 deletion

File tree

‎fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,8 +228,9 @@ public void addRoutineLoadJob(RoutineLoadJob routineLoadJob, String dbName, Stri
228228
+ " routine load jobs are running. exceed limit.");
229229
}
230230

231-
unprotectedAddJob(routineLoadJob);
231+
// Persist the initial job before the scheduler can change its state or create tasks.
232232
Env.getCurrentEnv().getEditLog().logCreateRoutineLoadJob(routineLoadJob);
233+
unprotectedAddJob(routineLoadJob);
233234
} finally {
234235
writeUnlock();
235236
}
Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.load.routineload;
19+
20+
import org.apache.doris.analysis.UserIdentity;
21+
import org.apache.doris.catalog.Env;
22+
import org.apache.doris.common.io.Text;
23+
import org.apache.doris.common.io.Writable;
24+
import org.apache.doris.common.jmockit.Deencapsulation;
25+
import org.apache.doris.load.routineload.RoutineLoadJob.JobState;
26+
import org.apache.doris.load.routineload.kafka.KafkaProgress;
27+
import org.apache.doris.load.routineload.kafka.KafkaRoutineLoadJob;
28+
import org.apache.doris.load.routineload.kinesis.KinesisProgress;
29+
import org.apache.doris.load.routineload.kinesis.KinesisRoutineLoadJob;
30+
import org.apache.doris.persist.EditLog;
31+
import org.apache.doris.qe.ConnectContext;
32+
import org.apache.doris.transaction.GlobalTransactionMgrIface;
33+
import org.apache.doris.transaction.TxnStateCallbackFactory;
34+
35+
import com.google.common.collect.Lists;
36+
import com.google.gson.JsonParser;
37+
import org.junit.jupiter.api.AfterEach;
38+
import org.junit.jupiter.api.Assertions;
39+
import org.junit.jupiter.api.BeforeEach;
40+
import org.junit.jupiter.params.ParameterizedTest;
41+
import org.junit.jupiter.params.provider.EnumSource;
42+
import org.mockito.MockedStatic;
43+
import org.mockito.Mockito;
44+
45+
import java.io.ByteArrayInputStream;
46+
import java.io.ByteArrayOutputStream;
47+
import java.io.DataInputStream;
48+
import java.io.DataOutputStream;
49+
import java.io.IOException;
50+
import java.util.EnumSet;
51+
import java.util.Map;
52+
53+
public class RoutineLoadJobPersistenceTest {
54+
private MockedStatic<Env> envMock;
55+
private ConnectContext previousContext;
56+
private EditLog editLog;
57+
private RoutineLoadManager manager;
58+
private TxnStateCallbackFactory callbackFactory;
59+
60+
@BeforeEach
61+
public void setUp() {
62+
previousContext = ConnectContext.get();
63+
ConnectContext.remove();
64+
envMock = Mockito.mockStatic(Env.class);
65+
Env env = Mockito.mock(Env.class);
66+
GlobalTransactionMgrIface transactionMgr = Mockito.mock(GlobalTransactionMgrIface.class);
67+
editLog = Mockito.mock(EditLog.class);
68+
manager = new RoutineLoadManager();
69+
callbackFactory = new TxnStateCallbackFactory();
70+
envMock.when(Env::getCurrentEnv).thenReturn(env);
71+
envMock.when(Env::getCurrentGlobalTransactionMgr).thenReturn(transactionMgr);
72+
Mockito.when(env.getEditLog()).thenReturn(editLog);
73+
Mockito.when(env.getRoutineLoadManager()).thenReturn(manager);
74+
Mockito.when(env.getRoutineLoadTaskScheduler()).thenReturn(Mockito.mock(RoutineLoadTaskScheduler.class));
75+
Mockito.when(transactionMgr.getCallbackFactory()).thenReturn(callbackFactory);
76+
}
77+
78+
@AfterEach
79+
public void tearDown() {
80+
envMock.close();
81+
ConnectContext.remove();
82+
if (previousContext != null) {
83+
previousContext.setThreadLocalInfo();
84+
}
85+
}
86+
87+
@ParameterizedTest
88+
@EnumSource(value = LoadDataSourceType.class, names = {"KAFKA", "KINESIS"})
89+
public void testCreateJournalBeforePublishingJob(LoadDataSourceType dataSourceType) throws Exception {
90+
RoutineLoadJob job = createJob(dataSourceType);
91+
Mockito.doAnswer(invocation -> {
92+
// Exercise scheduling at the old race window, before the create record is serialized.
93+
for (RoutineLoadJob visibleJob : manager.getRoutineLoadJobByState(EnumSet.of(JobState.NEED_SCHEDULE))) {
94+
setSourceProgress(visibleJob, dataSourceType);
95+
visibleJob.divideRoutineLoadJob(1);
96+
}
97+
byte[] record = serialize(invocation.getArgument(0));
98+
Assertions.assertEquals(JobState.NEED_SCHEDULE.name(), serializedState(record));
99+
Assertions.assertNull(manager.getJob(job.getId()));
100+
Assertions.assertNull(callbackFactory.getCallback(job.getId()));
101+
Assertions.assertEquals(0, job.getSizeOfRoutineLoadTaskInfoList());
102+
return null;
103+
}).when(editLog).logCreateRoutineLoadJob(job);
104+
105+
manager.addRoutineLoadJob(job, "db", "tbl");
106+
107+
Mockito.verify(editLog).logCreateRoutineLoadJob(job);
108+
Assertions.assertSame(job, manager.getJob(job.getId()));
109+
Assertions.assertSame(job, callbackFactory.getCallback(job.getId()));
110+
setSourceProgress(job, dataSourceType);
111+
job.divideRoutineLoadJob(1);
112+
Assertions.assertEquals(JobState.RUNNING, job.getState());
113+
Assertions.assertEquals(1, job.getSizeOfRoutineLoadTaskInfoList());
114+
}
115+
116+
private RoutineLoadJob createJob(LoadDataSourceType dataSourceType) {
117+
if (dataSourceType == LoadDataSourceType.KINESIS) {
118+
return new KinesisRoutineLoadJob(1L, "job", 1L, 1L, "us-east-1", "stream", UserIdentity.ADMIN);
119+
}
120+
return new KafkaRoutineLoadJob(1L, "job", 1L, 1L, "127.0.0.1:9092", "topic", UserIdentity.ADMIN);
121+
}
122+
123+
private void setSourceProgress(RoutineLoadJob job, LoadDataSourceType dataSourceType) {
124+
if (dataSourceType == LoadDataSourceType.KINESIS) {
125+
job.progress = new KinesisProgress(Map.of("shard-0", "100"));
126+
Deencapsulation.setField(job, "openKinesisShards", Lists.newArrayList("shard-0"));
127+
} else {
128+
job.progress = new KafkaProgress(Map.of(0, 100L));
129+
Deencapsulation.setField(job, "currentKafkaPartitions", Lists.newArrayList(0));
130+
}
131+
}
132+
133+
private byte[] serialize(Writable value) throws IOException {
134+
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
135+
try (DataOutputStream out = new DataOutputStream(bytes)) {
136+
value.write(out);
137+
}
138+
return bytes.toByteArray();
139+
}
140+
141+
private String serializedState(byte[] record) throws IOException {
142+
try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(record))) {
143+
return JsonParser.parseString(Text.readString(in)).getAsJsonObject().get("st").getAsString();
144+
}
145+
}
146+
}

0 commit comments

Comments
 (0)