Skip to content

Commit 509cfea

Browse files
authored
[feature](Load)(step2)support nereids load job schedule (#26356)
We will Integrate new load job manager into new job scheduling framework so that the insert into task can be scheduled after the broker load sql is converted to insert into TVF(table value function) sql. issue: #24221 Now support: 1. load data by tvf insert into sql, but just for simple load(columns need to be defined in the table) 2. show load stmt - job id, label name, job state, time info - simple progress 3. cancel load from db 4. support that enable new load through Config.enable_nereids_load 5. can replay job after restarting doris TODO: - support partition insert job - support show statistics from BE - support multiple task and collect task statistic - support transactional task - need add ut case
1 parent b8fd55b commit 509cfea

33 files changed

Lines changed: 1324 additions & 374 deletions

File tree

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1931,6 +1931,18 @@ public class Config extends ConfigBase {
19311931
@ConfField(masterOnly = true)
19321932
public static boolean enable_hms_events_incremental_sync = false;
19331933

1934+
/**
1935+
* If set to true, doris will try to parse the ddl of a hive view and try to execute the query
1936+
* otherwise it will throw an AnalysisException.
1937+
*/
1938+
@ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL, description = {
1939+
"当前默认设置为 false,开启后支持使用新优化器的load语句导入数据,失败后会降级旧的load语句。",
1940+
"Now default set to true, After this function is enabled, the load statement of "
1941+
+ "the new optimizer can be used to import data. If this function fails, "
1942+
+ "the old load statement will be degraded."})
1943+
public static boolean enable_nereids_load = false;
1944+
1945+
19341946
/**
19351947
* Maximum number of events to poll in each RPC.
19361948
*/

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -71,11 +71,6 @@ statement
7171
(withRemoteStorageSystem)?
7272
(PROPERTIES LEFT_PAREN properties=propertyItemList RIGHT_PAREN)?
7373
(commentSpec)? #load
74-
| LOAD LABEL lableName=identifier
75-
LEFT_PAREN dataDescs+=dataDesc (COMMA dataDescs+=dataDesc)* RIGHT_PAREN
76-
resourceDesc
77-
(PROPERTIES LEFT_PAREN properties=propertyItemList RIGHT_PAREN)?
78-
(commentSpec)? #resourceLoad
7974
| LOAD mysqlDataDesc
8075
(PROPERTIES LEFT_PAREN properties=propertyItemList RIGHT_PAREN)?
8176
(commentSpec)? #mysqlLoad
@@ -131,7 +126,7 @@ dataDesc
131126
(PARTITION partition=identifierList)?
132127
(COLUMNS TERMINATED BY comma=STRING_LITERAL)?
133128
(LINES TERMINATED BY separator=STRING_LITERAL)?
134-
(FORMAT AS format=identifier)?
129+
(FORMAT AS format=identifierOrStringLiteral)?
135130
(columns=identifierList)?
136131
(columnsFromPath=colFromPath)?
137132
(columnMapping=colMappingList)?
@@ -167,6 +162,11 @@ refreshMethod
167162
: COMPLETE | AUTO
168163
;
169164

165+
identifierOrStringLiteral
166+
: identifier
167+
| STRING_LITERAL
168+
;
169+
170170
identifierOrText
171171
: errorCapturingIdentifier
172172
| STRING_LITERAL
@@ -224,7 +224,8 @@ mappingExpr
224224
;
225225

226226
withRemoteStorageSystem
227-
: WITH S3 LEFT_PAREN
227+
: resourceDesc
228+
| WITH S3 LEFT_PAREN
228229
brokerProperties=propertyItemList
229230
RIGHT_PAREN
230231
| WITH HDFS LEFT_PAREN

‎fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -117,10 +117,8 @@ public void analyze(Analyzer analyzer) throws UserException {
117117
analyzerSqlStmt();
118118
// check its insert stmt,currently only support insert stmt
119119
//todo when support other stmt,need to check stmt type and generate jobInstance
120-
InsertJob job = new InsertJob();
121120
JobExecutionConfiguration jobExecutionConfiguration = new JobExecutionConfiguration();
122121
jobExecutionConfiguration.setExecuteType(executeType);
123-
job.setCreateTimeMs(System.currentTimeMillis());
124122
TimerDefinition timerDefinition = new TimerDefinition();
125123

126124
if (null != onceJobStartTimestamp) {
@@ -148,17 +146,19 @@ public void analyze(Analyzer analyzer) throws UserException {
148146
}
149147
checkJobName(labelName.getLabelName());
150148
jobExecutionConfiguration.setTimerDefinition(timerDefinition);
151-
job.setJobConfig(jobExecutionConfiguration);
152-
153-
job.setComment(comment);
154-
job.setCurrentDbName(labelName.getDbName());
155-
job.setJobName(labelName.getLabelName());
156-
job.setCreateUser(ConnectContext.get().getCurrentUserIdentity());
157-
job.setJobStatus(JobStatus.RUNNING);
158-
job.setJobId(Env.getCurrentEnv().getNextId());
159149
String originStmt = getOrigStmt().originStmt;
160150
String executeSql = parseExecuteSql(originStmt);
161-
job.setExecuteSql(executeSql);
151+
// create job use label name as its job name
152+
String jobName = labelName.getLabelName();
153+
InsertJob job = new InsertJob(jobName,
154+
JobStatus.RUNNING,
155+
labelName.getDbName(),
156+
comment,
157+
ConnectContext.get().getCurrentUserIdentity(),
158+
jobExecutionConfiguration,
159+
System.currentTimeMillis(),
160+
executeSql);
161+
//job.checkJobParams();
162162
jobInstance = job;
163163
}
164164

‎fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadStmt.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,13 @@ public Set<JobState> getStates() {
108108
return states;
109109
}
110110

111+
public org.apache.doris.load.loadv2.JobState getStateV2() {
112+
if (Strings.isNullOrEmpty(stateValue)) {
113+
return null;
114+
}
115+
return org.apache.doris.load.loadv2.JobState.valueOf(stateValue);
116+
}
117+
111118
public boolean isAccurateMatch() {
112119
return isAccurateMatch;
113120
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,7 @@
182182
import org.apache.doris.mysql.privilege.AccessControllerManager;
183183
import org.apache.doris.mysql.privilege.Auth;
184184
import org.apache.doris.mysql.privilege.PrivPredicate;
185+
import org.apache.doris.nereids.jobs.load.LabelProcessor;
185186
import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVPropertyInfo;
186187
import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVRefreshInfo;
187188
import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
@@ -362,6 +363,7 @@ public class Env {
362363

363364
private ExportTaskRegister exportTaskRegister;
364365
private JobManager<? extends AbstractJob<?, ?>, ?> jobManager;
366+
private LabelProcessor labelProcessor;
365367
private TransientTaskManager transientTaskManager;
366368

367369
private MasterDaemon labelCleaner; // To clean old LabelInfo, ExportJobInfos
@@ -641,8 +643,11 @@ private Env(boolean isCheckpointCatalog) {
641643
}
642644
this.metastoreEventsProcessor = new MetastoreEventsProcessor();
643645
this.jobManager = new JobManager<>();
646+
this.labelProcessor = new LabelProcessor();
644647
this.transientTaskManager = new TransientTaskManager();
645648
this.exportTaskRegister = new ExportTaskRegister(transientTaskManager);
649+
this.transientTaskManager = new TransientTaskManager();
650+
646651
this.replayedJournalId = new AtomicLong(0L);
647652
this.stmtIdCounter = new AtomicLong(0L);
648653
this.isElectable = false;
@@ -3907,6 +3912,10 @@ public JobManager getJobManager() {
39073912
return jobManager;
39083913
}
39093914

3915+
public LabelProcessor getLabelProcessor() {
3916+
return labelProcessor;
3917+
}
3918+
39103919
public TransientTaskManager getTransientTaskManager() {
39113920
return transientTaskManager;
39123921
}

‎fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java‎

Lines changed: 85 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
import org.apache.doris.catalog.ScalarType;
2424
import org.apache.doris.common.io.Text;
2525
import org.apache.doris.common.io.Writable;
26+
import org.apache.doris.common.util.LogBuilder;
27+
import org.apache.doris.common.util.LogKey;
2628
import org.apache.doris.common.util.TimeUtils;
2729
import org.apache.doris.job.common.JobStatus;
2830
import org.apache.doris.job.common.TaskStatus;
@@ -76,14 +78,55 @@ public abstract class AbstractJob<T extends AbstractTask, C> implements Job<T, C
7678
private JobExecutionConfiguration jobConfig;
7779

7880
@SerializedName(value = "ctms")
79-
private Long createTimeMs;
81+
private long createTimeMs;
8082

81-
@SerializedName(value = "sql")
82-
String executeSql;
83+
@SerializedName(value = "stm")
84+
private long startTimeMs = -1L;
8385

8486
@SerializedName(value = "ftm")
8587
private long finishTimeMs;
8688

89+
@SerializedName(value = "sql")
90+
String executeSql;
91+
92+
public AbstractJob() {}
93+
94+
public AbstractJob(Long id) {
95+
jobId = id;
96+
}
97+
98+
/**
99+
* executeSql and runningTasks is not required for load.
100+
*/
101+
public AbstractJob(Long jobId, String jobName, JobStatus jobStatus,
102+
String currentDbName,
103+
String comment,
104+
UserIdentity createUser,
105+
JobExecutionConfiguration jobConfig) {
106+
this(jobId, jobName, jobStatus, currentDbName, comment,
107+
createUser, jobConfig, System.currentTimeMillis(), null, null);
108+
}
109+
110+
public AbstractJob(Long jobId, String jobName, JobStatus jobStatus,
111+
String currentDbName,
112+
String comment,
113+
UserIdentity createUser,
114+
JobExecutionConfiguration jobConfig,
115+
Long createTimeMs,
116+
String executeSql,
117+
List<T> runningTasks) {
118+
this.jobId = jobId;
119+
this.jobName = jobName;
120+
this.jobStatus = jobStatus;
121+
this.currentDbName = currentDbName;
122+
this.comment = comment;
123+
this.createUser = createUser;
124+
this.jobConfig = jobConfig;
125+
this.createTimeMs = createTimeMs;
126+
this.executeSql = executeSql;
127+
this.runningTasks = runningTasks;
128+
}
129+
87130
private List<T> runningTasks = new ArrayList<>();
88131

89132
@Override
@@ -109,6 +152,10 @@ public void cancelAllTasks() throws JobException {
109152
.add("Comment")
110153
.build();
111154

155+
protected static long getNextJobId() {
156+
return System.nanoTime() + RandomUtils.nextInt();
157+
}
158+
112159
@Override
113160
public void cancelTaskById(long taskId) throws JobException {
114161
if (CollectionUtils.isEmpty(runningTasks)) {
@@ -154,17 +201,18 @@ public List<T> commonCreateTasks(TaskType taskType, C taskContext) {
154201
return createTasks(taskType, taskContext);
155202
}
156203

157-
public void initTasks(List<? extends AbstractTask> tasks) {
204+
public void initTasks(Collection<? extends T> tasks, TaskType taskType) {
205+
if (CollectionUtils.isEmpty(getRunningTasks())) {
206+
runningTasks = new ArrayList<>();
207+
}
158208
tasks.forEach(task -> {
159-
task.setJobId(jobId);
160-
task.setTaskId(getNextId());
209+
task.setTaskType(taskType);
210+
task.setJobId(getJobId());
161211
task.setCreateTimeMs(System.currentTimeMillis());
162212
task.setStatus(TaskStatus.PENDING);
163213
});
164-
if (CollectionUtils.isEmpty(getRunningTasks())) {
165-
setRunningTasks(new ArrayList<>());
166-
}
167-
getRunningTasks().addAll((Collection<? extends T>) tasks);
214+
getRunningTasks().addAll(tasks);
215+
this.startTimeMs = System.currentTimeMillis();
168216
}
169217

170218
public void checkJobParams() {
@@ -208,10 +256,22 @@ public void updateJobStatus(JobStatus newJobStatus) throws JobException {
208256
public static AbstractJob readFields(DataInput in) throws IOException {
209257
String jsonJob = Text.readString(in);
210258
AbstractJob job = GsonUtils.GSON.fromJson(jsonJob, AbstractJob.class);
211-
job.setRunningTasks(new ArrayList<>());
259+
job.runningTasks = new ArrayList<>();
212260
return job;
213261
}
214262

263+
public void logCreateOperation() {
264+
Env.getCurrentEnv().getEditLog().logCreateJob(this);
265+
}
266+
267+
public void logFinalOperation() {
268+
Env.getCurrentEnv().getEditLog().logEndJob(this);
269+
}
270+
271+
public void logUpdateOperation() {
272+
Env.getCurrentEnv().getEditLog().logUpdateJob(this);
273+
}
274+
215275
@Override
216276
public void onTaskFail(T task) throws JobException {
217277
updateJobStatusIfEnd();
@@ -303,7 +363,19 @@ public ShowResultSetMetaData getJobMetaData() {
303363
return builder.build();
304364
}
305365

306-
private static long getNextId() {
307-
return System.nanoTime() + RandomUtils.nextInt();
366+
@Override
367+
public void onRegister() throws JobException {}
368+
369+
@Override
370+
public void onUnRegister() throws JobException {}
371+
372+
@Override
373+
public void onReplayCreate() throws JobException {
374+
log.info(new LogBuilder(LogKey.SCHEDULER_JOB, getJobId()).add("msg", "replay create scheduler job").build());
375+
}
376+
377+
@Override
378+
public void onReplayEnd(AbstractJob<?, C> replayJob) throws JobException {
379+
log.info(new LogBuilder(LogKey.SCHEDULER_JOB, getJobId()).add("msg", "replay delete scheduler job").build());
308380
}
309381
}

‎fe/fe-core/src/main/java/org/apache/doris/job/base/Job.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,11 +99,34 @@ public interface Job<T extends AbstractTask, C> {
9999

100100
/**
101101
* Cancels all running tasks of this job.
102-
*
103102
* @throws JobException If cancelling a running task fails.
104103
*/
105104
void cancelAllTasks() throws JobException;
106105

106+
/**
107+
* register job
108+
* @throws JobException If register job failed.
109+
*/
110+
void onRegister() throws JobException;
111+
112+
/**
113+
* register job failed
114+
* @throws JobException If failed.
115+
*/
116+
void onUnRegister() throws JobException;
117+
118+
/**
119+
* replay create job
120+
* @throws JobException If replay create failed.
121+
*/
122+
void onReplayCreate() throws JobException;
123+
124+
/**
125+
* replay finished or cancelled job
126+
* @throws JobException If replay end failed.
127+
*/
128+
void onReplayEnd(AbstractJob<?, C> replayJob) throws JobException;
129+
107130
/**
108131
* Notifies the job when a task execution fails.
109132
*

‎fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecuteType.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ public enum JobExecuteType {
3737
*/
3838
MANUAL,
3939
/**
40-
* The job will be executed immediately.
40+
* The job will be executed only once and immediately.
4141
*/
4242
INSTANT,
4343
}

‎fe/fe-core/src/main/java/org/apache/doris/job/common/JobStatus.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ public enum JobStatus {
3636
* The stop state cannot be resumed
3737
*/
3838
STOPPED,
39-
4039
/**
4140
* When the task is finished, the finished state will be triggered.
4241
*/

‎fe/fe-core/src/main/java/org/apache/doris/job/common/JobType.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,5 @@
1919

2020
public enum JobType {
2121
INSERT,
22-
MV
22+
MV,
2323
}

0 commit comments

Comments
 (0)