Skip to content

Commit 6807723

Browse files
0AyanamiReiclaude
andcommitted
[fix](fe) Make routine load ALTER atomic and survive replay failures
### What problem does this PR solve? Issue Number: None Related PR: #66634 Problem Summary: Two failure paths of persisting the ALTER ROUTINE LOAD definition could stop FE or diverge master and followers. 1. Replay failure stopped journal replay. replayLoadDefinition wrapped UserException in IllegalStateException, and runtime exceptions such as the ParseException of the persisted ALTER, NoSuchElementException from getDb(dbId).get(), or the nereids AnalysisException of an expression passed through unchanged. Kafka and Kinesis replayModifyProperties only catch UserException, so the exception escaped journal replay and the master restart or follower replay failed. A realistic trigger is an upgrade whose analysis rules (function signatures, type inference, configs) reject an ALTER written by the previous version. Replay now follows the gsonPostProcess convention: catch the exception, cancel the job with a warning, and remove its txn callback. Nothing is applied before analysis and statement building succeed, so the job keeps its previous definition. The database is looked up with getDbOrMetaException instead of Optional.get(). 2. Master ALTER was not atomic. modifyProperties modified job properties, the load description and session variables before mergeLoadDescToOriginStatement, which calls getTableName(), the parser and Preconditions.checkState. If that step threw, the in-memory job had changed but no edit log was written, so the master ran the new definition while followers and a restarted master kept the old one. The step is split into mergeLoadDesc (overlay the ALTER clauses on the current definition without touching the job), buildLoadDefinitionStatement (build and verify the CREATE statement) and applyLoadDefinition (install it). modifyProperties builds and verifies the new statement before modifying the job, so a failure leaves the job and the journal unchanged. Replay uses the same helpers. ### Release note None ### Check List (For Author) - Test: Unit Test - New RoutineLoadJobPersistenceTest cases for Kafka and Kinesis: replay of an unparseable ALTER, an unanalyzable expression, a missing database and a missing table cancels the job without throwing; a master ALTER that fails to build the persisted statement leaves job properties, load definition, session variables, origStmt and the edit log unchanged. All 10 fail against the previous implementation. - ./run-fe-ut.sh --run RoutineLoadJobPersistenceTest, KafkaRoutineLoadJobTest, KinesisRoutineLoadJobTest, AlterRoutineLoadOperationLogTest, RoutineLoadJobTest, RoutineLoadManagerTest: 89 tests, 0 failures, 0 errors - BUILD_TYPE=ASAN ./build.sh --fe: passed, including checkstyle - The docker regression suite test_routine_load_alter_checkpoint_restart_fe was not run - Behavior changed: Yes. An ALTER ROUTINE LOAD journal entry whose load clauses can not be replayed cancels the job on that FE instead of stopping FE startup or replay, the same as an unrestorable job in the image. - Does this need documentation: No Co-Authored-By: Claude <[email protected]>
1 parent 79006c1 commit 6807723

4 files changed

Lines changed: 238 additions & 51 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java‎

Lines changed: 81 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -2048,48 +2048,60 @@ public void gsonPostProcess() throws IOException {
20482048
ctx.cleanup();
20492049
}
20502050
} catch (Exception e) {
2051-
// Terminalize this unusable job as CANCELLED. Set endTimestamp only if unset (avoid
2052-
// refreshing on every image load) and keep the existing cancel reason if present.
2053-
state = JobState.CANCELLED;
2054-
routineLoadTaskInfoList.clear();
2055-
long failureTimestamp = System.currentTimeMillis();
2056-
if (endTimestamp == -1) {
2057-
endTimestamp = failureTimestamp;
2058-
}
2059-
if (cancelReason == null) {
2060-
cancelReason = new ErrorReason(InternalErrorCode.INTERNAL_ERR,
2061-
"FE restart deserialize failed at " + TimeUtils.longToTimeString(failureTimestamp)
2062-
+ ": " + e.getMessage());
2063-
}
2051+
cancelUnrecoverableJob("FE restart deserialize failed", e);
20642052
LOG.warn("error happens when parsing create routine load stmt: " + origStmt.originStmt, e);
20652053
}
20662054
if (userIdentity != null) {
20672055
userIdentity.setIsAnalyzed();
20682056
}
20692057
}
20702058

2059+
// Terminalize a job whose persisted definition can not be restored as CANCELLED. Set endTimestamp
2060+
// only if unset (avoid refreshing on every image load) and keep the existing cancel reason if present.
2061+
private void cancelUnrecoverableJob(String reason, Exception e) {
2062+
state = JobState.CANCELLED;
2063+
routineLoadTaskInfoList.clear();
2064+
long failureTimestamp = System.currentTimeMillis();
2065+
if (endTimestamp == -1) {
2066+
endTimestamp = failureTimestamp;
2067+
}
2068+
if (cancelReason == null) {
2069+
cancelReason = new ErrorReason(InternalErrorCode.INTERNAL_ERR,
2070+
reason + " at " + TimeUtils.longToTimeString(failureTimestamp) + ": " + e.getMessage());
2071+
}
2072+
}
2073+
20712074
protected void replayLoadDefinition(OriginStatement alterStatement, Long sqlMode,
20722075
Map<String, String> alterSessionVariables) {
20732076
if (alterStatement == null) {
20742077
return;
20752078
}
2076-
Database database = Env.getCurrentEnv().getInternalCatalog().getDb(dbId).get();
2077-
ConnectContext ctx = createLoadDefinitionContext(database, sqlMode);
2078-
// Old journals have no snapshot and retain the existing recovery context.
2079-
ctx.getSessionVariable().setAffectQueryResultInPlanSessionVariables(alterSessionVariables);
20802079
try {
2081-
ctx.setThreadLocalInfo();
2082-
AlterRoutineLoadCommand command = (AlterRoutineLoadCommand) parsePersistedStatement(alterStatement);
2083-
if (command.hasLoadProperty()) {
2084-
setRoutineLoadDesc(command.analyzeLoadProperties(ctx, this));
2085-
updateLoadDefinitionSessionVariables(ctx.getSessionVariable().getAffectQueryResultInPlanVariables(),
2086-
ctx.getSessionVariable().getSqlMode());
2087-
mergeLoadDescToOriginStatement();
2080+
Database database = Env.getCurrentEnv().getInternalCatalog().getDbOrMetaException(dbId);
2081+
ConnectContext ctx = createLoadDefinitionContext(database, sqlMode);
2082+
// Old journals have no snapshot and retain the existing recovery context.
2083+
ctx.getSessionVariable().setAffectQueryResultInPlanSessionVariables(alterSessionVariables);
2084+
try {
2085+
ctx.setThreadLocalInfo();
2086+
AlterRoutineLoadCommand command =
2087+
(AlterRoutineLoadCommand) parsePersistedStatement(alterStatement);
2088+
if (command.hasLoadProperty()) {
2089+
RoutineLoadDesc loadDesc = mergeLoadDesc(command.analyzeLoadProperties(ctx, this));
2090+
OriginStatement loadDefinitionStmt = buildLoadDefinitionStatement(loadDesc);
2091+
applyLoadDefinition(loadDesc, loadDefinitionStmt,
2092+
ctx.getSessionVariable().getAffectQueryResultInPlanVariables(),
2093+
ctx.getSessionVariable().getSqlMode());
2094+
}
2095+
} finally {
2096+
ctx.cleanup();
20882097
}
2089-
} catch (UserException e) {
2090-
throw new IllegalStateException("failed to replay routine load definition for job " + id, e);
2091-
} finally {
2092-
ctx.cleanup();
2098+
} catch (Exception e) {
2099+
// Like gsonPostProcess(), an ALTER that this FE can not analyze any more (for example, after an
2100+
// upgrade changed the analysis rules) must not stop journal replay. The job keeps its previous
2101+
// definition and is cancelled, so it never loads data with a definition other than the ALTERed one.
2102+
cancelUnrecoverableJob("FE replay alter routine load failed", e);
2103+
Env.getCurrentGlobalTransactionMgr().getCallbackFactory().removeCallback(id);
2104+
LOG.warn("error happens when replaying alter routine load stmt of job {}, cancel it", id, e);
20932105
}
20942106
}
20952107

@@ -2119,34 +2131,64 @@ private ConnectContext createLoadDefinitionContext(Database database, Long sqlMo
21192131
return ctx;
21202132
}
21212133

2122-
protected void mergeLoadDescToOriginStatement() throws UserException {
2123-
List<ImportColumnDesc> columns =
2124-
columnDescs == null ? null : Lists.newArrayList(columnDescs.descs);
2125-
RoutineLoadDesc loadDesc = new RoutineLoadDesc(columnSeparator, lineDelimiter, columns,
2126-
precedingFilter, whereExpr, partitionNamesInfo, deleteCondition, mergeType, sequenceCol);
2134+
/**
2135+
* Overlay the load clauses of an ALTER on the current load definition without modifying this job.
2136+
* A clause replaces the current one only when the ALTER specifies it, the same as setRoutineLoadDesc().
2137+
*/
2138+
protected RoutineLoadDesc mergeLoadDesc(RoutineLoadDesc alterLoadDesc) {
2139+
List<ImportColumnDesc> columns = alterLoadDesc.getColumnsInfo();
2140+
if (columns == null && columnDescs != null) {
2141+
columns = Lists.newArrayList(columnDescs.descs);
2142+
}
2143+
return new RoutineLoadDesc(
2144+
alterLoadDesc.getColumnSeparator() != null ? alterLoadDesc.getColumnSeparator() : columnSeparator,
2145+
alterLoadDesc.getLineDelimiter() != null ? alterLoadDesc.getLineDelimiter() : lineDelimiter,
2146+
columns,
2147+
alterLoadDesc.getPrecedingFilter() != null ? alterLoadDesc.getPrecedingFilter() : precedingFilter,
2148+
alterLoadDesc.getFilter() != null ? alterLoadDesc.getFilter() : whereExpr,
2149+
alterLoadDesc.getPartitionNamesInfo() != null
2150+
? alterLoadDesc.getPartitionNamesInfo() : partitionNamesInfo,
2151+
alterLoadDesc.getDeleteCondition() != null ? alterLoadDesc.getDeleteCondition() : deleteCondition,
2152+
alterLoadDesc.getMergeType(),
2153+
alterLoadDesc.hasSequenceCol() ? alterLoadDesc.getSequenceColName() : sequenceCol);
2154+
}
2155+
2156+
/**
2157+
* Build the CREATE statement persisted as origStmt for the given effective load definition, and check
2158+
* that it can be parsed back. This does not modify the job, so callers must finish every step that may
2159+
* fail before applying the result with applyLoadDefinition().
2160+
*/
2161+
protected OriginStatement buildLoadDefinitionStatement(RoutineLoadDesc loadDesc) throws UserException {
21272162
StringBuilder sql = new StringBuilder("CREATE ROUTINE LOAD ")
21282163
.append(SqlUtils.getIdentSql(name));
21292164
if (!isMultiTable) {
21302165
sql.append(" ON ").append(SqlUtils.getIdentSql(getTableName()));
21312166
}
2132-
sql.append(" WITH ").append(mergeType.name());
2167+
sql.append(" WITH ").append(loadDesc.getMergeType().name());
21332168
String loadClauseSql = loadDesc.toSql();
21342169
if (!loadClauseSql.isEmpty()) {
21352170
sql.append(" ").append(loadClauseSql);
21362171
}
21372172
sql.append(" PROPERTIES (\"desired_concurrent_number\" = \"1\")");
21382173
sql.append(buildPersistedDataSourceSql());
2139-
OriginStatement effectiveStatement = new OriginStatement(sql.toString(), 0);
2140-
Preconditions.checkState(parsePersistedStatement(effectiveStatement) instanceof CreateRoutineLoadCommand,
2141-
effectiveStatement.originStmt);
2142-
origStmt = effectiveStatement;
2174+
OriginStatement loadDefinitionStmt = new OriginStatement(sql.toString(), 0);
2175+
Preconditions.checkState(parsePersistedStatement(loadDefinitionStmt) instanceof CreateRoutineLoadCommand,
2176+
loadDefinitionStmt.originStmt);
2177+
return loadDefinitionStmt;
21432178
}
21442179

2145-
protected void updateLoadDefinitionSessionVariables(Map<String, String> alterSessionVariables, long sqlMode) {
2180+
/**
2181+
* Install a load definition built by buildLoadDefinitionStatement(), together with the session
2182+
* variables that the persisted statement must be analyzed with.
2183+
*/
2184+
protected void applyLoadDefinition(RoutineLoadDesc loadDesc, OriginStatement loadDefinitionStmt,
2185+
Map<String, String> alterSessionVariables, long sqlMode) {
2186+
setRoutineLoadDesc(loadDesc);
21462187
if (alterSessionVariables != null) {
21472188
sessionVariables.putAll(alterSessionVariables);
21482189
}
21492190
sessionVariables.put(SessionVariable.SQL_MODE, Long.toString(sqlMode));
2191+
origStmt = loadDefinitionStmt;
21502192
}
21512193

21522194
private String buildPersistedDataSourceSql() {

‎fe/fe-core/src/main/java/org/apache/doris/load/routineload/kafka/KafkaRoutineLoadJob.java‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.apache.doris.common.util.SmallFileMgr.SmallFile;
4343
import org.apache.doris.common.util.TimeUtils;
4444
import org.apache.doris.datasource.kafka.KafkaUtil;
45+
import org.apache.doris.load.RoutineLoadDesc;
4546
import org.apache.doris.load.routineload.ErrorReason;
4647
import org.apache.doris.load.routineload.LoadDataSourceType;
4748
import org.apache.doris.load.routineload.RLTaskTxnCommitAttachment;
@@ -57,6 +58,7 @@
5758
import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo;
5859
import org.apache.doris.persist.AlterRoutineLoadJobOperationLog;
5960
import org.apache.doris.qe.ConnectContext;
61+
import org.apache.doris.qe.OriginStatement;
6062
import org.apache.doris.rpc.RpcException;
6163
import org.apache.doris.service.FrontendOptions;
6264
import org.apache.doris.thrift.TFileCompressType;
@@ -790,11 +792,19 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti
790792
throw new DdlException("Only supports modification of PAUSED jobs");
791793
}
792794

795+
// Build the new load definition before modifying the job, so a failure leaves the job and the
796+
// journal unchanged.
797+
RoutineLoadDesc loadDesc = null;
798+
OriginStatement loadDefinitionStmt = null;
799+
if (command.hasLoadProperty()) {
800+
loadDesc = mergeLoadDesc(command.getRoutineLoadDesc());
801+
loadDefinitionStmt = buildLoadDefinitionStatement(loadDesc);
802+
}
803+
793804
modifyPropertiesInternal(jobProperties, dataSourceProperties);
794805
if (command.hasLoadProperty()) {
795-
setRoutineLoadDesc(command.getRoutineLoadDesc());
796-
updateLoadDefinitionSessionVariables(command.getSessionVariables(), command.getSqlMode());
797-
mergeLoadDescToOriginStatement();
806+
applyLoadDefinition(loadDesc, loadDefinitionStmt, command.getSessionVariables(),
807+
command.getSqlMode());
798808
}
799809

800810
AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id,

‎fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.doris.common.util.LogBuilder;
3636
import org.apache.doris.common.util.LogKey;
3737
import org.apache.doris.datasource.kinesis.KinesisUtil;
38+
import org.apache.doris.load.RoutineLoadDesc;
3839
import org.apache.doris.load.routineload.ErrorReason;
3940
import org.apache.doris.load.routineload.LoadDataSourceType;
4041
import org.apache.doris.load.routineload.RLTaskTxnCommitAttachment;
@@ -50,6 +51,7 @@
5051
import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo;
5152
import org.apache.doris.persist.AlterRoutineLoadJobOperationLog;
5253
import org.apache.doris.qe.ConnectContext;
54+
import org.apache.doris.qe.OriginStatement;
5355
import org.apache.doris.thrift.TFileCompressType;
5456
import org.apache.doris.thrift.TPartialUpdateNewRowPolicy;
5557
import org.apache.doris.transaction.TransactionState;
@@ -686,11 +688,19 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti
686688
throw new DdlException("Only supports modification of PAUSED jobs");
687689
}
688690

691+
// Build the new load definition before modifying the job, so a failure leaves the job and the
692+
// journal unchanged.
693+
RoutineLoadDesc loadDesc = null;
694+
OriginStatement loadDefinitionStmt = null;
695+
if (command.hasLoadProperty()) {
696+
loadDesc = mergeLoadDesc(command.getRoutineLoadDesc());
697+
loadDefinitionStmt = buildLoadDefinitionStatement(loadDesc);
698+
}
699+
689700
modifyPropertiesInternal(jobProperties, dataSourceProperties);
690701
if (command.hasLoadProperty()) {
691-
setRoutineLoadDesc(command.getRoutineLoadDesc());
692-
updateLoadDefinitionSessionVariables(command.getSessionVariables(), command.getSqlMode());
693-
mergeLoadDescToOriginStatement();
702+
applyLoadDefinition(loadDesc, loadDefinitionStmt, command.getSessionVariables(),
703+
command.getSqlMode());
694704
}
695705

696706
AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(this.id,

0 commit comments

Comments
 (0)