Skip to content

Commit 4394fa3

Browse files
committed
[fix](routineload) Complete persistence compatibility coverage
### What problem does this PR solve? Issue Number: N/A Related PR: apache#64878 Problem Summary: Routine Load now persists its effective load definition directly, but nested legacy Expr SQL-carrier fields also need stable serialization, legacy image/log fixtures must come from the merge base, and ALTER replay must not apply new leader-side CSV validation to historical journals. Complete those compatibility requirements and add three-FE failover coverage for leader journal write, follower replay, checkpoint, and restart recovery. ### Release note Routine Load jobs preserve effective load clauses across ALTER, follower replay, checkpoint, and FE restart. Legacy images continue to migrate from the original CREATE statement; rollback to an older FE remains structurally readable but does not preserve new ALTER semantics. ### Check List (For Author) - Test: Not run per requested handoff; FE unit and Docker regression coverage were added. - Behavior changed: Yes. Routine Load persistence and ALTER replay retain the current effective definition. - Does this need documentation: Yes. Document rolling-upgrade and rollback limitations.
1 parent 419ac45 commit 4394fa3

14 files changed

Lines changed: 379 additions & 13 deletions

File tree

‎fe/fe-catalog/src/main/java/org/apache/doris/analysis/MatchPredicate.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ public String getName() {
7272
private String invertedIndexParserStopwords = "";
7373
private String invertedIndexAnalyzerName = "";
7474
// Fields for SQL generation
75+
@SerializedName("ea")
7576
private String explicitAnalyzer = "";
7677

7778
private MatchPredicate() {

‎fe/fe-catalog/src/main/java/org/apache/doris/analysis/SlotRef.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@ public class SlotRef extends Expr {
3838
@SerializedName("col")
3939
private String col;
4040
// Used in toSql
41+
@SerializedName("lbl")
4142
private String label;
43+
@SerializedName("scp")
4244
private List<String> subColPath;
4345

4446
// results of analysis

‎fe/fe-catalog/src/main/java/org/apache/doris/analysis/TimeV2Literal.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,21 @@
2020
import org.apache.doris.catalog.ScalarType;
2121
import org.apache.doris.catalog.Type;
2222

23+
import com.google.gson.annotations.SerializedName;
24+
2325
public class TimeV2Literal extends LiteralExpr {
2426
public static final TimeV2Literal MIN_VALUE = new TimeV2Literal(838, 59, 59, 999999, 6, true);
2527
public static final TimeV2Literal MAX_VALUE = new TimeV2Literal(838, 59, 59, 999999, 6, false);
2628

29+
@SerializedName("h")
2730
protected int hour;
31+
@SerializedName("M")
2832
protected int minute;
33+
@SerializedName("s")
2934
protected int second;
35+
@SerializedName("ms")
3036
protected int microsecond;
37+
@SerializedName("neg")
3138
protected boolean negative;
3239

3340
/**

‎fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2088,6 +2088,7 @@ private void restoreLegacyDefinition() throws UserException {
20882088

20892089
public abstract NereidsRoutineLoadTaskInfo toNereidsRoutineLoadTaskInfo() throws UserException;
20902090

2091+
// Leader-only validation. Replay must accept values written by older FE versions.
20912092
protected void validateCommonJobProperties(Map<String, String> jobProperties) throws UserException {
20922093
validateCsvFormatProperties(jobProperties);
20932094
if (jobProperties.containsKey(CreateRoutineLoadInfo.UNIQUE_KEY_UPDATE_MODE)) {
@@ -2106,9 +2107,8 @@ protected void validateCommonJobProperties(Map<String, String> jobProperties) th
21062107
}
21072108
}
21082109

2109-
// for ALTER ROUTINE LOAD. Validate all common properties before changing any common runtime state.
2110+
// Apply ALTER ROUTINE LOAD properties. The leader validates before mutation; replay trusts the journal.
21102111
protected void modifyCommonJobProperties(Map<String, String> jobProperties) throws UserException {
2111-
validateCommonJobProperties(jobProperties);
21122112
if (jobProperties.containsKey(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY)) {
21132113
this.desireTaskConcurrentNum = Integer.parseInt(
21142114
jobProperties.remove(CreateRoutineLoadInfo.DESIRED_CONCURRENT_NUMBER_PROPERTY));

‎fe/fe-core/src/main/java/org/apache/doris/load/routineload/kinesis/KinesisRoutineLoadJob.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -684,6 +684,7 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti
684684
throw new DdlException("Only supports modification of PAUSED jobs");
685685
}
686686

687+
validateCommonJobProperties(jobProperties);
687688
modifyPropertiesInternal(jobProperties, dataSourceProperties);
688689
setRoutineLoadDesc(command.getRoutineLoadDesc());
689690

@@ -698,7 +699,6 @@ public void modifyProperties(AlterRoutineLoadCommand command) throws UserExcepti
698699
private void modifyPropertiesInternal(Map<String, String> jobProperties,
699700
KinesisDataSourceProperties dataSourceProperties)
700701
throws UserException {
701-
validateCommonJobProperties(jobProperties);
702702
if (dataSourceProperties != null) {
703703
List<Pair<String, String>> shardPositions = Lists.newArrayList();
704704
Map<String, String> customKinesisProperties = Maps.newHashMap();

‎fe/fe-core/src/test/java/org/apache/doris/analysis/ExprGsonSerializationTest.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,13 +110,41 @@ public void testExprHolderRoundTrip() {
110110
Assertions.assertEquals(json, GsonUtilsCatalog.GSON.toJson(restored));
111111
}
112112

113+
@Test
114+
public void testRoutineLoadExprSqlRoundTrip() throws Exception {
115+
assertExprSqlRoundTrip(new MatchPredicate(MatchPredicate.Operator.MATCH_ANY,
116+
createNamedSlotRef("content"), new StringLiteral("hello"),
117+
Type.BOOLEAN, NullableMode.DEPEND_ON_ARGUMENT, null, false, "english"));
118+
assertExprSqlRoundTrip(new TimeV2Literal(12, 34, 56, 123456, 6, true));
119+
assertExprSqlRoundTrip(createNamedSlotRef("col1"));
120+
121+
SlotRef quotedSlot = new SlotRef(null, "a`b");
122+
quotedSlot.setLabel("`a``b`");
123+
quotedSlot.setType(Type.BIGINT);
124+
assertExprSqlRoundTrip(quotedSlot);
125+
126+
SlotRef subPathSlot = createNamedSlotRef("variant_col");
127+
setDeclaredField(SlotRef.class, subPathSlot, "subColPath", Arrays.asList("nested", "field"));
128+
String subPathJson = GsonUtilsCatalog.GSON.toJson(subPathSlot, Expr.class);
129+
SlotRef restoredSubPath = (SlotRef) GsonUtilsCatalog.GSON.fromJson(subPathJson, Expr.class);
130+
Assertions.assertEquals(Arrays.asList("nested", "field"), restoredSubPath.getSubColPath());
131+
}
132+
113133
private void assertExprRoundTrip(Class<? extends Expr> expectedClass, Expr expr) {
114134
String json = GsonUtilsCatalog.GSON.toJson(expr, Expr.class);
115135
Expr restored = GsonUtilsCatalog.GSON.fromJson(json, Expr.class);
116136
Assertions.assertEquals(expectedClass, restored.getClass());
117137
Assertions.assertEquals(json, GsonUtilsCatalog.GSON.toJson(restored, Expr.class));
118138
}
119139

140+
private void assertExprSqlRoundTrip(Expr expr) {
141+
String expectedSql = expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE);
142+
String json = GsonUtilsCatalog.GSON.toJson(expr, Expr.class);
143+
Expr restored = GsonUtilsCatalog.GSON.fromJson(json, Expr.class);
144+
Assertions.assertEquals(expectedSql,
145+
restored.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE));
146+
}
147+
120148
private Map<Class<? extends Expr>, Expr> createExprSamples() throws Exception {
121149
LinkedHashMap<Class<? extends Expr>, Expr> samples = new LinkedHashMap<>();
122150
samples.put(ArithmeticExpr.class, createArithmeticExpr());

‎fe/fe-core/src/test/java/org/apache/doris/load/routineload/KafkaRoutineLoadJobTest.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,21 @@ public void testAlterPersistsLoadDescAndCsvPropertiesForReplay() throws Exceptio
327327
assertAlterState(imageRoundTrip(follower));
328328
}
329329

330+
@Test
331+
public void testReplayLegacyCsvPropertiesDoesNotRunNewValidation() {
332+
KafkaRoutineLoadJob follower = createPausedJob();
333+
Map<String, String> legacyJobProperties = Maps.newHashMap();
334+
legacyJobProperties.put(CsvFileFormatProperties.PROP_ENCLOSE, "legacy");
335+
AlterRoutineLoadJobOperationLog legacyLog = new AlterRoutineLoadJobOperationLog(
336+
follower.getId(), legacyJobProperties, null);
337+
338+
follower.replayModifyProperties(legacyLog);
339+
340+
Assert.assertEquals((byte) 'l', follower.getEnclose());
341+
Map<String, String> persistedJobProperties = Deencapsulation.getField(follower, "jobProperties");
342+
Assert.assertEquals("legacy", persistedJobProperties.get(CsvFileFormatProperties.PROP_ENCLOSE));
343+
}
344+
330345
private static KafkaRoutineLoadJob createPausedJob() {
331346
KafkaRoutineLoadJob job = new KafkaRoutineLoadJob(1L, "job1", 1L,
332347
1L, "127.0.0.1:9020", "topic1", UserIdentity.ADMIN);

‎fe/fe-core/src/test/java/org/apache/doris/load/routineload/KinesisRoutineLoadJobTest.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -274,10 +274,12 @@ public void testAlterReplayKeepsDeltaAndCsvCachesInCheckpointParity() throws Exc
274274
Mockito.verify(editLog).logAlterRoutineLoadJob(logCaptor.capture());
275275
}
276276

277-
AlterRoutineLoadJobOperationLog log = logCaptor.getValue();
277+
AlterRoutineLoadJobOperationLog log = journalRoundTrip(logCaptor.getValue());
278278
replay.replayModifyProperties(log);
279279

280-
Assert.assertSame(delta, log.getRoutineLoadDesc());
280+
Assert.assertNotSame(delta, log.getRoutineLoadDesc());
281+
Assert.assertEquals("\n", log.getRoutineLoadDesc().getLineDelimiter().getSeparator());
282+
Assert.assertEquals("sequence_col", log.getRoutineLoadDesc().getSequenceColName());
281283
assertAlterResult(leader);
282284
assertAlterResult(replay);
283285
Assert.assertEquals(JsonParser.parseString(checkpointJson(leader)),
@@ -428,6 +430,17 @@ private void assertAlterResult(KinesisRoutineLoadJob job) {
428430
Assert.assertTrue(job.getEmptyFieldAsNull());
429431
}
430432

433+
private AlterRoutineLoadJobOperationLog journalRoundTrip(AlterRoutineLoadJobOperationLog log)
434+
throws Exception {
435+
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
436+
try (DataOutputStream out = new DataOutputStream(bytes)) {
437+
log.write(out);
438+
}
439+
try (DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) {
440+
return AlterRoutineLoadJobOperationLog.read(in);
441+
}
442+
}
443+
431444
private String checkpointJson(RoutineLoadJob job) throws Exception {
432445
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
433446
try (DataOutputStream out = new DataOutputStream(bytes)) {

‎fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,22 @@
1919

2020
import org.apache.doris.analysis.BinaryPredicate;
2121
import org.apache.doris.analysis.Expr;
22+
import org.apache.doris.analysis.ExprToSqlVisitor;
2223
import org.apache.doris.analysis.ImportColumnDesc;
2324
import org.apache.doris.analysis.IntLiteral;
25+
import org.apache.doris.analysis.MatchPredicate;
2426
import org.apache.doris.analysis.Separator;
2527
import org.apache.doris.analysis.SlotRef;
28+
import org.apache.doris.analysis.StringLiteral;
29+
import org.apache.doris.analysis.TimeV2Literal;
30+
import org.apache.doris.analysis.ToSqlParams;
2631
import org.apache.doris.analysis.UserIdentity;
2732
import org.apache.doris.catalog.Database;
2833
import org.apache.doris.catalog.Env;
34+
import org.apache.doris.catalog.Function.NullableMode;
2935
import org.apache.doris.catalog.OlapTable;
3036
import org.apache.doris.catalog.Table;
37+
import org.apache.doris.catalog.Type;
3138
import org.apache.doris.catalog.info.PartitionNamesInfo;
3239
import org.apache.doris.common.io.Text;
3340
import org.apache.doris.common.jmockit.Deencapsulation;
@@ -82,13 +89,21 @@ public void testDirectStateImageRoundTripDoesNotParseOrigStmt() throws Exception
8289
Separator lineDelimiter = analyzedSeparator("\\n");
8390
List<ImportColumnDesc> columns = Lists.newArrayList(
8491
new ImportColumnDesc("source_col"),
85-
new ImportColumnDesc("mapped_col", new IntLiteral(7L)));
86-
Expr precedingFilter = predicate(BinaryPredicate.Operator.GT, "source_col", 1L);
87-
Expr whereExpr = predicate(BinaryPredicate.Operator.LE, "mapped_col", 10L);
92+
new ImportColumnDesc("mapped_col", new TimeV2Literal(12, 34, 56, 123456, 6, true)));
93+
SlotRef matchSlot = namedSlot("content");
94+
Expr precedingFilter = new MatchPredicate(MatchPredicate.Operator.MATCH_ANY,
95+
matchSlot, new StringLiteral("hello world"), Type.BOOLEAN,
96+
NullableMode.DEPEND_ON_ARGUMENT, null, false, "english");
97+
SlotRef quotedSlot = namedSlot("a`b");
98+
Expr whereExpr = new BinaryPredicate(BinaryPredicate.Operator.GT, quotedSlot, new IntLiteral(10L));
8899
Expr deleteCondition = predicate(BinaryPredicate.Operator.EQ, "delete_flag", 1L);
89100
PartitionNamesInfo partitions = new PartitionNamesInfo(false, Lists.newArrayList("p1", "p2"));
90101
job.setRoutineLoadDesc(new RoutineLoadDesc(columnSeparator, lineDelimiter, columns,
91102
precedingFilter, whereExpr, partitions, deleteCondition, LoadTask.MergeType.MERGE, "seq_col"));
103+
String expectedColumnSql = exprToSql(columns.get(1).getExpr());
104+
String expectedPrecedingSql = exprToSql(precedingFilter);
105+
String expectedWhereSql = exprToSql(whereExpr);
106+
String expectedDeleteSql = exprToSql(deleteCondition);
92107

93108
job.desireTaskConcurrentNum = 5;
94109
job.maxErrorNum = 17L;
@@ -141,6 +156,10 @@ public void testDirectStateImageRoundTripDoesNotParseOrigStmt() throws Exception
141156
Assert.assertNotNull(restored.columnDescs.descs.get(1).getExpr());
142157
Assert.assertNotNull(restored.getPrecedingFilter());
143158
Assert.assertNotNull(restored.getWhereExpr());
159+
Assert.assertEquals(expectedColumnSql, exprToSql(restored.columnDescs.descs.get(1).getExpr()));
160+
Assert.assertEquals(expectedPrecedingSql, exprToSql(restored.getPrecedingFilter()));
161+
Assert.assertEquals(expectedWhereSql, exprToSql(restored.getWhereExpr()));
162+
Assert.assertEquals(expectedDeleteSql, exprToSql(restored.getDeleteCondition()));
144163
Assert.assertEquals("\\x01", restored.getColumnSeparator().getOriSeparator());
145164
Assert.assertEquals("\u0001", restored.getColumnSeparator().getSeparator());
146165
Assert.assertEquals("\\n", restored.getLineDelimiter().getOriSeparator());
@@ -391,6 +410,17 @@ private static Expr predicate(BinaryPredicate.Operator operator, String column,
391410
return new BinaryPredicate(operator, new SlotRef(null, column), new IntLiteral(value));
392411
}
393412

413+
private static SlotRef namedSlot(String column) {
414+
SlotRef slotRef = new SlotRef(null, column);
415+
slotRef.setLabel("`" + column.replace("`", "``") + "`");
416+
slotRef.setType(Type.VARCHAR);
417+
return slotRef;
418+
}
419+
420+
private static String exprToSql(Expr expr) {
421+
return expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE);
422+
}
423+
394424
private static JsonObject imageJson(RoutineLoadJob job) throws IOException {
395425
return imageJson(writeImage(job));
396426
}

‎fe/fe-core/src/test/java/org/apache/doris/persist/AlterRoutineLoadOperationLogTest.java‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,18 @@
1818
package org.apache.doris.persist;
1919

2020
import org.apache.doris.analysis.BinaryPredicate;
21+
import org.apache.doris.analysis.Expr;
22+
import org.apache.doris.analysis.ExprToSqlVisitor;
2123
import org.apache.doris.analysis.ImportColumnDesc;
2224
import org.apache.doris.analysis.IntLiteral;
25+
import org.apache.doris.analysis.MatchPredicate;
2326
import org.apache.doris.analysis.Separator;
27+
import org.apache.doris.analysis.SlotRef;
2428
import org.apache.doris.analysis.StringLiteral;
29+
import org.apache.doris.analysis.TimeV2Literal;
30+
import org.apache.doris.analysis.ToSqlParams;
31+
import org.apache.doris.catalog.Function.NullableMode;
32+
import org.apache.doris.catalog.Type;
2533
import org.apache.doris.catalog.info.PartitionNamesInfo;
2634
import org.apache.doris.common.UserException;
2735
import org.apache.doris.common.util.TimeUtils;
@@ -72,16 +80,20 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User
7280
Separator lineDelimiter = new Separator("\n", "\\n");
7381
List<ImportColumnDesc> columns = Lists.newArrayList(
7482
new ImportColumnDesc("source_col"),
75-
new ImportColumnDesc("mapped_col", new StringLiteral("mapped_value")));
76-
BinaryPredicate precedingFilter = new BinaryPredicate(BinaryPredicate.Operator.GT,
77-
new IntLiteral(3L), new IntLiteral(2L));
78-
BinaryPredicate where = new BinaryPredicate(BinaryPredicate.Operator.EQ,
79-
new StringLiteral("selected"), new StringLiteral("selected"));
83+
new ImportColumnDesc("mapped_col", new TimeV2Literal(12, 34, 56, 123456, 6, true)));
84+
Expr precedingFilter = new MatchPredicate(MatchPredicate.Operator.MATCH_ANY,
85+
namedSlot("content"), new StringLiteral("hello world"), Type.BOOLEAN,
86+
NullableMode.DEPEND_ON_ARGUMENT, null, false, "english");
87+
Expr where = new BinaryPredicate(BinaryPredicate.Operator.GT,
88+
namedSlot("a`b"), new IntLiteral(10L));
8089
PartitionNamesInfo partitions = new PartitionNamesInfo(true, Lists.newArrayList("p1", "p2"));
8190
BinaryPredicate deleteCondition = new BinaryPredicate(BinaryPredicate.Operator.EQ,
8291
new IntLiteral(1L), new IntLiteral(1L));
8392
RoutineLoadDesc routineLoadDesc = new RoutineLoadDesc(columnSeparator, lineDelimiter, columns,
8493
precedingFilter, where, partitions, deleteCondition, LoadTask.MergeType.MERGE, "sequence_col");
94+
String expectedColumnSql = exprToSql(columns.get(1).getExpr());
95+
String expectedPrecedingSql = exprToSql(precedingFilter);
96+
String expectedWhereSql = exprToSql(where);
8597
AlterRoutineLoadJobOperationLog log = new AlterRoutineLoadJobOperationLog(jobId,
8698
jobProperties, routineLoadDataSourceProperties, routineLoadDesc);
8799
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
@@ -115,6 +127,9 @@ public void testSerializeAlterRoutineLoadOperationLog() throws IOException, User
115127
Assert.assertNotNull(restoredDesc.getColumnsInfo().get(1).getExpr());
116128
Assert.assertNotNull(restoredDesc.getPrecedingFilter());
117129
Assert.assertNotNull(restoredDesc.getFilter());
130+
Assert.assertEquals(expectedColumnSql, exprToSql(restoredDesc.getColumnsInfo().get(1).getExpr()));
131+
Assert.assertEquals(expectedPrecedingSql, exprToSql(restoredDesc.getPrecedingFilter()));
132+
Assert.assertEquals(expectedWhereSql, exprToSql(restoredDesc.getFilter()));
118133
Assert.assertTrue(restoredDesc.getPartitionNamesInfo().isTemp());
119134
Assert.assertEquals(Lists.newArrayList("p1", "p2"),
120135
restoredDesc.getPartitionNamesInfo().getPartitionNames());
@@ -146,4 +161,15 @@ private static byte[] loadBase64Fixture(String resource) throws IOException {
146161
}
147162
}
148163

164+
private static SlotRef namedSlot(String column) {
165+
SlotRef slotRef = new SlotRef(null, column);
166+
slotRef.setLabel("`" + column.replace("`", "``") + "`");
167+
slotRef.setType(Type.VARCHAR);
168+
return slotRef;
169+
}
170+
171+
private static String exprToSql(Expr expr) {
172+
return expr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITHOUT_TABLE);
173+
}
174+
149175
}

0 commit comments

Comments
 (0)