Skip to content

Commit 3543341

Browse files
authored
[fix](streaming-job) misc fixes for typo/log/validation/visibility (apache#63480)
## Summary Small fixes accumulated during a code review of the streaming-job module. None of these change runtime semantics for currently-supported scenarios; they tighten validation, fix a typo, and harden a concurrent field. - `StreamingInsertJob.cleanup()`: log was filling `dbId=%s` with `getJobId()`. Fix to `getDbId()`. - `StreamingInsertJob.getShowSQL()`: typo `TO DATABSE` → `TO DATABASE` in the synthesized SQL shown by SHOW JOBS. - `JdbcSourceOffsetProvider.currentOffset`: mark `volatile`. The write in `updateOffset()` happens outside `splitsLock` while `hasMoreDataToConsume()` / `getShowCurrentOffset()` read it without holding the lock. - `DataSourceConfigValidator.isValidValue`: `snapshot_split_size` / `snapshot_parallelism` previously only checked null/empty. Reject non-numeric and non-positive values up front so users get a clear CREATE-time error instead of a context-less `NumberFormatException` at runtime. - `CdcStreamTableValuedFunction.validate`: previously only checked `jdbc_url` / `type` / `table` / `offset`. Now also rejects missing `database` (MySQL), missing `schema` (PostgreSQL), and unsupported `type` values, so the error surfaces at TVF parse time rather than later inside `getRemoteDbName`.
1 parent 090b476 commit 3543341

8 files changed

Lines changed: 282 additions & 26 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/DataSourceConfigValidator.java‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,12 @@ public static void validateSource(Map<String, String> input,
109109
}
110110
}
111111

112-
// Cross-field: verify-ca must be paired with a CA cert; otherwise the reader will
113-
// silently fall back to the JVM default truststore and likely fail to connect.
112+
validateSslVerifyCaPair(input);
113+
}
114+
115+
// Cross-field: verify-ca must be paired with a CA cert; otherwise the reader will
116+
// silently fall back to the JVM default truststore and likely fail to connect.
117+
public static void validateSslVerifyCaPair(Map<String, String> input) throws IllegalArgumentException {
114118
if (DataSourceConfigKeys.SSL_MODE_VERIFY_CA.equals(input.get(DataSourceConfigKeys.SSL_MODE))
115119
&& (input.get(DataSourceConfigKeys.SSL_ROOTCERT) == null
116120
|| input.get(DataSourceConfigKeys.SSL_ROOTCERT).trim().isEmpty())) {
@@ -153,19 +157,44 @@ private static boolean isValidValue(String key, String value, String dataSourceT
153157
return isValidOffset(value, dataSourceType);
154158
}
155159

156-
// slot_name / publication_name are interpolated into PG DDL without quoting,
157-
// so enforce unquoted-identifier grammar to prevent injection and runtime errors.
158160
if (key.equals(DataSourceConfigKeys.SLOT_NAME)
159161
|| key.equals(DataSourceConfigKeys.PUBLICATION_NAME)) {
160-
return value.length() <= PG_MAX_IDENTIFIER_LENGTH
161-
&& PG_IDENTIFIER_PATTERN.matcher(value).matches();
162+
return isValidPgIdentifier(value);
162163
}
163-
if (key.equals(DataSourceConfigKeys.SSL_MODE) && !ALLOW_SSL_MODES.contains(value)) {
164-
return false;
164+
if (key.equals(DataSourceConfigKeys.SSL_MODE)) {
165+
return isValidSslMode(value);
166+
}
167+
if (key.equals(DataSourceConfigKeys.SNAPSHOT_SPLIT_SIZE)
168+
|| key.equals(DataSourceConfigKeys.SNAPSHOT_PARALLELISM)) {
169+
return isPositiveInt(value);
165170
}
166171
return true;
167172
}
168173

174+
public static boolean isPositiveInt(String value) {
175+
if (value == null) {
176+
return false;
177+
}
178+
try {
179+
return Integer.parseInt(value) > 0;
180+
} catch (NumberFormatException e) {
181+
return false;
182+
}
183+
}
184+
185+
// slot_name / publication_name are interpolated into PG DDL without quoting,
186+
// so enforce unquoted-identifier grammar to prevent injection and runtime errors.
187+
public static boolean isValidPgIdentifier(String value) {
188+
return value != null
189+
&& !value.isEmpty()
190+
&& value.length() <= PG_MAX_IDENTIFIER_LENGTH
191+
&& PG_IDENTIFIER_PATTERN.matcher(value).matches();
192+
}
193+
194+
public static boolean isValidSslMode(String value) {
195+
return ALLOW_SSL_MODES.contains(value);
196+
}
197+
169198
/**
170199
* Check if the offset value is valid for the given data source type.
171200
* Supported: initial, snapshot, latest, JSON binlog/lsn position.

‎fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1108,7 +1108,7 @@ private String getShowSQL() {
11081108
.append("'='").append(entry.getValue()).append("',");
11091109
}
11101110
sb.deleteCharAt(sb.length() - 1);
1111-
sb.append(" ) TO DATABSE ").append(targetDb);
1111+
sb.append(" ) TO DATABASE ").append(targetDb);
11121112
if (!targetProperties.isEmpty()) {
11131113
sb.append(" (");
11141114
for (Map.Entry<String, String> entry : targetProperties.entrySet()) {
@@ -1554,7 +1554,7 @@ public void cleanup() throws JobException {
15541554
if (resp.getStatus().getCode() != Cloud.MetaServiceCode.OK) {
15551555
log.warn("failed to delete streaming job, response: {}", resp);
15561556
throw new JobException("deleteJobKey failed for jobId=%s, dbId=%s, status=%s",
1557-
getJobId(), getJobId(), resp.getStatus());
1557+
getJobId(), getDbId(), resp.getStatus());
15581558
}
15591559
} catch (RpcException e) {
15601560
log.warn("failed to delete streaming job {}", resp, e);

‎fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
8787
List<SnapshotSplit> remainingSplits = new ArrayList<>();
8888
List<SnapshotSplit> finishedSplits = new ArrayList<>();
8989

90-
JdbcOffset currentOffset;
90+
volatile JdbcOffset currentOffset;
9191
Map<String, String> endBinlogOffset;
9292

9393
@SerializedName("chw")
@@ -213,10 +213,10 @@ public InsertIntoTableCommand rewriteTvfParams(InsertIntoTableCommand originComm
213213

214214
@Override
215215
public void updateOffset(Offset offset) {
216-
this.currentOffset = (JdbcOffset) offset;
217-
if (currentOffset.snapshotSplit()) {
216+
JdbcOffset newOffset = (JdbcOffset) offset;
217+
if (newOffset.snapshotSplit()) {
218218
synchronized (splitsLock) {
219-
List<? extends AbstractSourceSplit> splits = currentOffset.getSplits();
219+
List<? extends AbstractSourceSplit> splits = newOffset.getSplits();
220220
for (AbstractSourceSplit split : splits) {
221221
SnapshotSplit snapshotSplit = (SnapshotSplit) split;
222222
String splitId = split.getSplitId();
@@ -247,10 +247,11 @@ public void updateOffset(Offset offset) {
247247
}
248248
}
249249
} else {
250-
BinlogSplit binlogSplit = (BinlogSplit) currentOffset.getSplits().get(0);
250+
BinlogSplit binlogSplit = (BinlogSplit) newOffset.getSplits().get(0);
251251
binlogOffsetPersist = new HashMap<>(binlogSplit.getStartingOffset());
252252
binlogOffsetPersist.put(SPLIT_ID, BinlogSplit.BINLOG_SPLIT_ID);
253253
}
254+
this.currentOffset = newOffset;
254255
}
255256

256257
@Override

‎fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -260,15 +260,15 @@ private List<Map<String, String>> buildCumulativeSnapshotOffset(
260260
* adds it to finishedSplits. During txn replay remainingSplits is empty so removeIf returns
261261
* false naturally — chunkHighWatermarkMap is still updated for replayIfNeed to use later.
262262
*
263-
* <p>Binlog: currentOffset is set above. Also mirror startingOffset into binlogOffsetPersist
264-
* so it survives FE checkpoint via image (currentOffset has no @SerializedName).
263+
* <p>Binlog: mirror startingOffset into binlogOffsetPersist so it survives FE checkpoint via
264+
* image (currentOffset has no @SerializedName).
265265
*/
266266
@Override
267267
public void updateOffset(Offset offset) {
268-
this.currentOffset = (JdbcOffset) offset;
269-
if (currentOffset.snapshotSplit()) {
268+
JdbcOffset newOffset = (JdbcOffset) offset;
269+
if (newOffset.snapshotSplit()) {
270270
synchronized (splitsLock) {
271-
for (AbstractSourceSplit split : currentOffset.getSplits()) {
271+
for (AbstractSourceSplit split : newOffset.getSplits()) {
272272
SnapshotSplit ss = (SnapshotSplit) split;
273273
boolean removed = remainingSplits.removeIf(v -> {
274274
if (v.getSplitId().equals(ss.getSplitId())) {
@@ -289,12 +289,13 @@ public void updateOffset(Offset offset) {
289289
}
290290
} else {
291291
// Mirror binlog offset into bop so it survives FE checkpoint
292-
BinlogSplit bs = (BinlogSplit) currentOffset.getSplits().get(0);
292+
BinlogSplit bs = (BinlogSplit) newOffset.getSplits().get(0);
293293
if (MapUtils.isNotEmpty(bs.getStartingOffset())) {
294294
binlogOffsetPersist = new HashMap<>(bs.getStartingOffset());
295295
binlogOffsetPersist.put(SPLIT_ID, BinlogSplit.BINLOG_SPLIT_ID);
296296
}
297297
}
298+
this.currentOffset = newOffset;
298299
}
299300

300301
/**

‎fe/fe-core/src/main/java/org/apache/doris/tablefunction/CdcStreamTableValuedFunction.java‎

Lines changed: 64 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.doris.job.cdc.DataSourceConfigKeys;
2626
import org.apache.doris.job.cdc.request.FetchRecordRequest;
2727
import org.apache.doris.job.common.DataSourceType;
28+
import org.apache.doris.job.extensions.insert.streaming.DataSourceConfigValidator;
2829
import org.apache.doris.job.util.StreamingJobUtils;
2930
import org.apache.doris.thrift.TBrokerFileStatus;
3031
import org.apache.doris.thrift.TFileType;
@@ -106,18 +107,77 @@ private String generateParams(Map<String, String> properties) throws AnalysisExc
106107
}
107108

108109
private void validate(Map<String, String> properties) throws AnalysisException {
109-
if (!properties.containsKey(DataSourceConfigKeys.JDBC_URL)) {
110+
if (StringUtils.isEmpty(properties.get(DataSourceConfigKeys.JDBC_URL))) {
110111
throw new AnalysisException("jdbc_url is required");
111112
}
112-
if (!properties.containsKey(DataSourceConfigKeys.TYPE)) {
113+
if (StringUtils.isEmpty(properties.get(DataSourceConfigKeys.TYPE))) {
113114
throw new AnalysisException("type is required");
114115
}
115-
if (!properties.containsKey(DataSourceConfigKeys.TABLE)) {
116+
if (StringUtils.isEmpty(properties.get(DataSourceConfigKeys.TABLE))) {
116117
throw new AnalysisException("table is required");
117118
}
118-
if (!properties.containsKey(DataSourceConfigKeys.OFFSET)) {
119+
if (StringUtils.isEmpty(properties.get(DataSourceConfigKeys.OFFSET))) {
119120
throw new AnalysisException("offset is required");
120121
}
122+
DataSourceType sourceType;
123+
try {
124+
sourceType = DataSourceType.valueOf(properties.get(DataSourceConfigKeys.TYPE).toUpperCase());
125+
} catch (IllegalArgumentException e) {
126+
throw new AnalysisException("Unsupported type: " + properties.get(DataSourceConfigKeys.TYPE));
127+
}
128+
switch (sourceType) {
129+
case MYSQL:
130+
if (StringUtils.isEmpty(properties.get(DataSourceConfigKeys.DATABASE))) {
131+
throw new AnalysisException("database is required for MySQL");
132+
}
133+
break;
134+
case POSTGRES:
135+
if (StringUtils.isEmpty(properties.get(DataSourceConfigKeys.SCHEMA))) {
136+
throw new AnalysisException("schema is required for PostgreSQL");
137+
}
138+
validatePgIdentifierIfPresent(properties, DataSourceConfigKeys.SLOT_NAME);
139+
validatePgIdentifierIfPresent(properties, DataSourceConfigKeys.PUBLICATION_NAME);
140+
break;
141+
default:
142+
throw new AnalysisException("Unsupported type: " + sourceType);
143+
}
144+
String offset = properties.get(DataSourceConfigKeys.OFFSET);
145+
if (!DataSourceConfigValidator.isValidOffset(offset, sourceType.name())) {
146+
throw new AnalysisException("Invalid value for key 'offset': " + offset);
147+
}
148+
String sslMode = properties.get(DataSourceConfigKeys.SSL_MODE);
149+
if (sslMode != null && !DataSourceConfigValidator.isValidSslMode(sslMode)) {
150+
throw new AnalysisException("Invalid value for key 'ssl_mode': " + sslMode);
151+
}
152+
try {
153+
DataSourceConfigValidator.validateSslVerifyCaPair(properties);
154+
} catch (IllegalArgumentException e) {
155+
throw new AnalysisException(e.getMessage());
156+
}
157+
validatePositiveIntIfPresent(properties, DataSourceConfigKeys.SNAPSHOT_SPLIT_SIZE);
158+
validatePositiveIntIfPresent(properties, DataSourceConfigKeys.SNAPSHOT_PARALLELISM);
159+
}
160+
161+
private static void validatePositiveIntIfPresent(Map<String, String> properties, String key)
162+
throws AnalysisException {
163+
String value = properties.get(key);
164+
if (value == null) {
165+
return;
166+
}
167+
if (!DataSourceConfigValidator.isPositiveInt(value)) {
168+
throw new AnalysisException("Invalid value for key '" + key + "': " + value);
169+
}
170+
}
171+
172+
private static void validatePgIdentifierIfPresent(Map<String, String> properties, String key)
173+
throws AnalysisException {
174+
String value = properties.get(key);
175+
if (value == null) {
176+
return;
177+
}
178+
if (!DataSourceConfigValidator.isValidPgIdentifier(value)) {
179+
throw new AnalysisException("Invalid value for key '" + key + "': " + value);
180+
}
121181
}
122182

123183
private void generateFileStatus() {

‎fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/DataSourceConfigValidatorTest.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,4 +224,56 @@ public void testSlotNameAcceptsValidPgIdentifiers() {
224224
DataSourceConfigValidator.validateSource(props, DataSourceType.POSTGRES.name());
225225
}
226226
}
227+
228+
@Test
229+
public void testSnapshotSplitSizeAcceptsPositiveInteger() {
230+
Map<String, String> props = new HashMap<>();
231+
props.put(DataSourceConfigKeys.JDBC_URL, "jdbc:mysql://host/db");
232+
props.put(DataSourceConfigKeys.SNAPSHOT_SPLIT_SIZE, "8192");
233+
DataSourceConfigValidator.validateSource(props, DataSourceType.MYSQL.name());
234+
}
235+
236+
@Test
237+
public void testSnapshotSplitSizeRejectsNonNumeric() {
238+
Map<String, String> props = new HashMap<>();
239+
props.put(DataSourceConfigKeys.JDBC_URL, "jdbc:mysql://host/db");
240+
props.put(DataSourceConfigKeys.SNAPSHOT_SPLIT_SIZE, "abc");
241+
assertReject(props);
242+
}
243+
244+
@Test
245+
public void testSnapshotSplitSizeRejectsZeroAndNegative() {
246+
for (String v : new String[]{"0", "-1"}) {
247+
Map<String, String> props = new HashMap<>();
248+
props.put(DataSourceConfigKeys.JDBC_URL, "jdbc:mysql://host/db");
249+
props.put(DataSourceConfigKeys.SNAPSHOT_SPLIT_SIZE, v);
250+
assertReject(props);
251+
}
252+
}
253+
254+
@Test
255+
public void testSnapshotParallelismAcceptsPositiveInteger() {
256+
Map<String, String> props = new HashMap<>();
257+
props.put(DataSourceConfigKeys.JDBC_URL, "jdbc:mysql://host/db");
258+
props.put(DataSourceConfigKeys.SNAPSHOT_PARALLELISM, "4");
259+
DataSourceConfigValidator.validateSource(props, DataSourceType.MYSQL.name());
260+
}
261+
262+
@Test
263+
public void testSnapshotParallelismRejectsNonNumeric() {
264+
Map<String, String> props = new HashMap<>();
265+
props.put(DataSourceConfigKeys.JDBC_URL, "jdbc:mysql://host/db");
266+
props.put(DataSourceConfigKeys.SNAPSHOT_PARALLELISM, "abc");
267+
assertReject(props);
268+
}
269+
270+
@Test
271+
public void testSnapshotParallelismRejectsZeroAndNegative() {
272+
for (String v : new String[]{"0", "-2"}) {
273+
Map<String, String> props = new HashMap<>();
274+
props.put(DataSourceConfigKeys.JDBC_URL, "jdbc:mysql://host/db");
275+
props.put(DataSourceConfigKeys.SNAPSHOT_PARALLELISM, v);
276+
assertReject(props);
277+
}
278+
}
227279
}

‎regression-test/suites/job_p0/streaming_job/cdc/tvf/test_cdc_stream_tvf_mysql.groovy‎

Lines changed: 82 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,87 @@ suite("test_cdc_stream_tvf_mysql", "p0,external,mysql,external_docker,external_d
5757
exception "offset is required"
5858
}
5959

60+
test {
61+
sql """select * from cdc_stream(
62+
"type" = "mysql",
63+
"jdbc_url" = "jdbc:mysql://localhost:3306",
64+
"table" = "t1",
65+
"offset" = "latest")"""
66+
exception "database is required for MySQL"
67+
}
68+
69+
test {
70+
sql """select * from cdc_stream(
71+
"type" = "unknown_db",
72+
"jdbc_url" = "jdbc:foo://localhost:3306",
73+
"table" = "t1",
74+
"offset" = "latest")"""
75+
exception "Unsupported type"
76+
}
77+
78+
test {
79+
sql """select * from cdc_stream(
80+
"type" = "",
81+
"jdbc_url" = "jdbc:mysql://localhost:3306",
82+
"table" = "t1",
83+
"offset" = "latest")"""
84+
exception "type is required"
85+
}
86+
87+
test {
88+
sql """select * from cdc_stream(
89+
"type" = "mysql",
90+
"jdbc_url" = "jdbc:mysql://localhost:3306",
91+
"database" = "db1",
92+
"table" = "t1",
93+
"offset" = "latest",
94+
"snapshot_split_size" = "abc")"""
95+
exception "Invalid value for key 'snapshot_split_size'"
96+
}
97+
98+
test {
99+
sql """select * from cdc_stream(
100+
"type" = "mysql",
101+
"jdbc_url" = "jdbc:mysql://localhost:3306",
102+
"database" = "db1",
103+
"table" = "t1",
104+
"offset" = "latest",
105+
"snapshot_parallelism" = "0")"""
106+
exception "Invalid value for key 'snapshot_parallelism'"
107+
}
108+
109+
test {
110+
sql """select * from cdc_stream(
111+
"type" = "mysql",
112+
"jdbc_url" = "jdbc:mysql://localhost:3306",
113+
"database" = "db1",
114+
"table" = "t1",
115+
"offset" = "abc")"""
116+
exception "Invalid value for key 'offset'"
117+
}
118+
119+
test {
120+
sql """select * from cdc_stream(
121+
"type" = "mysql",
122+
"jdbc_url" = "jdbc:mysql://localhost:3306",
123+
"database" = "db1",
124+
"table" = "t1",
125+
"offset" = "latest",
126+
"ssl_mode" = "bogus")"""
127+
exception "Invalid value for key 'ssl_mode'"
128+
}
129+
130+
test {
131+
sql """select * from cdc_stream(
132+
"type" = "mysql",
133+
"jdbc_url" = "jdbc:mysql://localhost:3306",
134+
"database" = "db1",
135+
"table" = "t1",
136+
"offset" = "latest",
137+
"ssl_mode" = "verify-ca")"""
138+
exception "ssl_mode 'verify-ca' requires ssl_rootcert to be set"
139+
}
140+
60141
// --- Data setup ---
61142

62143
connect("root", "123456", "jdbc:mysql://${externalEnvIp}:${mysql_port}") {
@@ -161,7 +242,7 @@ suite("test_cdc_stream_tvf_mysql", "p0,external,mysql,external_docker,external_d
161242
"database" = "${mysqlDb}",
162243
"table" = "${table1}",
163244
"offset" = 'notjson')"""
164-
exception "Unsupported offset: notjson"
245+
exception "Invalid value for key 'offset'"
165246
}
166247

167248
// --- Non-existent table ---

0 commit comments

Comments
 (0)