Skip to content

Commit be49e67

Browse files
authored
[improve](streaming-job) async chunk splitting for cdc source job (apache#63079)
## Summary `StreamingInsertJob` (CDC FROM-TO and `cdc_stream` TVF paths) used to call `splitChunks()` synchronously inside `CREATE STREAMING JOB`, asking cdc_client to cut every chunk of every table before returning. On large/non-uniform PK tables this can take 30+ minutes — far beyond the BE→cdc_client BRPC 60s timeout, and the SQL client blocks the whole time. This PR makes splitting tick-driven by the FE scheduler: - `CREATE` returns immediately; no more synchronous `splitChunks()`. - Each scheduler tick `advanceSplits()` issues one short fetchSplits RPC (default `batchSize=100`) and pushes that batch into `remainingSplits`. Tasks dispatch as soon as the first batch lands, so end-to-end first-byte latency stays close to flink-cdc's. - cdc_client is **stateless** — every RPC reconstructs `ChunkSplitter` from the `(currentSplittingTable, nextSplitStart, nextSplitId)` triple supplied by FE; flink-cdc internals are untouched (uses the public `ChunkSplitter` API only). - Crash recovery uses three sources of truth: - editlog persists `committedSplitProgress` (3-field `SplitProgress`) + existing `chunkHighWatermarkMap` / `binlogOffsetPersist` - `streaming_job_meta` system table holds full `chunk_list` JSON per table (UPSERT each `advanceSplits`) - cdc_client memory holds nothing - Both FROM-TO (multi-table) and TVF (single-table) paths share the same `SourceOffsetProvider#initSplitProgress` / `noMoreSplits` / `advanceSplits` interface; `StreamingJobSchedulerTask.handlePendingState` pre-advances one batch so the first task doesn't wait a full `max_interval`.
1 parent 8d3bd8a commit be49e67

23 files changed

Lines changed: 3038 additions & 387 deletions

‎docker/thirdparties/docker-compose/postgresql/postgresql-14.yaml.tpl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,9 @@ services:
4040
- "-c"
4141
- "wal_level=logical"
4242
- "-c"
43-
- "max_wal_senders=30"
43+
- "max_wal_senders=64"
4444
- "-c"
45-
- "max_replication_slots=30"
45+
- "max_replication_slots=64"
4646
- "-c"
4747
- "ssl=on"
4848
- "-c"

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1182,6 +1182,9 @@ public class Config extends ConfigBase {
11821182
@ConfField(mutable = true, masterOnly = true)
11831183
public static int streaming_cdc_heavy_rpc_timeout_sec = 600;
11841184

1185+
@ConfField(mutable = true, masterOnly = true)
1186+
public static int streaming_cdc_fetch_splits_batch_size = 100;
1187+
11851188
/**
11861189
* the max timeout of get kafka meta.
11871190
*/

‎fe/fe-common/src/main/java/org/apache/doris/job/cdc/request/FetchTableSplitsRequest.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,18 @@
3333
public class FetchTableSplitsRequest extends JobBaseConfig {
3434
private String snapshotTable;
3535

36+
/**
37+
* The next split start, null = fresh start (START_BOUND).
38+
* cdc_client takes to construct ChunkSplitterState.
39+
*/
40+
private Object[] nextSplitStart;
41+
42+
/** Next split id; null = 0 (fresh start). */
43+
private Integer nextSplitId;
44+
45+
/** Splits to fetch in this RPC; null lets cdc_client use its default. */
46+
private Integer batchSize;
47+
3648
public FetchTableSplitsRequest(Long jobId, String name,
3749
Map<String, String> sourceProperties, String frontendAddress, String snapshotTable) {
3850
super(jobId.toString(), name, sourceProperties, frontendAddress);

‎fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@
9494
import java.io.IOException;
9595
import java.util.ArrayList;
9696
import java.util.Arrays;
97+
import java.util.Collections;
9798
import java.util.Comparator;
9899
import java.util.HashMap;
99100
import java.util.LinkedHashMap;
@@ -174,6 +175,11 @@ public class StreamingInsertJob extends AbstractJob<StreamingJobSchedulerTask, M
174175
@SerializedName("ccn")
175176
private volatile String cloudCluster;
176177

178+
/** Source tables this job syncs. */
179+
@Getter
180+
@SerializedName("st")
181+
private List<String> syncTables;
182+
177183
// The sampling window starts at the beginning of the sampling window.
178184
// If the error rate exceeds `max_filter_ratio` within the window, the sampling fails.
179185
@Setter
@@ -238,6 +244,7 @@ private void initSourceJob() {
238244
init();
239245
checkRequiredSourceProperties();
240246
List<String> createTbls = createTableIfNotExists();
247+
this.syncTables = createTbls;
241248
if (sourceProperties.get(DataSourceConfigKeys.INCLUDE_TABLES) == null) {
242249
// cdc need the final includeTables
243250
String includeTables = String.join(",", createTbls);
@@ -246,8 +253,7 @@ private void initSourceJob() {
246253
StreamingJobUtils.resolveAndValidateSource(
247254
dataSourceType, sourceProperties, String.valueOf(getJobId()), createTbls);
248255
this.offsetProvider = createOffsetProvider(getConvertedSourceProperties());
249-
JdbcSourceOffsetProvider rdsOffsetProvider = (JdbcSourceOffsetProvider) this.offsetProvider;
250-
rdsOffsetProvider.splitChunks(createTbls);
256+
this.offsetProvider.initOnCreate(this.syncTables);
251257
} catch (Exception ex) {
252258
log.warn("init streaming job for {} failed", dataSourceType, ex);
253259
throw new RuntimeException(ex.getMessage());
@@ -396,10 +402,13 @@ private void initInsertJob() {
396402
this.originTvfProps = currentTvf.getProperties().getMap();
397403
this.offsetProvider = createOffsetProvider(sourceProperties);
398404
this.offsetProvider.ensureInitialized(getJobId(), originTvfProps);
405+
// cdc_stream TVF has TABLE; S3 TVF doesn't, so syncTables stays null there.
406+
String tvfTable = originTvfProps.get(DataSourceConfigKeys.TABLE);
407+
this.syncTables = tvfTable == null ? null : Collections.singletonList(tvfTable);
399408
// Validate source-side resources (e.g. PG slot/publication ownership) once at job
400409
// creation so conflicts fail fast. No-op for standalone cdc_stream TVF (no job).
401410
StreamingJobUtils.validateTvfSource(tvfType, originTvfProps, String.valueOf(getJobId()));
402-
this.offsetProvider.initOnCreate();
411+
this.offsetProvider.initOnCreate(this.syncTables);
403412
// validate offset props, only for s3 cause s3 tvf no offset prop
404413
if (jobProperties.getOffsetProperty() != null
405414
&& S3TableValuedFunction.NAME.equalsIgnoreCase(tvfType)) {
@@ -718,6 +727,28 @@ protected void fetchMeta() throws JobException {
718727
}
719728
}
720729

730+
/**
731+
* Advance one batch of split fetching if there are still tables to split.
732+
* Called by scheduler each tick (PENDING/RUNNING). Mirrors fetchMeta error handling.
733+
*/
734+
public void advanceSplitsIfNeed() throws JobException {
735+
if (offsetProvider.noMoreSplits()) {
736+
return;
737+
}
738+
try {
739+
offsetProvider.advanceSplits();
740+
} catch (Exception ex) {
741+
log.warn("advance splits failed, job id: {}", getJobId(), ex);
742+
if (this.getFailureReason() == null
743+
|| !InternalErrorCode.MANUAL_PAUSE_ERR.equals(this.getFailureReason().getCode())) {
744+
this.setFailureReason(new FailureReason(
745+
InternalErrorCode.GET_REMOTE_DATA_ERROR,
746+
"Failed to advance splits, " + ex.getMessage()));
747+
this.updateJobStatus(JobStatus.PAUSED);
748+
}
749+
}
750+
}
751+
721752
public boolean needScheduleTask() {
722753
readLock();
723754
try {
@@ -1487,6 +1518,8 @@ private void persistOffsetProviderIfNeed() {
14871518

14881519
public void replayOffsetProviderIfNeed() throws JobException {
14891520
if (offsetProvider != null) {
1521+
// when fe restart, offsetProvider.jobId/sourceProperties may be null
1522+
offsetProvider.ensureInitialized(getJobId(), getProviderProps());
14901523
offsetProvider.replayIfNeed(this);
14911524
}
14921525
}

‎fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobSchedulerTask.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,12 @@ private void handlePendingState() throws JobException {
6666
}
6767
}
6868
streamingInsertJob.replayOffsetProviderIfNeed();
69+
// Pre-advance one batch so the first task has splits to consume
70+
streamingInsertJob.advanceSplitsIfNeed();
71+
if (streamingInsertJob.getJobStatus() == JobStatus.PAUSED) {
72+
// advanceSplits failed and paused the job; skip task dispatch this tick.
73+
return;
74+
}
6975
if (streamingInsertJob.hasReachedEnd()) {
7076
// Source already fully consumed (e.g. snapshot-only mode recovered after FE restart).
7177
// Transition directly to FINISHED without creating a new task.
@@ -80,6 +86,7 @@ private void handlePendingState() throws JobException {
8086
private void handleRunningState() throws JobException {
8187
streamingInsertJob.processTimeoutTasks();
8288
streamingInsertJob.fetchMeta();
89+
streamingInsertJob.advanceSplitsIfNeed();
8390
}
8491

8592
private void autoResumeHandler() throws JobException {

‎fe/fe-core/src/main/java/org/apache/doris/job/offset/SourceOffsetProvider.java‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,11 @@ public interface SourceOffsetProvider {
4545
default void ensureInitialized(Long jobId, Map<String, String> originTvfProps) throws JobException {}
4646

4747
/**
48-
* Performs one-time initialization that must run only on fresh job creation, not on FE restart.
49-
* For example, fetching and persisting snapshot splits to the meta table.
48+
* One-time initialization on fresh job creation (not on FE restart). Subclasses may
49+
* initialize split progress, fetch initial splits, or open remote readers.
5050
* Default: no-op (most providers need no extra setup).
5151
*/
52-
default void initOnCreate() throws JobException {}
52+
default void initOnCreate(List<String> syncTables) throws JobException {}
5353

5454
/**
5555
* Get next offset to consume
@@ -180,6 +180,24 @@ default boolean hasReachedEnd() {
180180
return false;
181181
}
182182

183+
/**
184+
* Advance one batch of split fetching, called by scheduler each tick during PENDING/RUNNING.
185+
* For providers without async splitting work (e.g. S3, Kafka), default is no-op.
186+
* Aligned with flink-cdc SnapshotSplitAssigner naming.
187+
*
188+
* @throws JobException if fetching splits fails fatally
189+
*/
190+
default void advanceSplits() throws JobException {}
191+
192+
/**
193+
* Returns true if no more splits will be produced.
194+
* For providers without splitting concept, always returns true.
195+
* Aligned with flink-cdc SnapshotSplitAssigner.noMoreSplits() naming.
196+
*/
197+
default boolean noMoreSplits() {
198+
return true;
199+
}
200+
183201
/**
184202
* Get the lag of the data source in seconds.
185203
* For CDC sources, lag = (now - last consumed event timestamp) in seconds.

0 commit comments

Comments
 (0)