Skip to content

Commit 2a4682b

Browse files
committed
[improve](streaming-job) admit multiple splits per scheduler tick to speed up many-table snapshot
1 parent 4fec154 commit 2a4682b

4 files changed

Lines changed: 188 additions & 1 deletion

File tree

‎fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingInsertJob.java‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,9 @@ public class StreamingInsertJob extends AbstractJob<StreamingJobSchedulerTask, M
134134
@SerializedName("props")
135135
private Map<String, String> properties;
136136
private StreamingJobProperties jobProperties;
137+
// Soft cap on produced-but-unconsumed splits buffered on FE; checked before each round,
138+
// so the actual backlog may overshoot by up to one fetch batch.
139+
private static final int MAX_PENDING_SPLITS = 512;
137140
@Getter
138141
@SerializedName("tvf")
139142
private String tvfType;
@@ -737,8 +740,20 @@ public void advanceSplitsIfNeed() throws JobException {
737740
if (offsetProvider.noMoreSplits()) {
738741
return;
739742
}
743+
// Admit as many splits as possible this tick until the pending backlog hits the soft cap,
744+
// then yield before the next tick (deadline) which resumes the rest.
745+
long intervalMs = jobProperties.getMaxIntervalSecond() * 1000L;
746+
long deadline = System.currentTimeMillis() + intervalMs - Math.min(1000L, intervalMs / 10);
740747
try {
741-
offsetProvider.advanceSplits();
748+
while (!offsetProvider.noMoreSplits()
749+
&& offsetProvider.pendingSplitCount() < MAX_PENDING_SPLITS
750+
&& System.currentTimeMillis() < deadline) {
751+
int before = offsetProvider.pendingSplitCount();
752+
offsetProvider.advanceSplits();
753+
if (offsetProvider.pendingSplitCount() <= before) {
754+
break; // nothing produced this round; avoid spin
755+
}
756+
}
742757
} catch (Exception ex) {
743758
log.warn("advance splits failed, job id: {}", getJobId(), ex);
744759
if (this.getFailureReason() == null

‎fe/fe-core/src/main/java/org/apache/doris/job/offset/SourceOffsetProvider.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,11 @@ default boolean noMoreSplits() {
198198
return true;
199199
}
200200

201+
/** Splits produced but not yet consumed (FE-side backlog). */
202+
default int pendingSplitCount() {
203+
return 0;
204+
}
205+
201206
/**
202207
* Get the lag of the data source in seconds.
203208
* For CDC sources, lag = (now - last consumed event timestamp) in seconds.

‎fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -653,6 +653,13 @@ public void initOnCreate(List<String> syncTables) throws JobException {
653653
}
654654
}
655655

656+
@Override
657+
public int pendingSplitCount() {
658+
synchronized (splitsLock) {
659+
return remainingSplits.size();
660+
}
661+
}
662+
656663
@Override
657664
public boolean noMoreSplits() {
658665
if (!checkNeedSplitChunks(sourceProperties)) {
Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.job.extensions.insert.streaming;
19+
20+
import org.apache.doris.common.jmockit.Deencapsulation;
21+
import org.apache.doris.job.offset.Offset;
22+
import org.apache.doris.job.offset.SourceOffsetProvider;
23+
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
24+
25+
import org.junit.Assert;
26+
import org.junit.Test;
27+
28+
import java.util.HashMap;
29+
import java.util.Map;
30+
31+
/**
32+
* Covers the bounded admission loop in {@link StreamingInsertJob#advanceSplitsIfNeed()}:
33+
* it must keep admitting within one tick yet terminate on every in-loop exit condition
34+
* (noMoreSplits / pending cap / no-progress round) without spinning. The deadline term is
35+
* wall-clock based and is covered by integration/regression rather than here.
36+
*/
37+
public class StreamingInsertJobAdvanceSplitsTest {
38+
39+
// Assumed value of StreamingInsertJob.MAX_PENDING_SPLITS; update both together if changed.
40+
private static final int CAP = 512;
41+
42+
private static StreamingInsertJob newJob(SourceOffsetProvider provider, long maxIntervalSec) {
43+
StreamingInsertJob job = Deencapsulation.newInstance(StreamingInsertJob.class);
44+
Map<String, String> props = new HashMap<>();
45+
props.put(StreamingJobProperties.MAX_INTERVAL_SECOND_PROPERTY, String.valueOf(maxIntervalSec));
46+
Deencapsulation.setField(job, "jobProperties", new StreamingJobProperties(props));
47+
Deencapsulation.setField(job, "offsetProvider", provider);
48+
return job;
49+
}
50+
51+
@Test
52+
public void testLoopsUntilNoMoreSplits() throws Exception {
53+
// Many small tables: one split per round, splitting completes after 5 rounds.
54+
FakeOffsetProvider provider = new FakeOffsetProvider(1, 5);
55+
newJob(provider, 3600).advanceSplitsIfNeed();
56+
Assert.assertEquals("must keep admitting across rounds until noMoreSplits", 5, provider.rounds);
57+
}
58+
59+
@Test
60+
public void testStopsAtPendingCap() throws Exception {
61+
// Fast producer that never finishes: must stop once the FE backlog cap is crossed.
62+
FakeOffsetProvider provider = new FakeOffsetProvider(100, -1);
63+
newJob(provider, 3600).advanceSplitsIfNeed();
64+
Assert.assertTrue("must stop once pending crosses the cap", provider.pendingSplitCount() >= CAP);
65+
Assert.assertEquals("must not overshoot beyond one round past the cap", 6, provider.rounds);
66+
}
67+
68+
@Test
69+
public void testBreaksWhenRoundProducesNothing() throws Exception {
70+
// A round that yields no new split (empty RPC / cursor moved) must break, not spin to deadline.
71+
FakeOffsetProvider provider = new FakeOffsetProvider(0, -1);
72+
newJob(provider, 3600).advanceSplitsIfNeed();
73+
Assert.assertEquals("must break after a no-progress round", 1, provider.rounds);
74+
}
75+
76+
@Test
77+
public void testSkipsWhenAlreadyDone() throws Exception {
78+
// Entry guard: noMoreSplits up front means the loop never runs.
79+
FakeOffsetProvider provider = new FakeOffsetProvider(1, 0);
80+
newJob(provider, 3600).advanceSplitsIfNeed();
81+
Assert.assertEquals("entry guard must skip the loop entirely", 0, provider.rounds);
82+
}
83+
84+
/** Minimal provider whose admission behaviour is fully controllable. */
85+
private static class FakeOffsetProvider implements SourceOffsetProvider {
86+
private int pending;
87+
private int rounds;
88+
private final int stepPerRound; // pending increment per advanceSplits call
89+
private final int roundsUntilDone; // noMoreSplits() flips true after this many rounds (-1 = never)
90+
91+
FakeOffsetProvider(int stepPerRound, int roundsUntilDone) {
92+
this.stepPerRound = stepPerRound;
93+
this.roundsUntilDone = roundsUntilDone;
94+
}
95+
96+
@Override
97+
public void advanceSplits() {
98+
rounds++;
99+
pending += stepPerRound;
100+
}
101+
102+
@Override
103+
public int pendingSplitCount() {
104+
return pending;
105+
}
106+
107+
@Override
108+
public boolean noMoreSplits() {
109+
return roundsUntilDone >= 0 && rounds >= roundsUntilDone;
110+
}
111+
112+
@Override
113+
public String getSourceType() {
114+
return "fake";
115+
}
116+
117+
@Override
118+
public Offset getNextOffset(StreamingJobProperties jobProps, Map<String, String> properties) {
119+
return null;
120+
}
121+
122+
@Override
123+
public String getShowCurrentOffset() {
124+
return "";
125+
}
126+
127+
@Override
128+
public String getShowMaxOffset() {
129+
return "";
130+
}
131+
132+
@Override
133+
public InsertIntoTableCommand rewriteTvfParams(InsertIntoTableCommand originCommand, Offset next, long id) {
134+
return originCommand;
135+
}
136+
137+
@Override
138+
public void updateOffset(Offset offset) {
139+
}
140+
141+
@Override
142+
public void fetchRemoteMeta(Map<String, String> properties) {
143+
}
144+
145+
@Override
146+
public boolean hasMoreDataToConsume() {
147+
return true;
148+
}
149+
150+
@Override
151+
public Offset deserializeOffset(String offset) {
152+
return null;
153+
}
154+
155+
@Override
156+
public Offset deserializeOffsetProperty(String offset) {
157+
return null;
158+
}
159+
}
160+
}

0 commit comments

Comments
 (0)