Skip to content

Commit e9837f9

Browse files
committed
[fix](fe) Preserve single-instance HTTP stream load execution
### What problem does this PR solve? Problem Summary: HTTP Stream Load consumes one BE-owned input stream and completes one StreamLoadContext. Nereids local shuffle could expand its plan to eight instances, invoking the BE completion callback repeatedly and satisfying the same promise more than once. Keep FILE_STREAM TVF scans on one instance during distribution and assert that contract before returning the HTTP plan. Strengthen ordinary and group-commit planning tests with parallel_pipeline_task_num=8. Always-on distribution also exposed missing query IDs in shared FE unit-test contexts and missing worker setup in the cloud table-stream metadata fixture. Initialize identity centrally, retain real distribution on fixture backends, apply the test heartbeat interval to an Env already initialized by static fixtures, and update the UNION regression's obsolete legacy-plan expectations. ### Release note None ### Check List (For Author) - Test: 529 FE unit cases passed across 39 classes (3 existing skipped cases); FE Checkstyle; FE build with OBS/COS packaging excluded. Full BE-backed regression was not run locally. - Behavior changed: Yes. Preserve HTTP Stream Load's single-instance completion contract with Nereids distribution. - Does this need documentation: No
1 parent ca136f9 commit e9837f9

9 files changed

Lines changed: 34 additions & 22 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ protected TFileCompressType getFileCompressType(FileSplit fileSplit) throws User
114114
}
115115

116116
@Override
117-
protected boolean isFileStreamType() {
117+
public boolean isFileStreamType() {
118118
return tableValuedFunction.getTFileType() == TFileType.FILE_STREAM;
119119
}
120120

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleRemoteTableJob.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,16 +70,21 @@ protected Map<DistributedPlanWorker, UninstancedScanSource> multipleMachinesPara
7070
}
7171

7272
/**
73-
* Avoid extra local-shuffle instances for a file TVF row-count scan with one
74-
* actual split assigned to this worker. The scan still feeds the upper COUNT
75-
* aggregation; this shortcut only reduces fragment-instance overhead.
73+
* Keep HTTP stream loads on one instance, and avoid extra local-shuffle
74+
* instances for a file TVF row-count scan with one actual split assigned to this worker.
75+
* The scan still feeds the upper COUNT aggregation; this shortcut only reduces fragment-instance overhead.
7676
* Dynamic split sources and existing external-table COUNT paths retain their
7777
* original parallelism.
7878
*/
7979
@Override
8080
protected int degreeOfParallelism(ScanSource scanSource, int maxParallel,
8181
boolean useLocalShuffleToAddParallel) {
8282
ScanNode scanNode = scanNodes.get(0);
83+
// HTTP stream load owns one input stream and completes its BE load context once.
84+
// Extra local-shuffle instances would invoke that completion callback more than once.
85+
if (scanNode instanceof TVFScanNode && ((TVFScanNode) scanNode).isFileStreamType()) {
86+
return 1;
87+
}
8388
if (scanNode instanceof TVFScanNode && scanNode.getPushDownAggNoGroupingOp() == TPushAggOp.COUNT
8489
&& scanSource instanceof DefaultScanSource) {
8590
ScanRanges scanRanges = ((DefaultScanSource) scanSource).scanNodeToScanRanges.get(scanNode);

‎fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,9 @@ public TPipelineFragmentParams getStreamLoadPlan() throws Exception {
303303
Preconditions.checkState(plans.size() == 1, "HTTP stream load must use one backend");
304304
TPipelineFragmentParamsList fragments = plans.values().iterator().next();
305305
Preconditions.checkState(fragments.getParamsListSize() == 1, "HTTP stream load must use one fragment");
306-
return fragments.getParamsList().get(0);
306+
TPipelineFragmentParams fragment = fragments.getParamsList().get(0);
307+
Preconditions.checkState(fragment.getLocalParamsSize() == 1, "HTTP stream load must use one instance");
308+
return fragment;
307309
}
308310

309311
public boolean isEos() {

‎fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommandTableStreamTest.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import org.apache.doris.nereids.parser.NereidsParser;
4545
import org.apache.doris.nereids.rules.analysis.CloudTableStreamReadStateHook;
4646
import org.apache.doris.nereids.trees.plans.Plan;
47+
import org.apache.doris.nereids.trees.plans.distribute.worker.BackendDistributedPlanWorkerManager;
4748
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableStreamScan;
4849
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
4950
import org.apache.doris.nereids.util.PlanChecker;
@@ -59,7 +60,9 @@
5960

6061
import org.junit.jupiter.api.Assertions;
6162
import org.junit.jupiter.api.Test;
63+
import org.mockito.AdditionalAnswers;
6264
import org.mockito.ArgumentCaptor;
65+
import org.mockito.MockedConstruction;
6366
import org.mockito.MockedStatic;
6467
import org.mockito.Mockito;
6568

@@ -560,6 +563,8 @@ public void testCloudPartitionLimitAppliedDuringInitPlan() throws Exception {
560563
for (Backend backend : previousSystemInfo.getAllClusterBackends(false)) {
561564
cloudSystemInfo.addBackend(backend);
562565
}
566+
BackendDistributedPlanWorkerManager workers =
567+
new BackendDistributedPlanWorkerManager(connectContext, false, false);
563568
MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class);
564569
AbstractInsertExecutor p1Executor = null;
565570
try {
@@ -568,7 +573,11 @@ public void testCloudPartitionLimitAppliedDuringInitPlan() throws Exception {
568573
Config.meta_service_endpoint = "127.0.0.1:20121";
569574
Deencapsulation.setField(Env.getCurrentEnv(), "systemInfo", cloudSystemInfo);
570575
try (MockedStatic<MetaServiceProxy> mockedProxy = Mockito.mockStatic(MetaServiceProxy.class);
571-
MockedStatic<MasterTsoProvider> mockedTso = Mockito.mockStatic(MasterTsoProvider.class)) {
576+
MockedStatic<MasterTsoProvider> mockedTso = Mockito.mockStatic(MasterTsoProvider.class);
577+
MockedConstruction<BackendDistributedPlanWorkerManager> mockedWorkers = Mockito.mockConstruction(
578+
BackendDistributedPlanWorkerManager.class,
579+
Mockito.withSettings().defaultAnswer(AdditionalAnswers.delegatesTo(workers)))) {
580+
// The fixture mocks cloud metadata while distributing on the local test backends.
572581
// The read-state fixture uses offsets 100..130; keep the TTL reference in the same clock.
573582
mockedTso.when(() -> MasterTsoProvider.getCurrentTso(connectContext)).thenReturn(130L);
574583
mockedProxy.when(MetaServiceProxy::getInstance).thenReturn(proxy);

‎fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
package org.apache.doris.nereids.util;
1919

2020
import org.apache.doris.analysis.UserIdentity;
21-
import org.apache.doris.catalog.Env;
2221
import org.apache.doris.nereids.CascadesContext;
2322
import org.apache.doris.nereids.NereidsPlanner;
2423
import org.apache.doris.nereids.StatementContext;
@@ -28,6 +27,7 @@
2827
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
2928
import org.apache.doris.qe.ConnectContext;
3029
import org.apache.doris.qe.OriginStatement;
30+
import org.apache.doris.utframe.UtFrameUtils;
3131

3232
import org.apache.commons.lang3.StringUtils;
3333

@@ -106,12 +106,7 @@ public static LogicalPlan analyze(String sql) {
106106
*/
107107
public static ConnectContext createCtx(UserIdentity user, String host) {
108108
try {
109-
ConnectContext ctx = new ConnectContext();
110-
ctx.setCurrentUserIdentity(user);
111-
ctx.setRemoteIP(host);
112-
ctx.setEnv(Env.getCurrentEnv());
113-
ctx.setThreadLocalInfo();
114-
return ctx;
109+
return UtFrameUtils.createDefaultCtx(user, host);
115110
} catch (Throwable t) {
116111
throw new IllegalStateException("can not create test connect context", t);
117112
}

‎fe/fe-core/src/test/java/org/apache/doris/qe/CoordinatorTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,7 @@ public void testHttpStreamLoadPlanUsesNereidsTransport(String groupCommitMode) t
236236
context.setQueryId(new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()));
237237
context.setBackendId(Env.getCurrentSystemInfo().getAllBackendsByAllCluster().keySet().iterator().next());
238238
context.getSessionVariable().groupCommit = groupCommitMode;
239+
context.getSessionVariable().setPipelineTaskNum("8");
239240
context.setThreadLocalInfo();
240241
try {
241242
StmtExecutor executor = new StmtExecutor(context,
@@ -249,7 +250,7 @@ public void testHttpStreamLoadPlanUsesNereidsTransport(String groupCommitMode) t
249250
Assertions.assertEquals(groupCommitMode.isEmpty()
250251
? TDataSinkType.OLAP_TABLE_SINK : TDataSinkType.GROUP_COMMIT_BLOCK_SINK,
251252
params.getFragment().getOutputSink().getType());
252-
Assertions.assertFalse(params.getLocalParams().isEmpty());
253+
Assertions.assertEquals(1, params.getLocalParamsSize());
253254
} finally {
254255
connectContext.setThreadLocalInfo();
255256
}

‎fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,8 @@ public final void beforeAll() throws Exception {
166166
Config.heartbeat_interval_second = 1;
167167
beforeCreatingConnectContext();
168168
connectContext = createDefaultCtx();
169+
// Static test fixtures may have initialized Env before the heartbeat interval was set.
170+
Env.getCurrentHeartbeatMgr().setInterval(Config.heartbeat_interval_second * 1000L);
169171
connectContext.getSessionVariable().feDebug = true;
170172
beforeCluster();
171173
createDorisCluster();
@@ -296,11 +298,7 @@ public StatementBase analyzeAndGetStmtByNereids(String sql, ConnectContext ctx)
296298
}
297299

298300
public static ConnectContext createCtx(UserIdentity user, String host) throws IOException {
299-
ConnectContext ctx = new ConnectContext();
300-
ctx.setCurrentUserIdentity(user);
301-
ctx.setRemoteIP(host);
302-
ctx.setEnv(Env.getCurrentEnv());
303-
ctx.setThreadLocalInfo();
301+
ConnectContext ctx = UtFrameUtils.createDefaultCtx(user, host);
304302
ctx.setStatementContext(new StatementContext());
305303
return ctx;
306304
}

‎fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.apache.doris.qe.StmtExecutor;
3737
import org.apache.doris.system.Backend;
3838
import org.apache.doris.thrift.TNetworkAddress;
39+
import org.apache.doris.thrift.TUniqueId;
3940
import org.apache.doris.utframe.MockedBackendFactory.DefaultBeThriftServiceImpl;
4041
import org.apache.doris.utframe.MockedBackendFactory.DefaultHeartbeatServiceImpl;
4142
import org.apache.doris.utframe.MockedBackendFactory.DefaultPBackendServiceImpl;
@@ -70,6 +71,8 @@ public class UtFrameUtils {
7071
// Help to create a mocked ConnectContext.
7172
public static ConnectContext createDefaultCtx(UserIdentity userIdentity, String remoteIp) throws IOException {
7273
ConnectContext ctx = new ConnectContext();
74+
UUID queryId = UUID.randomUUID();
75+
ctx.setQueryId(new TUniqueId(queryId.getMostSignificantBits(), queryId.getLeastSignificantBits()));
7376
ctx.setCurrentUserIdentity(userIdentity);
7477
ctx.setRemoteIP(remoteIp);
7578
ctx.setEnv(Env.getCurrentEnv());

‎regression-test/suites/query_p0/union/test_union_instance.groovy‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@ suite("test_union_instance") {
6363
6464
set parallel_pipeline_task_num=1;
6565
set disable_nereids_rules='PRUNE_EMPTY_PARTITION';
66-
set enable_nereids_distribute_planner=false;
6766
"""
6867
explain {
6968
sql """
@@ -82,8 +81,8 @@ suite("test_union_instance") {
8281
"""
8382
contains "3:VNESTED LOOP JOIN"
8483
contains "4:VEXCHANGE"
85-
contains "8:VEXCHANGE"
86-
contains "6:VEXCHANGE"
84+
contains "10:VEXCHANGE"
85+
contains "join op: INNER JOIN(BUCKET_SHUFFLE)"
8786
contains "11:VUNION"
8887
}
8988
}

0 commit comments

Comments
 (0)