Skip to content

Commit 1037121

Browse files
HappenLeepull[bot]
authored andcommitted
[pipeline](exec) Support shared scan in jdbc and odbc scan node (apache#22826)
Support shared scan in jdbc and odbc scan node to improve exec performance
1 parent fe13b6e commit 1037121

5 files changed

Lines changed: 14 additions & 8 deletions

File tree

‎be/src/exec/exec_node.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -504,6 +504,8 @@ void ExecNode::collect_scan_nodes(vector<ExecNode*>* nodes) {
504504
collect_nodes(TPlanNodeType::DATA_GEN_SCAN_NODE, nodes);
505505
collect_nodes(TPlanNodeType::FILE_SCAN_NODE, nodes);
506506
collect_nodes(TPlanNodeType::META_SCAN_NODE, nodes);
507+
collect_nodes(TPlanNodeType::JDBC_SCAN_NODE, nodes);
508+
collect_nodes(TPlanNodeType::ODBC_SCAN_NODE, nodes);
507509
}
508510

509511
void ExecNode::init_runtime_profile(const std::string& name) {

‎be/src/pipeline/pipeline_fragment_context.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@
100100
#include "vec/exec/join/vhash_join_node.h"
101101
#include "vec/exec/scan/new_es_scan_node.h"
102102
#include "vec/exec/scan/new_file_scan_node.h"
103+
#include "vec/exec/scan/new_jdbc_scan_node.h"
103104
#include "vec/exec/scan/new_odbc_scan_node.h"
104105
#include "vec/exec/scan/new_olap_scan_node.h"
105106
#include "vec/exec/scan/vmeta_scan_node.h"

‎fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,6 @@
3838
import org.apache.doris.common.AnalysisException;
3939
import org.apache.doris.common.Pair;
4040
import org.apache.doris.common.UserException;
41-
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
42-
import org.apache.doris.planner.external.odbc.OdbcScanNode;
4341
import org.apache.doris.qe.ConnectContext;
4442
import org.apache.doris.qe.SessionVariable;
4543
import org.apache.doris.thrift.TPartitionType;
@@ -281,12 +279,8 @@ private PlanFragment createMergeFragment(PlanFragment inputFragment)
281279
* TODO: hbase scans are range-partitioned on the row key
282280
*/
283281
private PlanFragment createScanFragment(PlanNode node) throws UserException {
284-
if (node instanceof MysqlScanNode || node instanceof OdbcScanNode || node instanceof JdbcScanNode) {
282+
if (node instanceof MysqlScanNode) {
285283
return new PlanFragment(ctx.getNextFragmentId(), node, DataPartition.UNPARTITIONED);
286-
} else if (node instanceof SchemaScanNode) {
287-
return new PlanFragment(ctx.getNextFragmentId(), node, DataPartition.RANDOM);
288-
} else if (node instanceof DataGenScanNode) {
289-
return new PlanFragment(ctx.getNextFragmentId(), node, DataPartition.RANDOM);
290284
} else if (node instanceof OlapScanNode) {
291285
// olap scan node
292286
OlapScanNode olapScanNode = (OlapScanNode) node;

‎fe/fe-core/src/main/java/org/apache/doris/planner/external/jdbc/JdbcScanNode.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
3939
import org.apache.doris.planner.PlanNodeId;
4040
import org.apache.doris.planner.external.ExternalScanNode;
41+
import org.apache.doris.qe.ConnectContext;
4142
import org.apache.doris.statistics.StatisticalType;
4243
import org.apache.doris.statistics.StatsRecursiveDerive;
4344
import org.apache.doris.statistics.query.StatsDelta;
@@ -272,7 +273,8 @@ protected String debugString() {
272273

273274
@Override
274275
public int getNumInstances() {
275-
return 1;
276+
return ConnectContext.get().getSessionVariable().getEnablePipelineEngine()
277+
? ConnectContext.get().getSessionVariable().getParallelExecInstanceNum() : 1;
276278
}
277279

278280
@Override

‎fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.apache.doris.planner.PlanNodeId;
3333
import org.apache.doris.planner.external.ExternalScanNode;
3434
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
35+
import org.apache.doris.qe.ConnectContext;
3536
import org.apache.doris.statistics.StatisticalType;
3637
import org.apache.doris.statistics.StatsRecursiveDerive;
3738
import org.apache.doris.statistics.query.StatsDelta;
@@ -213,4 +214,10 @@ public StatsDelta genStatsDelta() throws AnalysisException {
213214
Env.getCurrentEnv().getCurrentCatalog().getDbOrAnalysisException(tbl.getQualifiedDbName()).getId(),
214215
tbl.getId(), -1L);
215216
}
217+
218+
@Override
219+
public int getNumInstances() {
220+
return ConnectContext.get().getSessionVariable().getEnablePipelineEngine()
221+
? ConnectContext.get().getSessionVariable().getParallelExecInstanceNum() : 1;
222+
}
216223
}

0 commit comments

Comments
 (0)