Skip to content

Commit 950ebd1

Browse files
jieguangzhoujieguangzhou
authored
Merge 40c95c5 into 3630b00 
2 parents 3630b00 + 40c95c5 commit 950ebd1

File tree

46 files changed

+2080
-7
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+2080
-7
lines changed

‎dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
3636
import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper;
3737
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
38+
import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
3839
import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
3940
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
4041
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
@@ -186,9 +187,10 @@ public Result<Object> updateDataSource(int id, User loginUser, BaseDataSourcePar
186187
return result;
187188
}
188189
// check password，if the password is not updated, set to the old password.
189-
BaseConnectionParam connectionParam =
190-
(BaseConnectionParam) DataSourceUtils.buildConnectionParams(dataSourceParam);
190+
ConnectionParam connectionParam = DataSourceUtils.buildConnectionParams(dataSourceParam);
191+
191192
String password = connectionParam.getPassword();
193+
192194
if (StringUtils.isBlank(password)) {
193195
String oldConnectionParams = dataSource.getConnectionParams();
194196
ObjectNode oldParams = JSONUtils.parseObject(oldConnectionParams);
@@ -383,6 +385,15 @@ public Result<Object> verifyDataSourceName(String name) {
383385
@Override
384386
public Result<Object> checkConnection(DbType type, ConnectionParam connectionParam) {
385387
Result<Object> result = new Result<>();
388+
if (type == DbType.SSH) {
389+
DataSourceProcessor sshDataSourceProcessor = DataSourceUtils.getDatasourceProcessor(type);
390+
if (sshDataSourceProcessor.testConnection(connectionParam)) {
391+
putMsg(result, Status.SUCCESS);
392+
} else {
393+
putMsg(result, Status.CONNECT_DATASOURCE_FAILURE);
394+
}
395+
return result;
396+
}
386397
try (Connection connection = DataSourceClientProvider.getInstance().getConnection(type, connectionParam)) {
387398
if (connection == null) {
388399
log.error("Connection test to {} datasource failed, connectionParam:{}.", type.getDescp(),

‎dolphinscheduler-api/src/main/resources/task-type-config.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ task:
2929
- 'DINKY'
3030
- 'FLINK_STREAM'
3131
- 'HIVECLI'
32+
- 'REMOTESHELL'
3233
cloud:
3334
- 'EMR'
3435
- 'K8S'

‎dolphinscheduler-bom/pom.xml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@
106106
<azure-resourcemanager.version>2.21.0</azure-resourcemanager.version>
107107
<azure-resourcemanager-datafactory.version>1.0.0-beta.19</azure-resourcemanager-datafactory.version>
108108
<google-cloud-storage.version>2.18.0</google-cloud-storage.version>
109+
<sshd.version>2.8.0</sshd.version>
109110
</properties>
110111

111112
<dependencyManagement>
@@ -738,6 +739,17 @@
738739
</exclusions>
739740
</dependency>
740741

742+
<dependency>
743+
<groupId>org.apache.sshd</groupId>
744+
<artifactId>sshd-sftp</artifactId>
745+
<version>${sshd.version}</version>
746+
</dependency>
747+
<dependency>
748+
<groupId>org.apache.sshd</groupId>
749+
<artifactId>sshd-scp</artifactId>
750+
<version>${sshd.version}</version>
751+
</dependency>
752+
741753
<dependency>
742754
<groupId>org.apache.spark</groupId>
743755
<artifactId>spark-sql_2.12</artifactId>

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,5 +107,10 @@
107107
<artifactId>dolphinscheduler-datasource-dameng</artifactId>
108108
<version>${project.version}</version>
109109
</dependency>
110+
<dependency>
111+
<groupId>org.apache.dolphinscheduler</groupId>
112+
<artifactId>dolphinscheduler-datasource-ssh</artifactId>
113+
<version>${project.version}</version>
114+
</dependency>
110115
</dependencies>
111116
</project>

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DataSourceProcessor.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,16 @@ public interface DataSourceProcessor {
9595
*/
9696
Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException, IOException;
9797

98+
/**
99+
* test connection, use for not jdbc datasource
100+
*
101+
* @param connectionParam connectionParam
102+
* @return true if connection is valid
103+
*/
104+
default boolean testConnection(ConnectionParam connectionParam) {
105+
return false;
106+
}
107+
98108
/**
99109
* @return {@link DbType}
100110
*/

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/pom.xml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<modelVersion>4.0.0</modelVersion>
21+
<parent>
22+
<groupId>org.apache.dolphinscheduler</groupId>
23+
<artifactId>dolphinscheduler-datasource-plugin</artifactId>
24+
<version>dev-SNAPSHOT</version>
25+
</parent>
26+
27+
<artifactId>dolphinscheduler-datasource-ssh</artifactId>
28+
<packaging>jar</packaging>
29+
<name>${project.artifactId}</name>
30+
31+
<dependencies>
32+
<dependency>
33+
<groupId>org.apache.dolphinscheduler</groupId>
34+
<artifactId>dolphinscheduler-spi</artifactId>
35+
<scope>provided</scope>
36+
</dependency>
37+
<dependency>
38+
<groupId>org.apache.dolphinscheduler</groupId>
39+
<artifactId>dolphinscheduler-datasource-api</artifactId>
40+
<version>${project.version}</version>
41+
</dependency>
42+
<dependency>
43+
<groupId>org.apache.sshd</groupId>
44+
<artifactId>sshd-scp</artifactId>
45+
</dependency>
46+
</dependencies>
47+
</project>

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHDataSourceChannel.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.plugin.datasource.ssh;
19+
20+
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
21+
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
22+
import org.apache.dolphinscheduler.spi.datasource.DataSourceClient;
23+
import org.apache.dolphinscheduler.spi.enums.DbType;
24+
25+
public class SSHDataSourceChannel implements DataSourceChannel {
26+
27+
@Override
28+
public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) {
29+
return new SSHDataSourceClient(baseConnectionParam, dbType);
30+
}
31+
32+
}

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHDataSourceChannelFactory.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.plugin.datasource.ssh;
19+
20+
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
21+
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory;
22+
23+
import com.google.auto.service.AutoService;
24+
25+
@AutoService(DataSourceChannelFactory.class)
26+
public class SSHDataSourceChannelFactory implements DataSourceChannelFactory {
27+
28+
@Override
29+
public String getName() {
30+
return "ssh";
31+
}
32+
33+
@Override
34+
public DataSourceChannel create() {
35+
return new SSHDataSourceChannel();
36+
}
37+
}

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHDataSourceClient.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.plugin.datasource.ssh;
19+
20+
import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient;
21+
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
22+
import org.apache.dolphinscheduler.spi.enums.DbType;
23+
24+
public class SSHDataSourceClient extends CommonDataSourceClient {
25+
26+
public SSHDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) {
27+
super(baseConnectionParam, dbType);
28+
}
29+
30+
}

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-ssh/src/main/java/org/apache/dolphinscheduler/plugin/datasource/ssh/SSHUtils.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.plugin.datasource.ssh;
19+
20+
import org.apache.dolphinscheduler.plugin.datasource.ssh.param.SSHConnectionParam;
21+
22+
import org.apache.commons.lang3.StringUtils;
23+
import org.apache.sshd.client.SshClient;
24+
import org.apache.sshd.client.session.ClientSession;
25+
import org.apache.sshd.common.config.keys.loader.KeyPairResourceLoader;
26+
import org.apache.sshd.common.util.security.SecurityUtils;
27+
28+
import java.security.KeyPair;
29+
import java.util.Collection;
30+
31+
public class SSHUtils {
32+
33+
private SSHUtils() {
34+
throw new IllegalStateException("Utility class");
35+
}
36+
37+
public static ClientSession getSession(SshClient client, SSHConnectionParam connectionParam) throws Exception {
38+
ClientSession session;
39+
session = client.connect(connectionParam.getUser(), connectionParam.getHost(), connectionParam.getPort())
40+
.verify(5000).getSession();
41+
// add password identity
42+
String password = connectionParam.getPassword();
43+
if (StringUtils.isNotEmpty(password)) {
44+
session.addPasswordIdentity(password);
45+
}
46+
47+
// add public key identity
48+
String publicKey = connectionParam.getPublicKey();
49+
if (StringUtils.isNotEmpty(publicKey)) {
50+
try {
51+
KeyPairResourceLoader loader = SecurityUtils.getKeyPairResourceParser();
52+
Collection<KeyPair> keyPairCollection = loader.loadKeyPairs(null, null, null, publicKey);
53+
for (KeyPair keyPair : keyPairCollection) {
54+
session.addPublicKeyIdentity(keyPair);
55+
}
56+
} catch (Exception e) {
57+
throw new Exception("Failed to add public key identity", e);
58+
}
59+
}
60+
return session;
61+
}
62+
}

0 commit comments

Comments
 (0)