Skip to content

Commit 18d176c

Browse files
zinwzinw
authored
feat: Introduce new datasource vertica (#14232)
Co-authored-by: Zinway Liu <[email protected]>
1 parent 7673312 commit 18d176c

File tree

31 files changed

+1042
-2
lines changed

31 files changed

+1042
-2
lines changed

‎docs/configs/docsdev.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,10 @@ export default {
254254
title: 'File Parameter',
255255
link: '/en-us/docs/dev/user_doc/guide/parameter/file-parameter.html',
256256
},
257+
{
258+
title: 'Vertica',
259+
link: '/en-us/docs/dev/user_doc/guide/datasource/vertica.html',
260+
},
257261
],
258262
},
259263
{
@@ -930,6 +934,10 @@ export default {
930934
title: 'SSH',
931935
link: '/zh-cn/docs/dev/user_doc/guide/task/ssh.html',
932936
},
937+
{
938+
title: 'Vertica',
939+
link: '/zh-cn/docs/dev/user_doc/guide/datasource/vertica.html',
940+
},
933941
],
934942
},
935943
{

‎docs/docs/en/guide/datasource/vertica.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Vertica
2+
3+
![vertica](../../../../img/new_ui/dev/datasource/vertica.png)
4+
5+
## Datasource Parameters
6+
7+
| **Parameter** | **Description** |
8+
|----------------------------|------------------------------------------------------------|
9+
| Datasource | Select VERTICA. |
10+
| Datasource Name | Enter the name of the DataSource. |
11+
| Description | Enter a description of the DataSource. |
12+
| IP/Host Name | Enter the Vertica service IP. |
13+
| Port | Enter the Vertica service port. |
14+
| Username | Set the username for Vertica connection. |
15+
| Password | Set the password for Vertica connection. |
16+
| Database name | Enter the database name of the Vertica connection. |
17+
| Jdbc connection parameters | Parameter settings for Vertica connection, in JSON format. |
18+
19+
## Native Supported
20+
21+
Yes, could use this datasource by default.

‎docs/docs/zh/guide/datasource/vertica.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# VERTICA 数据源
2+
3+
![vertica](../../../../img/new_ui/dev/datasource/vertica.png)
4+
5+
| **参数名称** | **参数描述** |
6+
|-----------|--------------------------------|
7+
| 数据源 | 选择 VERTICA |
8+
| 数据源名称 | 输入数据源的名称 |
9+
| 描述 | 输入数据源的描述 |
10+
| IP 主机名 | 输入连接 VERTICA 的 IP |
11+
| 端口 | 输入连接 VERTICA 的端口 |
12+
| 用户名 | 设置连接 VERTICA 的用户名 |
13+
| 密码 | 设置连接 VERTICA 的密码 |
14+
| 数据库名 | 输入连接 VERTICA 的数据库名称 |
15+
| JDBC 连接参数 | 用于 VERTICA 连接的参数设置，以 JSON 形式填写 |
16+
17+
## 是否原生支持
18+
19+
是，数据源不需要任务附加操作即可使用。

‎docs/img/new_ui/dev/datasource/vertica.png‎

236 KB
Loading

‎dolphinscheduler-bom/pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
<cron-utils.version>9.1.6</cron-utils.version>
5858
<h2.version>2.1.210</h2.version>
5959
<mysql-connector.version>8.0.16</mysql-connector.version>
60+
<vertica-jdbc.version>12.0.4-0</vertica-jdbc.version>
6061
<oracle-jdbc.version>21.5.0.0</oracle-jdbc.version>
6162
<dameng-jdbc.version>8.1.2.79</dameng-jdbc.version>
6263
<slf4j.version>1.7.36</slf4j.version>
@@ -398,6 +399,12 @@
398399
<version>${dameng-jdbc.version}</version>
399400
</dependency>
400401

402+
<dependency>
403+
<groupId>com.vertica.jdbc</groupId>
404+
<artifactId>vertica-jdbc</artifactId>
405+
<version>${vertica-jdbc.version}</version>
406+
</dependency>
407+
401408
<dependency>
402409
<groupId>com.h2database</groupId>
403410
<artifactId>h2</artifactId>

‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ public class DataSourceConstants {
4141
public static final String ORG_APACHE_KYUUBI_JDBC_DRIVER = "org.apache.kyuubi.jdbc.KyuubiHiveDriver";
4242
public static final String COM_OCEANBASE_JDBC_DRIVER = "com.oceanbase.jdbc.Driver";
4343
public static final String NET_SNOWFLAKE_JDBC_DRIVER = "net.snowflake.client.jdbc.SnowflakeDriver";
44+
public static final String COM_VERTICA_JDBC_DRIVER = "com.vertica.jdbc.Driver";
4445
/**
4546
* validation Query
4647
*/
@@ -60,6 +61,7 @@ public class DataSourceConstants {
6061
public static final String SNOWFLAKE_VALIDATION_QUERY = "select 1";
6162

6263
public static final String KYUUBI_VALIDATION_QUERY = "select 1";
64+
public static final String VERTICA_VALIDATION_QUERY = "select 1";
6365

6466
/**
6567
* jdbc url
@@ -82,6 +84,7 @@ public class DataSourceConstants {
8284
public static final String JDBC_DAMENG = "jdbc:dm://";
8385
public static final String JDBC_OCEANBASE = "jdbc:oceanbase://";
8486
public static final String JDBC_SNOWFLAKE = "jdbc:snowflake://";
87+
public static final String JDBC_VERTICA = "jdbc:vertica://";
8588

8689
/**
8790
* database type

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,5 +127,10 @@
127127
<artifactId>dolphinscheduler-datasource-kyuubi</artifactId>
128128
<version>${project.version}</version>
129129
</dependency>
130+
<dependency>
131+
<groupId>org.apache.dolphinscheduler</groupId>
132+
<artifactId>dolphinscheduler-datasource-vertica</artifactId>
133+
<version>${project.version}</version>
134+
</dependency>
130135
</dependencies>
131136
</project>

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTO.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,11 @@
1717

1818
package org.apache.dolphinscheduler.plugin.datasource.api.datasource;
1919

20+
import org.apache.dolphinscheduler.common.constants.Constants;
2021
import org.apache.dolphinscheduler.spi.enums.DbType;
2122

2223
import java.io.Serializable;
24+
import java.util.ArrayList;
2325
import java.util.Map;
2426

2527
/**
@@ -89,6 +91,39 @@ public void setPort(Integer port) {
8991
this.port = port;
9092
}
9193

94+
/**
95+
* extract the host and port from the address,
96+
* then set it
97+
* @param address address like 'jdbc:mysql://host:port' or 'jdbc:hive2://zk1:port,zk2:port,zk3:port'
98+
*/
99+
public void setHostAndPortByAddress(String address) {
100+
if (address == null) {
101+
throw new IllegalArgumentException("address is null.");
102+
}
103+
address = address.trim();
104+
105+
int doubleSlashIndex = address.indexOf(Constants.DOUBLE_SLASH);
106+
// trim address like 'jdbc:mysql://host:port/xxx' ends with '/xxx'
107+
int slashIndex = address.indexOf(Constants.SLASH, doubleSlashIndex + 2);
108+
String hostPortString = slashIndex == -1 ? address.substring(doubleSlashIndex + 2)
109+
: address.substring(doubleSlashIndex + 2, slashIndex);
110+
111+
ArrayList<String> hosts = new ArrayList<>();
112+
String portString = null;
113+
for (String hostPort : hostPortString.split(Constants.COMMA)) {
114+
String[] parts = hostPort.split(Constants.COLON);
115+
hosts.add(parts[0]);
116+
if (portString == null && parts.length > 1)
117+
portString = parts[1];
118+
}
119+
if (hosts.size() == 0 || portString == null) {
120+
throw new IllegalArgumentException(String.format("host:port '%s' illegal.", hostPortString));
121+
}
122+
123+
this.host = String.join(Constants.COMMA, hosts);
124+
this.port = Integer.parseInt(portString);
125+
}
126+
92127
public String getDatabase() {
93128
return database;
94129
}

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/BaseDataSourceParamDTOTest.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.plugin.datasource.api.datasource;
19+
20+
import static org.junit.jupiter.api.Assertions.assertEquals;
21+
import static org.junit.jupiter.api.Assertions.assertThrows;
22+
23+
import org.junit.jupiter.api.BeforeEach;
24+
import org.junit.jupiter.api.Test;
25+
26+
/**
27+
* Use MySQLDataSourceParamDTO extends BaseDataSourceParamDTO for test.
28+
*/
29+
public class BaseDataSourceParamDTOTest {
30+
31+
private MySQLDataSourceParamDTO mockDataSourceParamDTO;
32+
33+
@BeforeEach
34+
public void setUp() {
35+
mockDataSourceParamDTO = new MySQLDataSourceParamDTO();
36+
}
37+
38+
@Test
39+
public void setHostAndPortByAddressTest1() {
40+
mockDataSourceParamDTO.setHostAndPortByAddress("jdbc:mysql://1.2.3.4:3306");
41+
assertEquals("1.2.3.4", mockDataSourceParamDTO.getHost());
42+
assertEquals(3306, mockDataSourceParamDTO.getPort());
43+
}
44+
45+
@Test
46+
public void setHostAndPortByAddressTest2() {
47+
mockDataSourceParamDTO.setHostAndPortByAddress("jdbc:mysql://1.2.3.4:3306/database");
48+
assertEquals("1.2.3.4", mockDataSourceParamDTO.getHost());
49+
assertEquals(3306, mockDataSourceParamDTO.getPort());
50+
}
51+
52+
@Test
53+
public void setHostAndPortByAddressTest3() {
54+
mockDataSourceParamDTO.setHostAndPortByAddress("jdbc:mysql://h1,h2,h3:3306");
55+
assertEquals("h1,h2,h3", mockDataSourceParamDTO.getHost());
56+
assertEquals(3306, mockDataSourceParamDTO.getPort());
57+
}
58+
59+
@Test
60+
public void setHostAndPortByAddressTest4() {
61+
mockDataSourceParamDTO.setHostAndPortByAddress("jdbc:mysql://h1:3306,h2:3306,h3:3306");
62+
assertEquals("h1,h2,h3", mockDataSourceParamDTO.getHost());
63+
assertEquals(3306, mockDataSourceParamDTO.getPort());
64+
}
65+
66+
@Test
67+
public void setHostAndPortByAddressTest5() {
68+
Throwable exception = assertThrows(IllegalArgumentException.class,
69+
() -> mockDataSourceParamDTO.setHostAndPortByAddress("jdbc:mysql://h1"));
70+
assertEquals("host:port 'h1' illegal.", exception.getMessage());
71+
}
72+
73+
@Test
74+
public void setHostAndPortByAddressTest6() {
75+
Throwable exception = assertThrows(NumberFormatException.class,
76+
() -> mockDataSourceParamDTO.setHostAndPortByAddress("jdbc:mysql://h1:port"));
77+
assertEquals("For input string: \"port\"", exception.getMessage());
78+
}
79+
80+
}

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/MySQLDataSourceParamDTO.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.plugin.datasource.api.datasource;
19+
20+
import org.apache.dolphinscheduler.spi.enums.DbType;
21+
22+
/**
23+
* Used for test in BaseDataSourceParamDTOTest.
24+
*/
25+
public class MySQLDataSourceParamDTO extends BaseDataSourceParamDTO {
26+
27+
@Override
28+
public String toString() {
29+
return "MySQLDataSourceParamDTO{"
30+
+ "name='" + name + '\''
31+
+ ", note='" + note + '\''
32+
+ ", host='" + host + '\''
33+
+ ", port=" + port
34+
+ ", database='" + database + '\''
35+
+ ", userName='" + userName + '\''
36+
+ ", password='" + password + '\''
37+
+ ", other='" + other + '\''
38+
+ '}';
39+
}
40+
41+
@Override
42+
public DbType getType() {
43+
return DbType.MYSQL;
44+
}
45+
}

0 commit comments

Comments
 (0)