Skip to content

Commit e27bb5c

Browse files
ruanwenjunruanwenjun
committed
Use Druid to split sql
1 parent c3ffb13 commit e27bb5c

File tree

29 files changed

+405
-107
lines changed

29 files changed

+405
-107
lines changed

‎dolphinscheduler-bom/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
<java-websocket.version>1.5.1</java-websocket.version>
3535
<mybatis-plus.version>3.5.2</mybatis-plus.version>
3636
<quartz.version>2.3.2</quartz.version>
37-
<druid.version>1.2.4</druid.version>
37+
<druid.version>1.2.20</druid.version>
3838
<curator-test.version>2.12.0</curator-test.version>
3939
<jetcd.version>0.5.11</jetcd.version>
4040
<jetcd.test.version>0.7.1</jetcd.test.version>

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,5 +140,11 @@
140140
<groupId>com.zaxxer</groupId>
141141
<artifactId>HikariCP</artifactId>
142142
</dependency>
143+
144+
<dependency>
145+
<groupId>com.alibaba</groupId>
146+
<artifactId>druid</artifactId>
147+
</dependency>
148+
143149
</dependencies>
144150
</project>

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessor.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,14 @@
2929
import java.sql.Connection;
3030
import java.text.MessageFormat;
3131
import java.util.Collections;
32+
import java.util.List;
3233
import java.util.Map;
3334
import java.util.Set;
3435
import java.util.regex.Pattern;
3536

3637
import lombok.extern.slf4j.Slf4j;
3738

39+
import com.alibaba.druid.sql.parser.SQLParserUtils;
3840
import com.fasterxml.jackson.core.type.TypeReference;
3941
import com.google.common.collect.Sets;
4042

@@ -128,4 +130,9 @@ public boolean checkDataSourceConnectivity(ConnectionParam connectionParam) {
128130
return false;
129131
}
130132
}
133+
134+
@Override
135+
public List<String> splitAndRemoveComment(String sql) {
136+
return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.other);
137+
}
131138
}

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/DataSourceProcessor.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.io.IOException;
2424
import java.sql.Connection;
2525
import java.sql.SQLException;
26+
import java.util.List;
2627

2728
public interface DataSourceProcessor {
2829

@@ -113,4 +114,6 @@ public interface DataSourceProcessor {
113114
* get datasource processor
114115
*/
115116
DataSourceProcessor create();
117+
118+
List<String> splitAndRemoveComment(String sql);
116119
}

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/test/java/org/apache/dolphinscheduler/plugin/datasource/api/datasource/AbstractDataSourceProcessorTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ public void shouldNotIncludeMaliciousParams() {
5858
doThrow(new IllegalArgumentException()).when(mockDataSourceProcessor).checkOther(other);
5959
}
6060

61-
@org.junit.Test
61+
@Test
6262
public void transformOtherParamToMap() {
6363
AbstractDataSourceProcessor abstractDataSourceProcessor = new AbstractDataSourceProcessor() {
6464

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-clickhouse/src/main/java/org/apache/dolphinscheduler/plugin/datasource/clickhouse/param/ClickHouseDataSourceProcessor.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import java.util.List;
3737
import java.util.Map;
3838

39+
import com.alibaba.druid.sql.parser.SQLParserUtils;
3940
import com.google.auto.service.AutoService;
4041

4142
@AutoService(DataSourceProcessor.class)
@@ -126,6 +127,11 @@ public DataSourceProcessor create() {
126127
return new ClickHouseDataSourceProcessor();
127128
}
128129

130+
@Override
131+
public List<String> splitAndRemoveComment(String sql) {
132+
return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.clickhouse);
133+
}
134+
129135
private String transformOther(Map<String, String> otherMap) {
130136
if (MapUtils.isEmpty(otherMap)) {
131137
return null;

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-dameng/src/main/java/org/apache/dolphinscheduler/plugin/datasource/dameng/param/DamengDataSourceProcessor.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import java.util.List;
3939
import java.util.Map;
4040

41+
import com.alibaba.druid.sql.parser.SQLParserUtils;
4142
import com.google.auto.service.AutoService;
4243

4344
@AutoService(DataSourceProcessor.class)
@@ -132,6 +133,11 @@ public DataSourceProcessor create() {
132133
return new DamengDataSourceProcessor();
133134
}
134135

136+
@Override
137+
public List<String> splitAndRemoveComment(String sql) {
138+
return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.dm);
139+
}
140+
135141
private String transformOther(Map<String, String> paramMap) {
136142
if (MapUtils.isEmpty(paramMap)) {
137143
return null;

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-db2/src/main/java/org/apache/dolphinscheduler/plugin/datasource/db2/param/Db2DataSourceProcessor.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import java.util.List;
3838
import java.util.Map;
3939

40+
import com.alibaba.druid.sql.parser.SQLParserUtils;
4041
import com.google.auto.service.AutoService;
4142

4243
@AutoService(DataSourceProcessor.class)
@@ -126,6 +127,11 @@ public String getValidationQuery() {
126127
return DataSourceConstants.DB2_VALIDATION_QUERY;
127128
}
128129

130+
@Override
131+
public List<String> splitAndRemoveComment(String sql) {
132+
return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.db2);
133+
}
134+
129135
private String transformOther(Map<String, String> otherMap) {
130136
if (MapUtils.isEmpty(otherMap)) {
131137
return null;

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/test/java/org/apache/dolphinscheduler/plugin/doris/param/DorisDataSourceProcessorTest.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.dolphinscheduler.spi.enums.DbType;
2323

2424
import java.util.HashMap;
25+
import java.util.List;
2526
import java.util.Map;
2627

2728
import org.junit.jupiter.api.Assertions;
@@ -122,4 +123,18 @@ public void testGetDatasourceUniqueId() {
122123
dorisDatasourceProcessor.getDatasourceUniqueId(dorisConnectionParam, DbType.DORIS));
123124
}
124125
}
126+
127+
@Test
128+
public void splitAndRemoveComment() {
129+
String sql =
130+
"set enable_unique_key_partial_update = true;\r\n\r\n" +
131+
"insert into demo.table\r\n(age,name)\r\nselect 1, 'tom';\r\n\r\n" +
132+
"set enable_unique_key_partial_update = false;\r\n\r\n\r\n";
133+
List<String> sqls = dorisDatasourceProcessor.splitAndRemoveComment(sql);
134+
Assertions.assertEquals(3, sqls.size());
135+
Assertions.assertEquals("set enable_unique_key_partial_update = true", sqls.get(0));
136+
Assertions.assertEquals("insert into demo.table\r\n(age,name)\r\nselect 1, 'tom'", sqls.get(1));
137+
Assertions.assertEquals("set enable_unique_key_partial_update = false", sqls.get(2));
138+
}
139+
125140
}

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/param/HiveDataSourceProcessor.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import java.util.List;
4040
import java.util.Map;
4141

42+
import com.alibaba.druid.sql.parser.SQLParserUtils;
4243
import com.google.auto.service.AutoService;
4344

4445
@AutoService(DataSourceProcessor.class)
@@ -149,6 +150,11 @@ public DataSourceProcessor create() {
149150
return new HiveDataSourceProcessor();
150151
}
151152

153+
@Override
154+
public List<String> splitAndRemoveComment(String sql) {
155+
return SQLParserUtils.splitAndRemoveComment(sql, com.alibaba.druid.DbType.hive);
156+
}
157+
152158
private String transformOther(Map<String, String> otherMap) {
153159
if (MapUtils.isEmpty(otherMap)) {
154160
return null;

0 commit comments

Comments
 (0)