Skip to content

Commit 020092e

Browse files
ruanwenjunruanwenjun
authored
Set charset in upgrading tools to solve read upgrade sql mojibake (#14955)
1 parent 4b5eb1a commit 020092e

File tree

8 files changed

+408
-213
lines changed

8 files changed

+408
-213
lines changed

‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/ClasspathSqlScriptParser.java‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.common.sql;
19+
20+
import org.apache.commons.lang3.StringUtils;
21+
22+
import java.io.IOException;
23+
import java.io.InputStream;
24+
import java.io.InputStreamReader;
25+
import java.io.LineNumberReader;
26+
import java.io.Reader;
27+
import java.nio.charset.Charset;
28+
import java.nio.charset.StandardCharsets;
29+
import java.util.ArrayList;
30+
import java.util.List;
31+
32+
import org.springframework.core.io.ClassPathResource;
33+
import org.springframework.core.io.Resource;
34+
35+
public class ClasspathSqlScriptParser implements SqlScriptParser {
36+
37+
private final String sqlScriptPath;
38+
39+
private final Charset charset;
40+
41+
public ClasspathSqlScriptParser(String sqlScriptPath) {
42+
this.sqlScriptPath = sqlScriptPath;
43+
this.charset = StandardCharsets.UTF_8;
44+
}
45+
46+
@Override
47+
public List<String> getAllSql() throws IOException {
48+
Resource sqlScriptResource = new ClassPathResource(sqlScriptPath);
49+
List<String> result = new ArrayList<>();
50+
try (
51+
InputStream inputStream = sqlScriptResource.getInputStream();
52+
Reader sqlScriptReader = new InputStreamReader(inputStream, charset);
53+
LineNumberReader lineNumberReader = new LineNumberReader(sqlScriptReader)) {
54+
String sql;
55+
do {
56+
sql = parseNextSql(lineNumberReader);
57+
if (StringUtils.isNotBlank(sql)) {
58+
result.add(sql);
59+
}
60+
} while (StringUtils.isNotBlank(sql));
61+
}
62+
return result;
63+
}
64+
65+
private String parseNextSql(LineNumberReader lineNumberReader) throws IOException {
66+
String line;
67+
while ((line = lineNumberReader.readLine()) != null) {
68+
String trimLine = line.trim();
69+
if (StringUtils.isEmpty(trimLine) || isComment(trimLine)) {
70+
// Skip the empty line, comment line
71+
continue;
72+
}
73+
if (trimLine.startsWith("/*")) {
74+
skipLicenseHeader(lineNumberReader);
75+
continue;
76+
}
77+
if (trimLine.startsWith("delimiter")) {
78+
// begin to parse processor, until delimiter ;
79+
String[] split = trimLine.split(" ");
80+
return parseProcedure(lineNumberReader, split[1]);
81+
}
82+
// begin to parse sql until;
83+
List<String> sqlLines = new ArrayList<>();
84+
sqlLines.add(line);
85+
while (!line.endsWith(";")) {
86+
line = lineNumberReader.readLine();
87+
if (line == null) {
88+
break;
89+
}
90+
if (StringUtils.isBlank(line)) {
91+
continue;
92+
}
93+
sqlLines.add(line);
94+
}
95+
return String.join("\n", sqlLines);
96+
}
97+
return null;
98+
}
99+
100+
private void skipLicenseHeader(LineNumberReader lineNumberReader) throws IOException {
101+
String line;
102+
while ((line = lineNumberReader.readLine()) != null) {
103+
String trimLine = line.trim();
104+
if (StringUtils.isEmpty(trimLine) || isComment(trimLine)) {
105+
// Skip the empty line, comment line
106+
continue;
107+
}
108+
if (line.startsWith("*/")) {
109+
break;
110+
}
111+
}
112+
}
113+
114+
private String parseProcedure(LineNumberReader lineNumberReader, String delimiter) throws IOException {
115+
List<String> sqlLines = new ArrayList<>();
116+
// begin to parse processor, until delimiter ;
117+
String line;
118+
while (true) {
119+
line = lineNumberReader.readLine();
120+
if (line == null) {
121+
break;
122+
}
123+
if (StringUtils.isBlank(line)) {
124+
continue;
125+
}
126+
if (line.startsWith(delimiter)) {
127+
break;
128+
}
129+
sqlLines.add(line);
130+
}
131+
return String.join("\n", sqlLines);
132+
}
133+
134+
private boolean isComment(String line) {
135+
return line.startsWith("--") || line.startsWith("//");
136+
}
137+
}

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/ScriptRunnerTest.java renamed to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/SqlScriptParser.java

Lines changed: 6 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,25 +14,14 @@
1414
* See the License for the specific language governing permissions and
1515
* limitations under the License.
1616
*/
17-
package org.apache.dolphinscheduler.common.utils;
1817

19-
import java.io.StringReader;
18+
package org.apache.dolphinscheduler.common.sql;
2019

21-
import org.junit.jupiter.api.Assertions;
22-
import org.junit.jupiter.api.Test;
20+
import java.io.IOException;
21+
import java.util.List;
2322

24-
public class ScriptRunnerTest {
23+
public interface SqlScriptParser {
24+
25+
List<String> getAllSql() throws IOException;
2526

26-
@Test
27-
public void testRunScript() {
28-
// connection is null
29-
Exception exception = null;
30-
ScriptRunner s = new ScriptRunner(null, true, true);
31-
try {
32-
s.runScript(new StringReader("select 1"));
33-
} catch (Exception e) {
34-
exception = e;
35-
}
36-
Assertions.assertNotNull(exception);
37-
}
3827
}

‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/sql/SqlScriptRunner.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.common.sql;
19+
20+
import org.apache.commons.lang3.StringUtils;
21+
22+
import java.io.IOException;
23+
import java.sql.Connection;
24+
import java.sql.SQLException;
25+
import java.sql.Statement;
26+
import java.util.List;
27+
28+
import javax.sql.DataSource;
29+
30+
import lombok.extern.slf4j.Slf4j;
31+
32+
@Slf4j
33+
public class SqlScriptRunner {
34+
35+
private final DataSource dataSource;
36+
37+
private final SqlScriptParser sqlScriptParser;
38+
39+
/**
40+
* @param dataSource DataSource which used to execute the sql script.
41+
* @param sqlScriptFilePath Sqk script file path, the path should under classpath.
42+
*/
43+
public SqlScriptRunner(DataSource dataSource, String sqlScriptFilePath) {
44+
this.dataSource = dataSource;
45+
this.sqlScriptParser = new ClasspathSqlScriptParser(sqlScriptFilePath);
46+
}
47+
48+
public void execute() throws SQLException, IOException {
49+
List<String> allSql = sqlScriptParser.getAllSql();
50+
try (Connection connection = dataSource.getConnection()) {
51+
for (String sql : allSql) {
52+
if (StringUtils.isBlank(sql)) {
53+
continue;
54+
}
55+
try (Statement statement = connection.createStatement()) {
56+
// Since some sql doesn't have result so we believe if there is no exception then we think the sql
57+
// execute success.
58+
statement.execute(sql);
59+
log.info("Execute sql: {} success", sql);
60+
}
61+
}
62+
}
63+
}
64+
65+
}

‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java‎

Lines changed: 0 additions & 162 deletions
This file was deleted.

0 commit comments

Comments
 (0)