Skip to content

Commit 7c2e0eb

Browse files
kaka11chenHappenLee
authored andcommitted
(Fix)[hive-writer] Fixed the issue when partition values contain spaces when writing to s3. (apache#35645)
## Proposed changes Issue Number: close apache#31442 (Fix) [hive-writer] Fixed the issue when partition values contain spaces when writing to s3. ### Error msg ``` org.apache.doris.common.UserException: errCode = 2, detailMessage = java.net.URISyntaxException: Illegal character in path at index 114: oss://xxxxxxxxxxx/hive/tpcds1000_partition_oss/call_center/cc_call_center_sk=1/cc_mkt_class=A bit narrow forms matter animals. Consist/cc_market_manager=Daniel Weller/cc_rec_end_date=2001-12-31/f6b5ff4253414b06-9fd365ef68e5ddc5_133f02fb-a7e0-4109-9100-fb748a28259e-0.zlib.orc at org.apache.doris.common.util.S3URI.validateUri(S3URI.java:134) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.common.util.S3URI.parseUri(S3URI.java:120) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.common.util.S3URI.<init>(S3URI.java:116) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.common.util.S3URI.create(S3URI.java:108) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.fs.obj.S3ObjStorage.deleteObject(S3ObjStorage.java:194) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.fs.remote.ObjFileSystem.delete(ObjFileSystem.java:150) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.fs.remote.SwitchingFileSystem.delete(SwitchingFileSystem.java:92) ~[doris-fe.jar:1.2- ``` ### Root Cause Hadoop partition names will encode some special characters, but not space characters, which is different from URI encoding. Therefore, an error will be reported when constructing URI. ### Solution The solution is to use regular expressions to parse URI, and then pass in each part of URI to construct URI. This URI constructor will encode each part of URI.
1 parent 4520f1c commit 7c2e0eb

2 files changed

Lines changed: 43 additions & 9 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
import java.util.Map;
3131
import java.util.Optional;
3232
import java.util.Set;
33+
import java.util.regex.Matcher;
34+
import java.util.regex.Pattern;
3335
import java.util.stream.Collectors;
3436

3537
/**
@@ -60,11 +62,12 @@
6062
* Virtual Host AWS Client (Hadoop S3) Mixed Style: <code>isPathStyle = false && forceParsingByStandardUri = true</code>
6163
* Path AWS Client (Hadoop S3) Mixed Style: <code>isPathStyle = true && forceParsingByStandardUri = true</code>
6264
*
63-
* When the incoming location is url encoded, the encoded string will be returned.
64-
* For <code>getKey()</code>, <code>getQueryParams()</code> will return the encoding string
6565
*/
6666

6767
public class S3URI {
68+
69+
private static final Pattern URI_PATTERN =
70+
Pattern.compile("^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\\?([^#]*))?(#(.*))?");
6871
public static final String SCHEME_DELIM = "://";
6972
public static final String PATH_DELIM = "/";
7073
private static final Set<String> VALID_SCHEMES = ImmutableSet.of("http", "https", "s3", "s3a", "s3n",
@@ -117,7 +120,8 @@ private S3URI(String location, boolean isPathStyle, boolean forceParsingByStanda
117120
}
118121

119122
private void parseUri(String location, boolean forceParsingStandardUri) throws UserException {
120-
validateUri(location);
123+
parseURILocation(location);
124+
validateUri();
121125

122126
if (!forceParsingStandardUri && OS_SCHEMES.contains(uri.getScheme().toLowerCase())) {
123127
parseAwsCliStyleUri();
@@ -127,12 +131,29 @@ private void parseUri(String location, boolean forceParsingStandardUri) throws U
127131
parseEndpointAndRegion();
128132
}
129133

130-
private void validateUri(String location) throws UserException {
134+
/**
135+
* parse uri location and encode to a URI.
136+
* @param location
137+
* @throws UserException
138+
*/
139+
private void parseURILocation(String location) throws UserException {
140+
Matcher matcher = URI_PATTERN.matcher(location);
141+
if (!matcher.matches()) {
142+
throw new UserException("Failed to parse uri: " + location);
143+
}
144+
String scheme = matcher.group(2);
145+
String authority = matcher.group(4);
146+
String path = matcher.group(5);
147+
String query = matcher.group(7);
148+
String fragment = matcher.group(9);
131149
try {
132-
uri = new URI(location);
150+
uri = new URI(scheme, authority, path, query, fragment).normalize();
133151
} catch (URISyntaxException e) {
134152
throw new UserException(e);
135153
}
154+
}
155+
156+
private void validateUri() throws UserException {
136157
if (uri.getScheme() == null || !VALID_SCHEMES.contains(uri.getScheme().toLowerCase())) {
137158
throw new UserException("Invalid scheme: " + this.uri);
138159
}
@@ -143,7 +164,7 @@ private void parseAwsCliStyleUri() throws UserException {
143164
if (bucket == null) {
144165
throw new UserException("missing bucket: " + uri);
145166
}
146-
String path = uri.getRawPath();
167+
String path = uri.getPath();
147168
if (path.length() > 1) {
148169
key = path.substring(1);
149170
} else {
@@ -173,7 +194,7 @@ private void parseStandardUri() throws UserException {
173194

174195
private void addQueryParamsIfNeeded() {
175196
if (uri.getQuery() != null) {
176-
queryParams = splitQueryString(uri.getRawQuery()).stream().map((s) -> s.split("="))
197+
queryParams = splitQueryString(uri.getQuery()).stream().map((s) -> s.split("="))
177198
.map((s) -> s.length == 1 ? new String[] {s[0], null} : s).collect(
178199
Collectors.groupingBy((a) -> a[0],
179200
Collectors.mapping((a) -> a[1], Collectors.toList())));
@@ -201,7 +222,7 @@ private static List<String> splitQueryString(String queryString) {
201222
}
202223

203224
private void parsePathStyleUri() throws UserException {
204-
String path = uri.getRawPath();
225+
String path = uri.getPath();
205226

206227
if (!StringUtils.isEmpty(path) && !"/".equals(path)) {
207228
int index = path.indexOf('/', 1);
@@ -226,7 +247,7 @@ private void parsePathStyleUri() throws UserException {
226247
private void parseVirtualHostedStyleUri() throws UserException {
227248
bucket = uri.getHost().split("\\.")[0];
228249

229-
String path = uri.getRawPath();
250+
String path = uri.getPath();
230251
if (!StringUtils.isEmpty(path) && !"/".equals(path)) {
231252
key = path.substring(1);
232253
} else {

‎fe/fe-core/src/test/java/org/apache/doris/common/util/S3URITest.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,19 @@ public void testEncodedString() throws UserException {
170170
Assert.assertTrue(uri1.getQueryParams().get().get("partNumber").contains("88"));
171171
}
172172

173+
@Test
174+
public void testHadoopEncodedString() throws UserException {
175+
String p1 = "s3://bucket/path%20to%20file/abc%3Aqqq=xyz%2Fyyy zzz";
176+
boolean isPathStyle = false;
177+
boolean forceParsingStandardUri = false;
178+
S3URI uri1 = S3URI.create(p1, isPathStyle, forceParsingStandardUri);
179+
180+
Assert.assertEquals("bucket", uri1.getBucket());
181+
Assert.assertEquals("path%20to%20file/abc%3Aqqq=xyz%2Fyyy zzz", uri1.getKey());
182+
Assert.assertEquals(Optional.empty(), uri1.getEndpoint());
183+
Assert.assertEquals(Optional.empty(), uri1.getRegion());
184+
}
185+
173186
@Test(expected = UserException.class)
174187
public void missingBucket() throws UserException {
175188
S3URI.create("https:///");

0 commit comments

Comments
 (0)