Skip to content

Commit 92c44ee

Browse files
committed
[fix](fs) Match object storage endpoints case-insensitively
Object storage endpoints are DNS host names, but the provider guesses and the endpoint->region parsing compared them case-sensitively, so an equivalent but differently cased endpoint could either select no storage implementation at all ('No supported storage type found') or be routed to the wrong one. Match endpoint hosts case-insensitively for OSS / OSS-HDFS / S3 / COS / OBS and in the S3URI / LegacyS3Uri / S3ResourceCompat region parsing (including the case-sensitive scheme strip), and normalise the effective OSS endpoint to lowercase. Normalising is not cosmetic: the effective endpoint is handed to the BE as AWS_ENDPOINT, where the presigned-URL conversion matches '-internal.aliyuncs.com' case-sensitively.
1 parent dabd51e commit 92c44ee

22 files changed

Lines changed: 441 additions & 28 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,8 @@ private static String encodedLocation(String location) {
354354
public static boolean isHdfsOnOssEndpoint(String location) {
355355
// example: cn-shanghai.oss-dls.aliyuncs.com contains the "oss-dls.aliyuncs".
356356
// https://www.alibabacloud.com/help/en/e-mapreduce/latest/oss-kusisurumen
357-
return location.contains("oss-dls.aliyuncs");
357+
// The marker is a DNS hostname, so the match is case-insensitive.
358+
return StringUtils.containsIgnoreCase(location, "oss-dls.aliyuncs");
358359
}
359360

360361
/**

‎fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import java.net.URISyntaxException;
2828
import java.util.ArrayList;
2929
import java.util.List;
30+
import java.util.Locale;
3031
import java.util.Map;
3132
import java.util.Optional;
3233
import java.util.Set;
@@ -287,12 +288,15 @@ private void parseEndpointAndRegion() {
287288
if (endpointSplits.length < 2) {
288289
return;
289290
}
290-
if (endpointSplits[0].contains("oss-")) {
291+
// The endpoint is a DNS host, so the marker match (and the derived region it selects) is
292+
// case-insensitive: an upper/mixed case spelling must yield the same region as the
293+
// lowercase one.
294+
if (StringUtils.containsIgnoreCase(endpointSplits[0], "oss-")) {
291295
// compatible with the endpoint: oss-cn-bejing.aliyuncs.com
292-
region = endpointSplits[0];
296+
region = endpointSplits[0].toLowerCase(Locale.ROOT);
293297
return;
294298
}
295-
region = endpointSplits[1];
299+
region = endpointSplits[1].toLowerCase(Locale.ROOT);
296300
}
297301

298302
/**

‎fe/fe-core/src/main/java/org/apache/doris/datasource/storage/S3ResourceCompat.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
import java.util.Arrays;
2626
import java.util.List;
27+
import java.util.Locale;
2728
import java.util.Map;
2829
import java.util.regex.Pattern;
2930

@@ -191,8 +192,11 @@ public static String getRegionOfEndpoint(String endpoint) {
191192
// if endpoint contains '192.168.0.1:8999', return null region
192193
return null;
193194
}
194-
String[] endpointSplit = endpoint.replace("http://", "")
195-
.replace("https://", "")
195+
// The endpoint is a DNS host: normalise case and the scheme prefix (schemes are
196+
// case-insensitive too) first, so an upper/mixed case spelling yields the same region as
197+
// the lowercase one.
198+
String[] endpointSplit = endpoint.toLowerCase(Locale.ROOT)
199+
.replaceFirst("^https?://", "")
196200
.split("\\.");
197201
if (endpointSplit.length < 2) {
198202
return null;

‎fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,17 @@ public void testOSSLocationConvert() {
153153
Assertions.assertEquals(locationPath.getFileSystemType(), FileSystemType.HDFS);*/
154154
}
155155

156+
@Test
157+
public void testIsHdfsOnOssEndpointIsCaseInsensitive() {
158+
// DORIS-29438: the oss-dls marker is a DNS hostname and must match case-insensitively.
159+
Assertions.assertTrue(
160+
LocationPath.isHdfsOnOssEndpoint("oss://bucket.cn-hangzhou.oss-dls.aliyuncs.com/path"));
161+
Assertions.assertTrue(
162+
LocationPath.isHdfsOnOssEndpoint("oss://bucket.CN-HANGZHOU.OSS-DLS.ALIYUNCS.COM/path"));
163+
Assertions.assertFalse(
164+
LocationPath.isHdfsOnOssEndpoint("oss://bucket.oss-cn-hangzhou.aliyuncs.com/path"));
165+
}
166+
156167
@Test
157168
public void testCOSLocationConvert() {
158169
LocationPath locationPath = LocationPath.of("cos://test.com");

‎fe/fe-core/src/test/java/org/apache/doris/common/util/S3URITest.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,20 @@ public void testOSSPathStyle() throws UserException {
127127
Assertions.assertTrue(uri1.getQueryParams().get().get("partNumber").contains("88"));
128128
}
129129

130+
@Test
131+
public void testRegionParsingIsCaseInsensitive() throws UserException {
132+
// The endpoint host is a DNS name, so the derived region must not depend on the case the
133+
// user wrote.
134+
S3URI oss = S3URI.create(
135+
"https://my-bucket.OSS-CN-BEJING.ALIYUNCS.COM/resources/doc.txt", false, false);
136+
Assertions.assertEquals("my-bucket", oss.getBucket());
137+
Assertions.assertEquals("oss-cn-bejing", oss.getRegion().get());
138+
139+
S3URI aws = S3URI.create(
140+
"https://my-bucket.S3.US-WEST-1.AMAZONAWS.COM/resources/doc.txt", false, false);
141+
Assertions.assertEquals("us-west-1", aws.getRegion().get());
142+
}
143+
130144
@Test
131145
public void testCOSVirtualHostStyle() throws UserException {
132146
String p1 = "https://my-bucket.cos.ap-beijing.myqcloud.com/resources/doc.txt";
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.datasource.storage;
19+
20+
import org.junit.jupiter.api.Assertions;
21+
import org.junit.jupiter.api.Test;
22+
23+
/**
24+
* {@code getRegionOfEndpoint} derives the region from an endpoint host, so the result must not
25+
* depend on the case the user wrote for the host or the scheme.
26+
*/
27+
public class S3ResourceCompatTest {
28+
29+
@Test
30+
public void testOssRegionIsCaseInsensitive() {
31+
Assertions.assertEquals("oss-cn-beijing",
32+
S3ResourceCompat.getRegionOfEndpoint("oss-cn-beijing.aliyuncs.com"));
33+
Assertions.assertEquals("oss-cn-beijing",
34+
S3ResourceCompat.getRegionOfEndpoint("OSS-CN-BEIJING.ALIYUNCS.COM"));
35+
Assertions.assertEquals("oss-cn-beijing",
36+
S3ResourceCompat.getRegionOfEndpoint("HTTPS://OSS-CN-BEIJING.ALIYUNCS.COM"));
37+
}
38+
39+
@Test
40+
public void testPlainRegionIsCaseInsensitive() {
41+
Assertions.assertEquals("us-east-1",
42+
S3ResourceCompat.getRegionOfEndpoint("s3.us-east-1.amazonaws.com"));
43+
Assertions.assertEquals("us-east-1",
44+
S3ResourceCompat.getRegionOfEndpoint("HTTPS://S3.US-EAST-1.AMAZONAWS.COM"));
45+
}
46+
47+
@Test
48+
public void testIpEndpointHasNoRegion() {
49+
Assertions.assertNull(S3ResourceCompat.getRegionOfEndpoint("192.168.0.1:8999"));
50+
}
51+
}

‎fe/fe-filesystem/fe-filesystem-cos/src/main/java/org/apache/doris/filesystem/cos/CosFileSystemProvider.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,10 @@ public boolean supports(Map<String, String> properties) {
5151
return true;
5252
}
5353
String endpoint = firstPresent(properties, ENDPOINT_NAMES);
54-
return endpoint != null && endpoint.contains("myqcloud.com");
54+
// The endpoint is a DNS host: classify case-insensitively so an upper/mixed case spelling
55+
// routes exactly like its lowercase form (the legacy COSProperties.guessIsMe compared
56+
// case-sensitively).
57+
return endpoint != null && StringUtils.containsIgnoreCase(endpoint, "myqcloud.com");
5558
}
5659

5760
@Override
@@ -76,14 +79,14 @@ public boolean supportsGuess(Map<String, String> properties) {
7679
String value = properties.get(name);
7780
if (value != null) {
7881
if (!value.isEmpty()) {
79-
return value.contains("myqcloud.com");
82+
return StringUtils.containsIgnoreCase(value, "myqcloud.com");
8083
}
8184
break;
8285
}
8386
}
8487
for (Map.Entry<String, String> entry : properties.entrySet()) {
8588
if ("uri".equalsIgnoreCase(entry.getKey()) && entry.getValue() != null) {
86-
return entry.getValue().contains("myqcloud.com");
89+
return StringUtils.containsIgnoreCase(entry.getValue(), "myqcloud.com");
8790
}
8891
}
8992
return false;
Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.filesystem.cos;
19+
20+
import org.junit.jupiter.api.Assertions;
21+
import org.junit.jupiter.api.Test;
22+
23+
import java.util.HashMap;
24+
import java.util.Map;
25+
26+
/**
27+
* The Tencent COS endpoint is matched by DNS hostname, so classification must be case-insensitive
28+
* (same class of defect as DORIS-29438 / OSS).
29+
*/
30+
class CosFileSystemProviderTest {
31+
32+
private final CosFileSystemProvider provider = new CosFileSystemProvider();
33+
34+
@Test
35+
void supports_isCaseInsensitiveForCosEndpoint() {
36+
Map<String, String> lower = new HashMap<>();
37+
lower.put("cos.endpoint", "cos.ap-shanghai.myqcloud.com");
38+
Map<String, String> upper = new HashMap<>();
39+
upper.put("cos.endpoint", "COS.AP-SHANGHAI.MYQCLOUD.COM");
40+
41+
Assertions.assertTrue(provider.supports(lower));
42+
Assertions.assertTrue(provider.supports(upper), upper.toString());
43+
}
44+
45+
@Test
46+
void supportsGuess_isCaseInsensitiveForCosEndpointAndUri() {
47+
Map<String, String> upperEndpoint = new HashMap<>();
48+
upperEndpoint.put("s3.endpoint", "COS.AP-SHANGHAI.MYQCLOUD.COM");
49+
Assertions.assertTrue(provider.supportsGuess(upperEndpoint), upperEndpoint.toString());
50+
51+
Map<String, String> upperUri = new HashMap<>();
52+
upperUri.put("uri", "cos://my-bucket.COS.AP-SHANGHAI.MYQCLOUD.COM/resources/doc.txt");
53+
Assertions.assertTrue(provider.supportsGuess(upperUri), upperUri.toString());
54+
}
55+
56+
@Test
57+
void supportsGuess_rejectsForeignEndpointRegardlessOfCase() {
58+
Map<String, String> aws = new HashMap<>();
59+
aws.put("s3.endpoint", "HTTPS://S3.US-EAST-1.AMAZONAWS.COM");
60+
61+
Assertions.assertFalse(provider.supportsGuess(aws));
62+
Assertions.assertFalse(provider.supports(aws));
63+
}
64+
}

‎fe/fe-filesystem/fe-filesystem-minio/src/main/java/org/apache/doris/filesystem/minio/MinioFileSystemProvider.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,8 +131,11 @@ public boolean supportsGuess(Map<String, String> properties) {
131131
// region keys — replicate both exclusion legs.
132132
for (Map.Entry<String, String> entry : properties.entrySet()) {
133133
if ("uri".equalsIgnoreCase(entry.getKey()) && entry.getValue() != null) {
134-
// case-sensitive contains, byte-identical to legacy S3Properties.guessIsMe
135-
if (entry.getValue().contains("amazonaws.com")) {
134+
// The uri host is a DNS name: match case-insensitively, deliberately deviating
135+
// from legacy S3Properties.guessIsMe's case-sensitive contains. With the legacy
136+
// behaviour an upper-case AWS uri was not excluded here, so MinIO silently
137+
// claimed a map that belongs to S3.
138+
if (entry.getValue().toLowerCase(java.util.Locale.ROOT).contains("amazonaws.com")) {
136139
return false;
137140
}
138141
break;

‎fe/fe-filesystem/fe-filesystem-minio/src/test/java/org/apache/doris/filesystem/minio/MinioFileSystemProviderTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,29 @@ void supports_rejectsMinioKeysAlongsideAwsOrGcsEndpoint() {
9595
Assertions.assertFalse(provider.supports(gcs));
9696
}
9797

98+
@Test
99+
void supportsGuess_excludesAwsUriFromTheUriFallbackRegardlessOfCase() {
100+
// No endpoint alias present, so the guess falls back to the uri marker. That marker must be
101+
// matched case-insensitively: with the legacy case-sensitive contains an upper-case AWS uri
102+
// was not excluded, and MinIO silently claimed a map that belongs to S3.
103+
Map<String, String> upperAws = new HashMap<>();
104+
upperAws.put("uri", "s3://bucket.S3.US-EAST-1.AMAZONAWS.COM/key");
105+
upperAws.put("minio.access_key", "ak");
106+
107+
Map<String, String> lowerAws = new HashMap<>();
108+
lowerAws.put("uri", "s3://bucket.s3.us-east-1.amazonaws.com/key");
109+
lowerAws.put("minio.access_key", "ak");
110+
111+
Assertions.assertFalse(provider.supportsGuess(upperAws), upperAws.toString());
112+
Assertions.assertFalse(provider.supportsGuess(lowerAws), lowerAws.toString());
113+
114+
// Control: a genuinely non-AWS uri is still claimed by MinIO.
115+
Map<String, String> privateMinio = new HashMap<>();
116+
privateMinio.put("uri", "s3://bucket.minio.internal/key");
117+
privateMinio.put("minio.access_key", "ak");
118+
Assertions.assertTrue(provider.supportsGuess(privateMinio), privateMinio.toString());
119+
}
120+
98121
@Test
99122
void supports_rejectsForeignDialects() {
100123
Map<String, String> gcs = new HashMap<>();

0 commit comments

Comments
 (0)