Skip to content

Commit a7a0f19

Browse files
committed
[fix](tvf) Validate remote URIs before opening S3 Lance
### What problem does this PR solve? Issue Number: N/A Related PR: #69526 Problem Summary: S3 Lance could open a local dataset through BrokerProperties because the entry point rejected only LocalProperties. Validate the final dataset URI before loading metadata. Allow the pinned SDK's remote providers and reject local aliases and malformed authorities that could fall back to local paths. Keep valid ports, underscore buckets, hostless GooseFS and normalized S3/OSS HTTP locations working without changing catalog or non-Lance paths. ### Release note S3 Lance rejects local dataset URIs and malformed remote URI authorities before opening a dataset, regardless of storage-property selection. ### Check List (For Author) - Test: 31 focused FE unit tests passed on this branch; changed Java files pass Checkstyle and git diff --check passes. - Behavior changed: Yes. S3 Lance requires a valid supported remote URI. - Does this need documentation: No additional documentation for this security validation; the parent PR's configuration documentation is pending.
1 parent 564841d commit a7a0f19

2 files changed

Lines changed: 133 additions & 5 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java‎

Lines changed: 42 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,20 @@
2424
import org.apache.doris.common.UserException;
2525
import org.apache.doris.datasource.lance.metadata.LanceMetadataLoader;
2626
import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
27-
import org.apache.doris.datasource.property.storage.LocalProperties;
2827
import org.apache.doris.datasource.property.storage.StorageProperties;
2928
import org.apache.doris.thrift.TBrokerFileStatus;
3029
import org.apache.doris.thrift.TFileType;
3130

31+
import com.google.common.collect.ImmutableSet;
32+
import com.google.common.net.HostAndPort;
33+
34+
import java.net.URI;
35+
import java.net.URISyntaxException;
3236
import java.util.Collections;
3337
import java.util.List;
38+
import java.util.Locale;
3439
import java.util.Map;
40+
import java.util.Set;
3541

3642
/**
3743
* The Implement of table valued function
@@ -47,6 +53,11 @@
4753
public class S3TableValuedFunction extends ExternalFileTableValuedFunction {
4854
public static final String NAME = "s3";
4955

56+
// Remote providers registered by the pinned Lance SDK. File providers also have aliases,
57+
// so checking only for "file" (or a particular StorageProperties type) is insufficient.
58+
private static final Set<String> LANCE_REMOTE_SCHEMES = ImmutableSet.of(
59+
"s3", "s3+ddb", "az", "abfss", "gs", "goosefs", "oss", "cos", "hf", "tos");
60+
5061
public S3TableValuedFunction(Map<String, String> properties) throws AnalysisException {
5162
this(properties, null);
5263
}
@@ -64,10 +75,10 @@ public S3TableValuedFunction(Map<String, String> properties, List<TBrokerFileSta
6475
} catch (UserException e) {
6576
throw new RuntimeException(e);
6677
}
67-
// LocalProperties accepts file_path unchanged. Do not let S3 Lance open a local
68-
// dataset in FE, bypassing the LOCAL feature switch and BE secure directory.
69-
if (isLanceFormat() && storageProperties instanceof LocalProperties) {
70-
throw new AnalysisException("S3 Lance requires remote storage properties");
78+
// Validate the final URI, including when BrokerProperties passes it through unchanged.
79+
// A local dataset must not bypass the LOCAL switch, privileges or BE secure directory.
80+
if (isLanceFormat() && !isRemoteLanceUri(filePath)) {
81+
throw new AnalysisException("S3 Lance requires remote storage with a valid URI");
7182
}
7283
if (FeConstants.runningUnitTest) {
7384
// Just check
@@ -95,6 +106,32 @@ public S3TableValuedFunction(Map<String, String> properties, List<TBrokerFileSta
95106
}
96107

97108

109+
private static boolean isRemoteLanceUri(String path) {
110+
try {
111+
URI uri = new URI(path == null ? "" : path);
112+
String scheme = uri.getScheme();
113+
String authority = uri.getRawAuthority();
114+
if (scheme == null || !LANCE_REMOTE_SCHEMES.contains(scheme.toLowerCase(Locale.ROOT))
115+
|| uri.isOpaque()) {
116+
return false;
117+
}
118+
if (authority == null || authority.isEmpty()) {
119+
// GooseFS can resolve its remote master from storage options or configuration.
120+
return "goosefs".equalsIgnoreCase(scheme) && uri.getRawPath() != null
121+
&& uri.getRawPath().startsWith("/");
122+
}
123+
// Lance falls back to a local path if Rust's URL parser rejects the URI. Java's URI
124+
// accepts registry authorities with invalid ports, which Rust rejects. Validate the
125+
// host/port separately without disallowing legitimate bucket names with underscores.
126+
String host = HostAndPort.fromString(authority.substring(authority.lastIndexOf('@') + 1))
127+
.requireBracketsForIPv6().getHost();
128+
// Java accepts IPv6 zone identifiers, while Rust's URL parser rejects them.
129+
return !host.isEmpty() && !(host.contains(":") && host.contains("%"));
130+
} catch (URISyntaxException | IllegalArgumentException e) {
131+
return false;
132+
}
133+
}
134+
98135
// =========== implement abstract methods of ExternalFileTableValuedFunction =================
99136
@Override
100137
public TFileType getTFileType() {

‎fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.doris.common.AnalysisException;
2121
import org.apache.doris.common.FeConstants;
2222
import org.apache.doris.common.util.S3Util;
23+
import org.apache.doris.datasource.lance.metadata.LanceMetadataLoader;
2324
import org.apache.doris.fs.FileSystemFactory;
2425
import org.apache.doris.nereids.StatementContext;
2526
import org.apache.doris.nereids.trees.expressions.Properties;
@@ -42,6 +43,7 @@
4243

4344
import java.util.Arrays;
4445
import java.util.Collections;
46+
import java.util.HashMap;
4547
import java.util.Map;
4648
import java.util.concurrent.CompletableFuture;
4749

@@ -80,6 +82,95 @@ public void testLanceRejectsLocalStorageBeforeOpeningDataset() {
8082
Assertions.assertTrue(exception.getMessage().contains("requires remote storage"));
8183
}
8284

85+
@Test
86+
public void testLanceRejectsBrokerLocalUrisBeforeOpeningDataset() {
87+
FeConstants.runningUnitTest = false;
88+
try (MockedStatic<LanceMetadataLoader> loader = Mockito.mockStatic(LanceMetadataLoader.class)) {
89+
for (String uri : Arrays.asList("dataset.lance", "/tmp/dataset.lance", "../dataset.lance",
90+
"//localhost/tmp/dataset.lance", "file:///tmp/dataset.lance", "file:/tmp/dataset.lance",
91+
"file:dataset.lance", "FILE:///tmp/dataset.lance", "file://localhost/tmp/dataset.lance",
92+
"file-object-store://ignored/tmp/dataset.lance", "file+uring://ignored/tmp/dataset.lance",
93+
"FILE-OBJECT-STORE://ignored/tmp/dataset.lance", "FILE+URING://ignored/tmp/dataset.lance",
94+
"c://ignored/tmp/dataset.lance", "memory://dataset.lance", "shared-memory://dataset.lance",
95+
"s3://bucket:bad/../../dataset.lance", "s3://bucket:65536/../../dataset.lance",
96+
"s3://:9000/dataset.lance", "s3://user@/dataset.lance",
97+
"s3://[::1%25eth0]/dataset.lance", "https://bucket:bad/../../dataset.lance",
98+
"", "s3:/bucket/dataset.lance", "s3://", "s3://bucket/bad path")) {
99+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
100+
() -> new S3TableValuedFunction(ImmutableMap.of(
101+
"format", "lance", "uri", uri, "fs.broker.support", "true")), uri);
102+
Assertions.assertTrue(exception.getMessage().contains("requires remote storage"), uri);
103+
}
104+
loader.verifyNoInteractions();
105+
}
106+
}
107+
108+
@Test
109+
public void testLanceRejectsGuessedBrokerFileUrisBeforeOpeningDataset() {
110+
FeConstants.runningUnitTest = false;
111+
try (MockedStatic<LanceMetadataLoader> loader = Mockito.mockStatic(LanceMetadataLoader.class)) {
112+
for (String uri : Arrays.asList("file:///tmp/dataset.lance", "file://localhost/tmp/dataset.lance")) {
113+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
114+
() -> new S3TableValuedFunction(ImmutableMap.of(
115+
"format", "lance", "uri", uri, "broker.name", "test")));
116+
Assertions.assertTrue(exception.getMessage().contains("requires remote storage"));
117+
}
118+
loader.verifyNoInteractions();
119+
}
120+
}
121+
122+
@Test
123+
public void testLanceRejectsBrokerWithoutUriBeforeOpeningDataset() {
124+
FeConstants.runningUnitTest = false;
125+
try (MockedStatic<LanceMetadataLoader> loader = Mockito.mockStatic(LanceMetadataLoader.class)) {
126+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
127+
() -> new S3TableValuedFunction(ImmutableMap.of("format", "lance", "fs.broker.support", "true")));
128+
Assertions.assertTrue(exception.getMessage().contains("requires remote storage"));
129+
loader.verifyNoInteractions();
130+
}
131+
}
132+
133+
@Test
134+
public void testLanceRemoteUrisReachMetadataLoader() throws Exception {
135+
FeConstants.runningUnitTest = false;
136+
for (String uri : Arrays.asList("s3://bucket/dataset.lance", "s3+ddb://bucket/dataset.lance",
137+
"oss://bucket/dataset.lance", "gs://bucket/dataset.lance", "az://container/dataset.lance",
138+
"abfss://[email protected]/dataset.lance",
139+
"gs://bucket_with_underscore/dataset.lance", "s3://bucket:9000/dataset.lance",
140+
"goosefs://bucket/dataset.lance", "goosefs:///dataset.lance", "goosefs:/dataset.lance",
141+
"cos://bucket/dataset.lance",
142+
"hf://datasets/repo/data.lance",
143+
"tos://bucket/dataset.lance", "S3://bucket/dataset.lance")) {
144+
assertLanceMetadataLoad(ImmutableMap.of(
145+
"format", "lance", "uri", uri, "fs.broker.support", "true"), uri);
146+
}
147+
}
148+
149+
@Test
150+
public void testLanceHttpStorageUrisAreNormalizedBeforeValidation() throws Exception {
151+
FeConstants.runningUnitTest = false;
152+
Map<String, String> s3Properties = new HashMap<>(PROPERTIES);
153+
s3Properties.put("format", "lance");
154+
s3Properties.put("uri", "https://bucket.s3.us-west-2.amazonaws.com/dataset.lance");
155+
assertLanceMetadataLoad(s3Properties, "s3://bucket/dataset.lance");
156+
assertLanceMetadataLoad(ImmutableMap.of(
157+
"format", "lance", "uri", "https://bucket.oss-cn-hangzhou.aliyuncs.com/dataset.lance",
158+
"oss.endpoint", "oss-cn-hangzhou.aliyuncs.com", "oss.access_key", "ak", "oss.secret_key", "sk"),
159+
"s3://bucket/dataset.lance");
160+
}
161+
162+
private void assertLanceMetadataLoad(Map<String, String> properties, String uri) throws Exception {
163+
try (MockedStatic<LanceMetadataLoader> loader = Mockito.mockStatic(LanceMetadataLoader.class)) {
164+
IllegalStateException reachedLoader = new IllegalStateException("remote metadata loader reached");
165+
loader.when(() -> LanceMetadataLoader.loadLatestForTvf(Mockito.eq(uri), Mockito.anyList()))
166+
.thenThrow(reachedLoader);
167+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
168+
() -> new S3TableValuedFunction(properties));
169+
Assertions.assertSame(reachedLoader, exception.getCause(), uri);
170+
loader.verify(() -> LanceMetadataLoader.loadLatestForTvf(Mockito.eq(uri), Mockito.anyList()));
171+
}
172+
}
173+
83174
@Test
84175
public void testExactFilesKeepLiteralPathsAndSkipListing() throws Exception {
85176
FeConstants.runningUnitTest = false;

0 commit comments

Comments
 (0)