Skip to content

Commit c5f9890

Browse files
committed
[fix](tvf) Align guarded routes with filesystem consumers
### What problem does this PR solve? Issue Number: N/A Related PR: #69526 Problem Summary: Align glob discovery routes and exact nested ViewFS identity, and preserve supported remote providers. Keep the existing LOCAL setting, privileges and containment requirements. ### Release note Guarded Hadoop requests use exact nested ViewFS identities and reject escaped local listing routes. Supported provider compatibility is preserved. ### Check List (For Author) - Test: Unit Test (98 FE), isolated ViewFS endpoint comparison - Behavior changed: Yes (guarded route enforcement and supported provider compatibility) - Does this need documentation: Yes (startup and upgrade guidance remains pending)
1 parent 9cba45b commit c5f9890

3 files changed

Lines changed: 204 additions & 9 deletions

File tree

‎fe/be-java-extensions/hadoop-deps/src/main/java/org/apache/hadoop/fs/FileSystem.java‎

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3703,6 +3703,36 @@ public static String encode(List<Rule> rules) {
37033703
}
37043704
}
37053705

3706+
/** Routes touched by Globber, without constructing a filesystem or listing data. */
3707+
public static List<Rule> globRules(URI filesystem, String pattern) throws IOException {
3708+
List<Rule> rules = new ArrayList<>();
3709+
rules.add(new Rule(filesystem.toString(), pattern, true));
3710+
if (pattern.indexOf('\\') < 0) {
3711+
return rules;
3712+
}
3713+
for (String expanded : GlobExpander.expand(pattern)) {
3714+
Path flattened = new Path(expanded.isEmpty() ? Path.CUR_DIR : expanded);
3715+
String path = flattened.toUri().getPath();
3716+
Path candidate = new Path(flattened.isAbsolute() ? "/" : Path.CUR_DIR);
3717+
for (String component : path.split("/")) {
3718+
if (component.isEmpty()) {
3719+
continue;
3720+
}
3721+
GlobFilter filter = new GlobFilter(component);
3722+
if (filter.hasPattern()) {
3723+
// Globber lists this parent before it knows the matching child names. Keep
3724+
// prior wildcard components as a conservative subtree route, including when
3725+
// a later escaped '..' moves the final candidate outside that subtree.
3726+
rules.add(new Rule(filesystem.toString(), new Path(candidate, "*").toUri().getPath(), true));
3727+
}
3728+
component = component.replaceAll("\\\\(.)", "$1");
3729+
candidate = new Path(candidate, component);
3730+
}
3731+
rules.add(new Rule(filesystem.toString(), candidate.toUri().getPath(), true));
3732+
}
3733+
return rules;
3734+
}
3735+
37063736
private static IOException rejected() {
37073737
return new IOException("Hadoop TVF requires a proven remote filesystem route. "
37083738
+ "Use local() with LOCAL TVF privileges and secure-path restrictions.");
@@ -3798,6 +3828,10 @@ private FrozenConfiguration(Configuration source) throws IOException {
37983828
values.put(key, value);
37993829
}
38003830
}
3831+
// ViewFS's private InnerCache folds authority case (A/a). Guarded children must
3832+
// instead use FileSystem's exact-authority/profile cache below.
3833+
set("fs.viewfs.enable.inner.cache", "false");
3834+
values.put("fs.viewfs.enable.inner.cache", "false");
38013835
try {
38023836
java.security.MessageDigest digest = java.security.MessageDigest.getInstance("SHA-256");
38033837
java.io.DataOutputStream output = new java.io.DataOutputStream(
@@ -3942,11 +3976,6 @@ protected Provider provider(String scheme) {
39423976
}
39433977
try {
39443978
String implementation = FileSystem.getFileSystemClass(scheme, conf).getName();
3945-
if (Arrays.asList("io.juicefs.JuiceFileSystem", "org.apache.hadoop.fs.ozone.OzoneFileSystem",
3946-
"org.apache.hadoop.fs.ozone.RootedOzoneFileSystem", "org.apache.hadoop.fs.cosn.CosNFileSystem",
3947-
"com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter").contains(implementation)) {
3948-
return Provider.REMOTE;
3949-
}
39503979
return implementationProvider(implementation);
39513980
} catch (IOException | RuntimeException e) {
39523981
return Provider.UNKNOWN;
@@ -3971,6 +4000,12 @@ protected enum Provider {
39714000
"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem",
39724001
"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem",
39734002
"org.apache.hadoop.fs.azure.NativeAzureFileSystem",
4003+
"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure",
4004+
"io.juicefs.JuiceFileSystem",
4005+
"org.apache.hadoop.fs.ozone.OzoneFileSystem",
4006+
"org.apache.hadoop.fs.ozone.RootedOzoneFileSystem",
4007+
"org.apache.hadoop.fs.cosn.CosNFileSystem",
4008+
"com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter",
39744009
"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem",
39754010
"com.aliyun.jindodata.oss.JindoOssFileSystem",
39764011
"org.apache.hadoop.fs.obs.OBSFileSystem",
@@ -4024,6 +4059,13 @@ public void checkRoute(URI filesystem, String path, Set<String> activeTables) th
40244059
checkViewFsEnabled(filesystem, path, activeTables);
40254060
}
40264061

4062+
public void checkGlobRoute(URI filesystem, String pattern, Set<String> activeTables)
4063+
throws IOException, URISyntaxException {
4064+
for (Rule rule : globRules(filesystem, pattern)) {
4065+
checkRoute(filesystem, rule.path, activeTables);
4066+
}
4067+
}
4068+
40274069
protected Provider provider(String scheme) {
40284070
if ("file".equalsIgnoreCase(scheme)) {
40294071
return Provider.LOCAL;

‎fe/fe-core/src/main/java/org/apache/doris/tablefunction/HadoopTvfAdmission.java‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,12 @@
1919

2020
import org.apache.doris.common.Config;
2121
import org.apache.doris.common.util.LocationPath;
22+
import org.apache.doris.filesystem.spi.GlobPattern;
2223
import org.apache.doris.nereids.exceptions.AnalysisException;
2324

2425
import org.apache.hadoop.fs.FileSystem.TvfGuard;
2526

27+
import java.io.IOException;
2628
import java.net.URI;
2729
import java.net.URISyntaxException;
2830
import java.nio.charset.StandardCharsets;
@@ -104,6 +106,10 @@ public static void check(Access access, String path, Map<String, String> normali
104106
// Project the Java consumer independently from native convert_path/JNI.
105107
URI javaPath = hadoopPathUri(path);
106108
javaConsumer.checkRoute(javaPath, javaPath.getPath(), new HashSet<>());
109+
if (access == Access.HADOOP_READ && hasListingGlob(path)) {
110+
URI listingPath = hadoopPathUri(GlobPattern.expandNumericRanges(path));
111+
javaConsumer.routes.checkGlobRoute(listingPath, listingPath.getPath(), new HashSet<>());
112+
}
107113
if (access == Access.HADOOP_READ) {
108114
// FileQueryScanNode sends this filesystem identifier in range.fs_name. BE
109115
// falls back to hdfs_params.fs_name only when that range field is empty.
@@ -116,7 +122,7 @@ public static void check(Access access, String path, Map<String, String> normali
116122
}
117123
}
118124
}
119-
} catch (URISyntaxException | IllegalArgumentException e) {
125+
} catch (IOException | URISyntaxException | IllegalArgumentException e) {
120126
new HadoopTvfAdmission(access, normalized, false).reject();
121127
}
122128
}
@@ -135,15 +141,26 @@ public static void removeRuntimePolicy(Map<String, String> properties) {
135141
.startsWith("fs.doris.tvf."));
136142
}
137143

144+
private static boolean hasListingGlob(String path) {
145+
return path.indexOf('*') >= 0 || path.indexOf('?') >= 0
146+
|| path.indexOf('[') >= 0 || path.indexOf('{') >= 0;
147+
}
148+
138149
public static Map<String, String> protectJava(Access access, String path, Map<String, String> properties) {
139150
Map<String, String> guarded = new HashMap<>(properties);
140151
removeRuntimePolicy(guarded);
141152
if (access == Access.HADOOP_READ || !Config.enable_local_tvf) {
142153
try {
143154
URI uri = hadoopPathUri(path);
144-
guarded.put(TvfGuard.POLICY_KEY, TvfGuard.encode(java.util.Collections.singletonList(
145-
new TvfGuard.Rule(uri.toString(), uri.getPath(), true))));
146-
} catch (URISyntaxException | IllegalArgumentException e) {
155+
java.util.List<TvfGuard.Rule> rules;
156+
if (access == Access.HADOOP_READ && hasListingGlob(path)) {
157+
URI listingPath = hadoopPathUri(GlobPattern.expandNumericRanges(path));
158+
rules = TvfGuard.globRules(listingPath, listingPath.getPath());
159+
} else {
160+
rules = java.util.Collections.singletonList(new TvfGuard.Rule(uri.toString(), uri.getPath(), true));
161+
}
162+
guarded.put(TvfGuard.POLICY_KEY, TvfGuard.encode(rules));
163+
} catch (IOException | URISyntaxException | IllegalArgumentException e) {
147164
reject(access);
148165
}
149166
}

‎fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfRuntimeAdmissionTest.java‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,142 @@ private static void nativeRequest(Configuration configuration, String root, Stri
344344
configuration.set(TvfGuard.PATH_KEY, path);
345345
}
346346

347+
@Test
348+
void escapedListingComponentsAreCheckedBeforeDiscovery() throws Exception {
349+
Map<String, String> properties = readProperties("viewfs://cluster/da\\ta/*.csv");
350+
properties.put("fs.defaultFS", "viewfs://cluster");
351+
properties.put("fs.viewfs.mounttable.cluster.link./data", "file:///tmp");
352+
properties.put("fs.viewfs.mounttable.cluster.linkFallback", "hdfs://remote/base");
353+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
354+
() -> new HdfsTableValuedFunction(properties));
355+
properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/data");
356+
Assertions.assertDoesNotThrow(() -> new HdfsTableValuedFunction(properties));
357+
Configuration actual = defaults(Collections.singletonMap(
358+
"fs.viewfs.mounttable.cluster.link./data", "file:///tmp"));
359+
properties.remove("fs.viewfs.mounttable.cluster.link./data");
360+
overlay(actual, HadoopTvfAdmission.protectJava(
361+
HadoopTvfAdmission.Access.HADOOP_READ, properties.get("uri"), properties));
362+
Assertions.assertThrows(IOException.class,
363+
() -> TvfGuard.check(URI.create("viewfs://cluster"), actual));
364+
}
365+
366+
@Test
367+
void globTraversalChecksIntermediateParentsAndEscapedDotSegments() {
368+
Map<String, String> properties = hdfs("viewfs://cluster");
369+
properties.put("fs.viewfs.mounttable.cluster.link./data", "file:///tmp");
370+
properties.put("fs.viewfs.mounttable.cluster.linkFallback", "hdfs://remote/base");
371+
for (String path : new String[] {"viewfs://cluster/da\\ta/*/\\.\\./other/*.csv",
372+
"viewfs://cluster/{da\\ta/x,remote/y}/*.csv"}) {
373+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
374+
() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.HADOOP_READ, path, properties, false));
375+
}
376+
properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/data");
377+
Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.HADOOP_READ,
378+
"viewfs://cluster/da\\ta/\\*/file.csv", properties, false));
379+
properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp");
380+
properties.put("fs.viewfs.mounttable.cluster.link./literal\\name", "hdfs://remote/base");
381+
Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE,
382+
"viewfs://cluster/literal\\name/file.csv", properties));
383+
}
384+
385+
@Test
386+
void explicitJuiceFsUsesTheSameKnownRemoteProviderAsRuntime() throws Exception {
387+
for (boolean enabled : new boolean[] {false, true}) {
388+
Config.enable_local_tvf = enabled;
389+
Map<String, String> properties = readProperties("jfs://cluster/data.csv");
390+
properties.put("fs.defaultFS", "jfs://cluster");
391+
properties.put("fs.jfs.impl", "io.juicefs.JuiceFileSystem");
392+
properties.put("juicefs.cluster.meta", "redis://meta:6379/1");
393+
Assertions.assertDoesNotThrow(() -> new HdfsTableValuedFunction(properties));
394+
properties.put("fs.jfs.impl", LocalFileSystem.class.getName());
395+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
396+
() -> new HdfsTableValuedFunction(properties));
397+
}
398+
}
399+
400+
@Test
401+
void secureWasbDefaultAndDelegatedRoutesKeepRemoteAdmission() throws Exception {
402+
Configuration direct = defaults(Collections.emptyMap());
403+
Assertions.assertEquals("org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure",
404+
FileSystem.getFileSystemClass("wasbs", direct).getName());
405+
direct.set(TvfGuard.POLICY_KEY, TvfGuard.NATIVE_POLICY);
406+
nativeRequest(direct, "wasbs://[email protected]", "/exports/file");
407+
Assertions.assertDoesNotThrow(() -> TvfGuard.check(URI.create("wasbs://[email protected]"), direct));
408+
Configuration delegated = new Configuration(direct);
409+
nativeRequest(delegated, "viewfs://cluster", "/exports/file");
410+
delegated.set("fs.viewfs.mounttable.cluster.link./exports",
411+
"wasbs://[email protected]/base");
412+
Assertions.assertDoesNotThrow(() -> TvfGuard.check(URI.create("viewfs://cluster"), delegated));
413+
delegated.set("fs.wasbs.impl", LocalFileSystem.class.getName());
414+
Assertions.assertThrows(IOException.class, () -> TvfGuard.check(URI.create("viewfs://cluster"), delegated));
415+
}
416+
417+
@Test
418+
void guardedSnapshotDisablesOnlyViewFsInnerCache() throws Exception {
419+
Configuration source = defaults(Collections.emptyMap());
420+
source.setBoolean("fs.viewfs.enable.inner.cache", true);
421+
Configuration frozen = TvfGuard.prepareJavaRead(source, new Path("hdfs://runtime/data.csv"), TvfGuard.NATIVE_POLICY);
422+
Assertions.assertFalse(frozen.getBoolean("fs.viewfs.enable.inner.cache", true));
423+
Assertions.assertTrue(source.getBoolean("fs.viewfs.enable.inner.cache", false));
424+
Assertions.assertSame(source, TvfGuard.prepareJavaRead(source, new Path("file:///unused"), null));
425+
}
426+
427+
@Test
428+
void noGlobBackslashMatchesTheBranchListingConsumer() {
429+
Map<String, String> properties = hdfs("viewfs://cluster");
430+
properties.put("fs.viewfs.mounttable.cluster.link./data", "file:///tmp");
431+
properties.put("fs.viewfs.mounttable.cluster.linkFallback", "hdfs://remote/base");
432+
Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(
433+
HadoopTvfAdmission.Access.HADOOP_READ, "viewfs://cluster/da\\ta/file.csv", properties, false));
434+
}
435+
436+
@Test
437+
void nestedViewFsCaseDistinctTargetsKeepTheirActualIdentityAfterWarmup() throws Exception {
438+
Configuration source = defaults(Collections.emptyMap());
439+
source.set(TvfGuard.POLICY_KEY, TvfGuard.NATIVE_POLICY);
440+
nativeRequest(source, "viewfs://outer", "/warm/file.csv");
441+
source.setBoolean("fs.viewfs.enable.inner.cache", true);
442+
source.set("fs.viewfs.mounttable.outer.link./warm", "viewfs://A/remote");
443+
source.set("fs.viewfs.mounttable.outer.link./evil", "viewfs://a/secret");
444+
source.set("fs.viewfs.mounttable.A.link./remote", "hdfs://nn/base");
445+
source.set("fs.viewfs.mounttable.A.link./secret", "file:///unused-local");
446+
source.set("fs.viewfs.mounttable.a.linkFallback", "hdfs://nn/lower");
447+
java.util.List<String> accesses = new java.util.ArrayList<>();
448+
LocalFileSystem local = Mockito.mock(LocalFileSystem.class);
449+
Mockito.when(local.getUri()).thenReturn(URI.create("file:///"));
450+
Mockito.when(local.getWorkingDirectory()).thenReturn(new Path("/"));
451+
try (MockedStatic<ReflectionUtils> factory = Mockito.mockStatic(
452+
ReflectionUtils.class, Mockito.CALLS_REAL_METHODS)) {
453+
factory.when(() -> ReflectionUtils.newInstance(Mockito.eq(DistributedFileSystem.class),
454+
Mockito.any(Configuration.class))).thenAnswer(invocation -> {
455+
DistributedFileSystem remote = Mockito.mock(DistributedFileSystem.class);
456+
Mockito.when(remote.getConf()).thenReturn(invocation.getArgument(1));
457+
Mockito.when(remote.getUri()).thenReturn(URI.create("hdfs://nn"));
458+
Mockito.when(remote.getWorkingDirectory()).thenReturn(new Path("/"));
459+
Mockito.when(remote.getFileStatus(Mockito.any(Path.class))).thenAnswer(call -> {
460+
Path path = call.getArgument(0);
461+
accesses.add(path.toUri().getPath());
462+
return new org.apache.hadoop.fs.FileStatus(1, false, 1, 1, 0, path);
463+
});
464+
return remote;
465+
});
466+
factory.when(() -> ReflectionUtils.newInstance(Mockito.eq(LocalFileSystem.class),
467+
Mockito.any(Configuration.class))).thenReturn(local);
468+
FileSystem outer = FileSystem.newInstance(URI.create("viewfs://outer"), source);
469+
TvfGuard.Context context = TvfGuard.takeNativeContext();
470+
Assertions.assertNotNull(context);
471+
Assertions.assertFalse(outer.getConf().getBoolean("fs.viewfs.enable.inner.cache", true));
472+
outer.getFileStatus(new Path("/warm/file.csv"));
473+
context.checkNative("viewfs://outer", "/evil/file.csv");
474+
outer.getFileStatus(new Path("/evil/file.csv"));
475+
outer.getFileStatus(new Path("/evil/next.csv"));
476+
Assertions.assertTrue(accesses.contains("/base/file.csv"), accesses.toString());
477+
Assertions.assertTrue(accesses.contains("/lower/secret/file.csv"), accesses.toString());
478+
Assertions.assertTrue(accesses.contains("/lower/secret/next.csv"), accesses.toString());
479+
Mockito.verify(local, Mockito.never()).getFileStatus(Mockito.any(Path.class));
480+
}
481+
}
482+
347483
private Configuration defaults(Map<String, String> entries) throws Exception {
348484
StringBuilder xml = new StringBuilder("<configuration>");
349485
entries.forEach((key, value) -> xml.append("<property><name>").append(key)

0 commit comments

Comments
 (0)