Skip to content

Commit b1fb0a3

Browse files
committed
[fix](tvf) Allow finite ViewFS alias routes
1 parent 95b42fa commit b1fb0a3

2 files changed

Lines changed: 131 additions & 14 deletions

File tree

‎fe/be-java-extensions/hadoop-deps/src/main/java/org/apache/hadoop/fs/FileSystem.java‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4152,32 +4152,32 @@ private List<URI> defaults() throws URISyntaxException {
41524152
return defaults;
41534153
}
41544154

4155-
public void checkRoute(URI filesystem, String path, Set<String> activeTables) throws URISyntaxException {
4155+
public void checkRoute(URI filesystem, String path, Set<String> activeRoutes) throws URISyntaxException {
41564156
if (filesystem.getScheme() == null) {
41574157
for (URI defaultFs : defaults()) {
4158-
checkViewFsEnabled(defaultFs, path, activeTables);
4158+
checkViewFsEnabled(defaultFs, path, activeRoutes);
41594159
}
41604160
return;
41614161
}
41624162
if (filesystem.getAuthority() == null) {
41634163
boolean completed = false;
41644164
for (URI defaultFs : defaults()) {
41654165
if (filesystem.getScheme().equals(defaultFs.getScheme()) && defaultFs.getAuthority() != null) {
4166-
checkViewFsEnabled(defaultFs, path, activeTables);
4166+
checkViewFsEnabled(defaultFs, path, activeRoutes);
41674167
completed = true;
41684168
}
41694169
}
41704170
if (completed) {
41714171
return;
41724172
}
41734173
}
4174-
checkViewFsEnabled(filesystem, path, activeTables);
4174+
checkViewFsEnabled(filesystem, path, activeRoutes);
41754175
}
41764176

4177-
public void checkGlobRoute(URI filesystem, String pattern, Set<String> activeTables)
4177+
public void checkGlobRoute(URI filesystem, String pattern, Set<String> activeRoutes)
41784178
throws IOException, URISyntaxException {
41794179
for (Rule rule : globRules(filesystem, pattern)) {
4180-
withDescendants(rule.descendants).checkRoute(filesystem, rule.path, activeTables);
4180+
withDescendants(rule.descendants).checkRoute(filesystem, rule.path, activeRoutes);
41814181
}
41824182
}
41834183

@@ -4220,7 +4220,7 @@ private String[] targets(String key, String prefix) {
42204220
return multiple ? value.split(",", -1) : new String[] {value};
42214221
}
42224222

4223-
private void checkGlobTargets(String key, String prefix, Set<String> activeTables) throws URISyntaxException {
4223+
private void checkGlobTargets(String key, String prefix, Set<String> activeRoutes) throws URISyntaxException {
42244224
for (String target : targets(key, prefix)) {
42254225
if (target.isEmpty()) {
42264226
reject();
@@ -4229,11 +4229,11 @@ private void checkGlobTargets(String key, String prefix, Set<String> activeTable
42294229
// Listing can touch any descendant of a selected mount. Reuse the same route
42304230
// interpreter for nested tables instead of assuming their root is representative.
42314231
String path = uri.getPath() == null ? "/" : uri.getPath();
4232-
checkRoute(uri, path, activeTables);
4232+
checkRoute(uri, path, activeRoutes);
42334233
}
42344234
}
42354235

4236-
private void checkViewFsEnabled(URI filesystem, String path, Set<String> activeTables) {
4236+
private void checkViewFsEnabled(URI filesystem, String path, Set<String> activeRoutes) {
42374237
String scheme = filesystem.getScheme();
42384238
Provider provider = provider(scheme);
42394239
if (provider == Provider.LOCAL || provider == Provider.UNKNOWN || provider == Provider.DYNAMIC) {
@@ -4260,8 +4260,10 @@ private void checkViewFsEnabled(URI filesystem, String path, Set<String> activeT
42604260
table = property("fs.viewfs.mounttable.default.name.key");
42614261
table = table == null ? "default" : selector(table);
42624262
}
4263-
// Nested ViewFS cycles or arbitrarily deep delegation cannot establish remote storage.
4264-
if (activeTables.size() >= 32 || !activeTables.add(table)) {
4263+
// A finite alias may revisit a table at a different path. Prefix the table length
4264+
// to keep route identities unambiguous, and bound depth even when paths keep growing.
4265+
String route = table.length() + ":" + table + path;
4266+
if (activeRoutes.size() >= 32 || !activeRoutes.add(route)) {
42654267
reject();
42664268
}
42674269
try {
@@ -4317,7 +4319,7 @@ private void checkViewFsEnabled(URI filesystem, String path, Set<String> activeT
43174319
String entry = key.substring(prefix.length());
43184320
int sourceStart = entry.indexOf('/');
43194321
if (sourceStart >= 0 && entry.substring(sourceStart).startsWith(subtree)) {
4320-
checkGlobTargets(key, prefix, activeTables);
4322+
checkGlobTargets(key, prefix, activeRoutes);
43214323
}
43224324
}
43234325
}
@@ -4362,13 +4364,13 @@ private void checkViewFsEnabled(URI filesystem, String path, Set<String> activeT
43624364
(base.endsWith("/") ? base : base + "/")
43634365
+ (remaining.startsWith("/") ? remaining.substring(1) : remaining), null)
43644366
.normalize().getPath();
4365-
checkRoute(uri, nextPath, activeTables);
4367+
checkRoute(uri, nextPath, activeRoutes);
43664368
}
43674369
}
43684370
} catch (URISyntaxException e) {
43694371
reject();
43704372
} finally {
4371-
activeTables.remove(table);
4373+
activeRoutes.remove(route);
43724374
}
43734375
}
43744376

‎fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfRuntimeAdmissionTest.java‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -387,6 +387,121 @@ void globTraversalChecksIntermediateParentsAndEscapedDotSegments() {
387387
"viewfs://cluster/literal\\name/file.csv", properties));
388388
}
389389

390+
@Test
391+
void unguardedSameTableViewFsAliasUsesLazyRemoteDispatch() throws Exception {
392+
Map<String, String> properties = hdfs("viewfs://cluster");
393+
properties.put("fs.viewfs.mounttable.cluster.link./a", "viewfs://cluster/b");
394+
properties.put("fs.viewfs.mounttable.cluster.link./b", "hdfs://nn/base");
395+
Configuration runtime = defaults(properties);
396+
try (RemoteClients clients = new RemoteClients()) {
397+
try (FileSystem outer = FileSystem.newInstance(URI.create("viewfs://cluster"), runtime)) {
398+
outer.getFileStatus(new Path("/a/file.csv"));
399+
Assertions.assertTrue(clients.events.contains("stat:0:/base/file.csv"));
400+
} finally {
401+
FileSystem.closeAll();
402+
}
403+
clients.verifyClosedOnce();
404+
}
405+
}
406+
407+
@Test
408+
void guardedFiniteViewFsAliasesPreserveReadsWritesAndCleanup() throws Exception {
409+
String uri = "viewfs://cluster/a/file.csv";
410+
Map<String, String> properties = readProperties(uri);
411+
properties.put("fs.defaultFS", "viewfs://cluster");
412+
properties.put("fs.viewfs.mounttable.cluster.link./a", "viewfs://cluster/b");
413+
properties.put("fs.viewfs.mounttable.cluster.link./b", "hdfs://nn/base");
414+
for (boolean enabled : new boolean[] {false, true}) {
415+
Config.enable_local_tvf = enabled;
416+
Assertions.assertDoesNotThrow(() -> new HdfsTableValuedFunction(properties));
417+
Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.checkReadDiscovery(
418+
"viewfs://cluster/a/*.csv", properties));
419+
}
420+
Config.enable_local_tvf = false;
421+
Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(
422+
HadoopTvfAdmission.Access.NATIVE_WRITE, "/a/prefix_", properties));
423+
Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(
424+
HadoopTvfAdmission.Access.JAVA_DELETE, uri, properties));
425+
try (RemoteClients clients = new RemoteClients()) {
426+
try {
427+
Configuration read = TvfGuard.prepareJavaRead(defaults(properties),
428+
new Path(uri), TvfGuard.NATIVE_POLICY);
429+
try (FileSystem outer = FileSystem.newInstance(URI.create("viewfs://cluster"), read)) {
430+
outer.getFileStatus(new Path(uri));
431+
Assertions.assertTrue(clients.events.stream().anyMatch(
432+
event -> event.startsWith("stat:") && event.endsWith(":/base/file.csv")));
433+
}
434+
Configuration write = defaults(properties);
435+
overlay(write, HadoopTvfAdmission.protectNativeWrite("/a/prefix_", properties));
436+
nativeRequest(write, "viewfs://cluster", "/a/prefix_");
437+
TvfGuard.clearNativeContext();
438+
try (FileSystem outer = FileSystem.newInstance(URI.create("viewfs://cluster"), write)) {
439+
TvfGuard.Context context = TvfGuard.takeNativeContext();
440+
Assertions.assertNotNull(context);
441+
Assertions.assertDoesNotThrow(() -> context.checkNative("viewfs://cluster", "/a/next_"));
442+
}
443+
try (FileSystem outer = FileSystem.newInstance(URI.create("viewfs://cluster"),
444+
javaRequest(properties, HadoopTvfAdmission.Access.JAVA_DELETE, uri))) {
445+
Assertions.assertTrue(outer.delete(new Path(uri), true));
446+
Assertions.assertTrue(clients.events.stream().anyMatch(
447+
event -> event.startsWith("delete:") && event.endsWith(":/base/file.csv")));
448+
}
449+
} finally {
450+
FileSystem.closeAll();
451+
}
452+
clients.verifyClosedOnce();
453+
}
454+
}
455+
456+
@Test
457+
void guardedViewFsAliasesStillRejectLocalTargetsAndCycles() throws Exception {
458+
String uri = "viewfs://cluster/a/file.csv";
459+
for (String target : new String[] {"file:///unused-local", "viewfs://cluster/a",
460+
"viewfs://cluster/b/grow"}) {
461+
Map<String, String> properties = readProperties(uri);
462+
properties.put("fs.defaultFS", "viewfs://cluster");
463+
properties.put("fs.viewfs.mounttable.cluster.link./a", "viewfs://cluster/b");
464+
properties.put("fs.viewfs.mounttable.cluster.link./b", target);
465+
for (boolean enabled : new boolean[] {false, true}) {
466+
Config.enable_local_tvf = enabled;
467+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
468+
() -> new HdfsTableValuedFunction(properties), target);
469+
}
470+
Config.enable_local_tvf = false;
471+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
472+
() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE,
473+
"/a/prefix_", properties), target);
474+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
475+
() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, uri, properties), target);
476+
Configuration runtime = defaults(properties);
477+
Assertions.assertThrows(IOException.class, () -> TvfGuard.prepareJavaRead(
478+
runtime, new Path(uri), TvfGuard.NATIVE_POLICY), target);
479+
runtime.set(TvfGuard.POLICY_KEY, TvfGuard.NATIVE_WRITE_POLICY);
480+
nativeRequest(runtime, "viewfs://cluster", "/a/prefix_");
481+
try (RemoteClients clients = new RemoteClients()) {
482+
Assertions.assertThrows(IOException.class,
483+
() -> FileSystem.newInstance(URI.create("viewfs://cluster"), runtime), target);
484+
Assertions.assertTrue(clients.clients.isEmpty());
485+
}
486+
}
487+
}
488+
489+
@Test
490+
void finiteSameTableAliasChainsRetainTheDelegationDepthBound() {
491+
Map<String, String> properties = hdfs("viewfs://cluster");
492+
for (int i = 0; i < 31; i++) {
493+
properties.put("fs.viewfs.mounttable.cluster.link./a" + i, "viewfs://cluster/a" + (i + 1));
494+
}
495+
properties.put("fs.viewfs.mounttable.cluster.link./a31", "hdfs://nn/base");
496+
Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(
497+
HadoopTvfAdmission.Access.JAVA_DELETE, "viewfs://cluster/a0/file.csv", properties));
498+
properties.put("fs.viewfs.mounttable.cluster.link./a31", "viewfs://cluster/a32");
499+
properties.put("fs.viewfs.mounttable.cluster.link./a32", "hdfs://nn/base");
500+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
501+
() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE,
502+
"viewfs://cluster/a0/file.csv", properties));
503+
}
504+
390505
@Test
391506
void guardedJuiceFsRequiresProvenRemoteBackingStore() throws Exception {
392507
for (boolean enabled : new boolean[] {false, true}) {

0 commit comments

Comments
 (0)