Skip to content

Commit 2832d38

Browse files
committed
[fix](tvf) Preserve literal glob component identity
### What problem does this PR solve? Issue Number: N/A Related PR: #69527, #69528 Problem Summary: Keep GlobFilter pattern identity separate from unescaped literal path components. Encode actual probes as exact literal paths or literal subtree prefixes, retain earlier wildcard parents across dot normalization, and honor each rule mode in both FE and runtime routing. This admits valid escaped-star/question/bracket/brace/backslash paths without weakening real local-parent checks. ### Release note Preserve valid escaped literal filenames and listing prefixes under the remote-only Hadoop TVF policy. ### Check List (For Author) - Test: Unit Test; standard JDK17 FE wrapper, 110 selected tests passed, zero failures/errors/skips, every required report fresh and source frozen. Actual Hadoop Globber and FE/runtime controls included. All 15 Doris Java PR files passed Checkstyle; diff check passed. Complete SQL/Groovy and community CI not claimed. - Behavior changed: Yes. Preserve literal component identity in Hadoop discovery admission. - Does this need documentation: No new setting or interface; existing startup and upgrade guidance applies.
1 parent fe28b42 commit 2832d38

2 files changed

Lines changed: 115 additions & 44 deletions

File tree

‎fe/be-java-extensions/hadoop-deps/src/main/java/org/apache/hadoop/fs/FileSystem.java‎

Lines changed: 48 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -3711,32 +3711,52 @@ public static String encode(List<Rule> rules) {
37113711
}
37123712
}
37133713

3714-
/** Routes touched by Globber, without constructing a filesystem or listing data. */
3714+
/** Literal probes and subtree prefixes touched by Globber, without filesystem IO. */
37153715
public static List<Rule> globRules(URI filesystem, String pattern) throws IOException {
37163716
List<Rule> rules = new ArrayList<>();
37173717
for (String expanded : GlobExpander.expand(pattern)) {
37183718
Path flattened = new Path(expanded.isEmpty() ? Path.CUR_DIR : expanded);
37193719
String path = flattened.toUri().getPath();
3720-
Path candidate = new Path(flattened.isAbsolute() ? "/" : Path.CUR_DIR);
3720+
Path root = new Path(flattened.isAbsolute() ? "/" : Path.CUR_DIR);
3721+
Path candidate = root;
3722+
List<String> components = new ArrayList<>();
3723+
List<Boolean> patterned = new ArrayList<>();
37213724
for (String component : path.split("/")) {
37223725
if (component.isEmpty()) {
37233726
continue;
37243727
}
37253728
GlobFilter filter = new GlobFilter(component);
37263729
if (filter.hasPattern()) {
3727-
// Globber lists this parent before it knows the matching child names. Keep
3728-
// prior wildcard components as a conservative subtree route, including when
3729-
// a later escaped '..' moves the final candidate outside that subtree.
3730-
rules.add(new Rule(filesystem.toString(), new Path(candidate, "*").toUri().getPath(), true));
3730+
// Keep every real listing parent, even if later escaped '..' removes it.
3731+
rules.add(new Rule(filesystem.toString(),
3732+
globLiteralPrefix(root, components, patterned).toUri().getPath(), true));
37313733
}
37323734
component = component.replaceAll("\\\\(.)", "$1");
37333735
candidate = new Path(candidate, component);
3736+
if ("..".equals(component) && !components.isEmpty()
3737+
&& !"..".equals(components.get(components.size() - 1))) {
3738+
components.remove(components.size() - 1);
3739+
patterned.remove(patterned.size() - 1);
3740+
} else if (!".".equals(component)) {
3741+
components.add(component);
3742+
patterned.add(filter.hasPattern());
3743+
}
37343744
}
3735-
rules.add(new Rule(filesystem.toString(), candidate.toUri().getPath(), true));
3745+
boolean subtree = patterned.contains(Boolean.TRUE);
3746+
Path probe = subtree ? globLiteralPrefix(root, components, patterned) : candidate;
3747+
rules.add(new Rule(filesystem.toString(), probe.toUri().getPath(), subtree));
37363748
}
37373749
return rules;
37383750
}
37393751

3752+
private static Path globLiteralPrefix(Path root, List<String> components, List<Boolean> patterned) {
3753+
Path prefix = root;
3754+
for (int i = 0; i < components.size() && !patterned.get(i); i++) {
3755+
prefix = new Path(prefix, components.get(i));
3756+
}
3757+
return prefix;
3758+
}
3759+
37403760
/** Java requests own their mount clients; native connections keep their existing cache. */
37413761
private static final class RequestViewFileSystem extends org.apache.hadoop.fs.viewfs.ViewFileSystem {
37423762
private final Map<List<String>, FileSystem> children = new HashMap<>();
@@ -4060,6 +4080,16 @@ private EffectiveRoute(Map<String, String> properties, boolean descendants, Conf
40604080
this.conf = conf;
40614081
}
40624082

4083+
private EffectiveRoute(EffectiveRoute original, boolean descendants) {
4084+
super(original, descendants);
4085+
conf = original.conf;
4086+
}
4087+
4088+
@Override
4089+
protected Route withDescendants(boolean value) {
4090+
return new EffectiveRoute(this, value);
4091+
}
4092+
40634093
@Override
40644094
protected String property(String key) {
40654095
return conf.get(key);
@@ -4110,6 +4140,14 @@ protected enum Provider {
41104140
private final boolean descendants;
41114141
private final Map<String, String> properties;
41124142

4143+
protected Route(Route original, boolean descendants) {
4144+
this(original.properties, descendants);
4145+
}
4146+
4147+
protected Route withDescendants(boolean value) {
4148+
return new Route(this, value);
4149+
}
4150+
41134151
public Route(Map<String, String> properties, boolean descendants) {
41144152
// Both callers supply a statement-local snapshot that is never mutated afterward.
41154153
this.properties = properties;
@@ -4158,7 +4196,7 @@ public void checkRoute(URI filesystem, String path, Set<String> activeTables) th
41584196
public void checkGlobRoute(URI filesystem, String pattern, Set<String> activeTables)
41594197
throws IOException, URISyntaxException {
41604198
for (Rule rule : globRules(filesystem, pattern)) {
4161-
checkRoute(filesystem, rule.path, activeTables);
4199+
withDescendants(rule.descendants).checkRoute(filesystem, rule.path, activeTables);
41624200
}
41634201
}
41644202

@@ -4210,7 +4248,7 @@ private void checkGlobTargets(String key, String prefix, Set<String> activeTable
42104248
// Listing can touch any descendant of a selected mount. Reuse the same route
42114249
// interpreter for nested tables instead of assuming their root is representative.
42124250
String path = uri.getPath() == null ? "/" : uri.getPath();
4213-
checkRoute(uri, (path.endsWith("/") ? path : path + "/") + "*", activeTables);
4251+
checkRoute(uri, path, activeTables);
42144252
}
42154253
}
42164254

@@ -4291,26 +4329,7 @@ private void checkViewFsEnabled(URI filesystem, String path, Set<String> activeT
42914329
selectedKeys.add(key);
42924330
}
42934331
}
4294-
if (descendants && hasGlob(path)) {
4295-
String literal = path.substring(0, firstGlob(path));
4296-
int slash = literal.lastIndexOf('/');
4297-
literal = slash < 0 ? "/" : literal.substring(0, slash + 1);
4298-
for (String key : properties.keySet()) {
4299-
if (!key.startsWith(prefix)) {
4300-
continue;
4301-
}
4302-
String entry = key.substring(prefix.length());
4303-
int sourceStart = entry.indexOf('/');
4304-
if (sourceStart >= 0 && (entry.startsWith("link.") || entry.startsWith("linkMerge.")
4305-
|| entry.startsWith("linkNfly."))) {
4306-
String mount = entry.substring(sourceStart);
4307-
if (mount.startsWith(literal) || literal.startsWith(mount + "/")) {
4308-
checkGlobTargets(key, prefix, activeTables);
4309-
}
4310-
}
4311-
}
4312-
}
4313-
if (descendants && !hasGlob(path)) {
4332+
if (descendants) {
43144333
String subtree = path.endsWith("/") ? path : path + "/";
43154334
for (String key : properties.keySet()) {
43164335
if (key.startsWith(prefix)) {
@@ -4383,21 +4402,6 @@ private String selector(String value) {
43834402
return value;
43844403
}
43854404

4386-
private static int firstGlob(String path) {
4387-
int first = path.length();
4388-
for (char marker : new char[] {'*', '?', '[', '{'}) {
4389-
int index = path.indexOf(marker);
4390-
if (index >= 0) {
4391-
first = Math.min(first, index);
4392-
}
4393-
}
4394-
return first;
4395-
}
4396-
4397-
private static boolean hasGlob(String path) {
4398-
return firstGlob(path) < path.length();
4399-
}
4400-
44014405
protected void reject() {
44024406
throw new IllegalArgumentException("Hadoop TVF requires a proven remote filesystem route");
44034407
}

‎fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfRuntimeAdmissionTest.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -741,6 +741,73 @@ void discoveryOnlyChecksNormalizedProbesWhileReturnedPathsRemainLiteral() throws
741741
}
742742
}
743743

744+
@Test
745+
void escapedMetacharactersKeepLiteralIdentityBeforeAndAfterRealWildcards() throws Exception {
746+
Map<String, String> properties = hdfs("viewfs://literal");
747+
properties.put("fs.viewfs.mounttable.literal.link./data", "hdfs://nn/base");
748+
properties.put("fs.viewfs.mounttable.literal.link./data/private", "file:///unused-local");
749+
properties.put("fs.viewfs.mounttable.literal.linkFallback", "hdfs://nn/fallback");
750+
String[][] paths = {
751+
{"/data/\\*/file.csv", "/data/*/file.csv"},
752+
{"/data/\\*/p?obe.csv", "/data/*/probe.csv"},
753+
{"/data/\\?/p?obe.csv", "/data/?/probe.csv"},
754+
{"/data/\\[x\\]/p?obe.csv", "/data/[x]/probe.csv"},
755+
{"/data/\\{x\\}/p?obe.csv", "/data/{x}/probe.csv"},
756+
{"/data/literal\\\\name/p?obe.csv", "/data/literal\\name/probe.csv"}
757+
};
758+
for (boolean enabled : new boolean[] {false, true}) {
759+
Config.enable_local_tvf = enabled;
760+
try (RemoteClients clients = new RemoteClients()) {
761+
int baseline = filesystemCacheSize();
762+
for (String[] entry : paths) {
763+
String uri = "viewfs://literal" + entry[0];
764+
Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.checkReadDiscovery(uri, properties), uri);
765+
Configuration request = javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, uri);
766+
try (FileSystem filesystem = FileSystem.get(URI.create("viewfs://literal"), request)) {
767+
org.apache.hadoop.fs.FileStatus[] matches = filesystem.globStatus(new Path(uri));
768+
Assertions.assertEquals(1, matches.length, uri + clients.events);
769+
Assertions.assertEquals(entry[1], matches[0].getPath().toUri().getPath(), uri);
770+
Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(
771+
HadoopTvfAdmission.Access.HADOOP_READ,
772+
matches[0].getPath().toString(), properties, false));
773+
}
774+
clients.verifyClosedOnce();
775+
Assertions.assertEquals(baseline, filesystemCacheSize());
776+
}
777+
}
778+
String actualWildcard = "viewfs://literal/data/*/file.csv";
779+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
780+
() -> HadoopTvfAdmission.checkReadDiscovery(actualWildcard, properties));
781+
Assertions.assertThrows(IOException.class, () -> TvfGuard.check(URI.create("viewfs://literal"),
782+
javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, actualWildcard)));
783+
properties.put("fs.viewfs.mounttable.literal.link./data/*", "file:///unused-local");
784+
String selectedLocal = "viewfs://literal/data/\\*/file.csv";
785+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
786+
() -> HadoopTvfAdmission.checkReadDiscovery(selectedLocal, properties));
787+
Assertions.assertThrows(IOException.class, () -> TvfGuard.check(URI.create("viewfs://literal"),
788+
javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, selectedLocal)));
789+
properties.remove("fs.viewfs.mounttable.literal.link./data/*");
790+
}
791+
}
792+
793+
@Test
794+
void escapedParentNormalizationRetainsEarlierWildcardProbes() throws Exception {
795+
Map<String, String> properties = hdfs("viewfs://parents");
796+
properties.put("fs.viewfs.mounttable.parents.link./data", "hdfs://nn/base");
797+
properties.put("fs.viewfs.mounttable.parents.link./other", "hdfs://nn/other");
798+
properties.put("fs.viewfs.mounttable.parents.link./other/private", "file:///unused-local");
799+
properties.put("fs.viewfs.mounttable.parents.linkFallback", "hdfs://nn/fallback");
800+
String uri = "viewfs://parents/data/*/\\.\\./\\.\\./other/\\*/p?obe.csv";
801+
Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.checkReadDiscovery(uri, properties));
802+
Assertions.assertDoesNotThrow(() -> TvfGuard.check(URI.create("viewfs://parents"),
803+
javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, uri)));
804+
properties.put("fs.viewfs.mounttable.parents.link./data/private", "file:///unused-local");
805+
Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class,
806+
() -> HadoopTvfAdmission.checkReadDiscovery(uri, properties));
807+
Assertions.assertThrows(IOException.class, () -> TvfGuard.check(URI.create("viewfs://parents"),
808+
javaRequest(properties, HadoopTvfAdmission.Access.HADOOP_READ, uri)));
809+
}
810+
744811
private Configuration javaRequest(Map<String, String> properties, HadoopTvfAdmission.Access access,
745812
String path) throws Exception {
746813
Configuration configuration = defaults(Collections.emptyMap());

0 commit comments

Comments
 (0)