Skip to content

Commit 398acc1

Browse files
committed
[fix](tvf) Close LOCAL admission and path review gaps
### What problem does this PR solve? Issue Number: N/A Related PR: #69528 Problem Summary: Apply the startup-only LOCAL gate to INSERT and EXPLAIN INSERT. Escape the canonical secure root as a literal glob prefix. Validate raw S3 Lance ports and IPv6 before SDK access; decode file URI paths once after removing query and fragment while preserving bare paths. ### Release note LOCAL TVF INSERT and EXPLAIN INSERT honor enable_local_tvf. Secure directories containing glob metacharacters remain usable. LOCAL Lance preserves percent-encoded file URI identity; malformed remote Lance authorities are rejected before SDK access. ### Check List (For Author) - Test: Unit Test: 35 FE and 11 BE selected tests passed; relevant Java Checkstyle, clang-format 16, and git diff --check passed. Regression assertions added, full Groovy not run. - Behavior changed: Yes, enforce the existing LOCAL startup setting at the write entry and preserve valid local path identity. - Does this need documentation: Yes, the original PR release note describes the startup opt-in requirement; documentation PR remains pending.
1 parent c4103b6 commit 398acc1

9 files changed

Lines changed: 251 additions & 9 deletions

File tree

‎be/src/io/fs/local_file_system.cpp‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -466,6 +466,38 @@ Status LocalFileSystem::canonicalize_local_file(const std::string& dir,
466466
Path local_path;
467467
// Convert file: URIs to filesystem paths and reject remote schemes before any reader opens them.
468468
RETURN_IF_ERROR(convert_to_abs_path(file_path, local_path));
469+
if (file_path.rfind("file:", 0) == 0) {
470+
// file:a%20b names "a b", but a bare path containing "%20" must stay literal.
471+
// Decode once, preserve '+' and invalid escapes, matching Lance's file provider.
472+
auto encoded = local_path.string();
473+
// Query and fragment are URI components, not part of the filesystem path.
474+
encoded = encoded.substr(0, encoded.find_first_of("?#"));
475+
const auto hex_digit = [](char c) -> int {
476+
if (c >= '0' && c <= '9') return c - '0';
477+
if (c >= 'a' && c <= 'f') return c - 'a' + 10;
478+
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
479+
return -1;
480+
};
481+
std::string decoded;
482+
decoded.reserve(encoded.size());
483+
for (size_t i = 0; i < encoded.size(); ++i) {
484+
if (encoded[i] == '%' && i + 2 < encoded.size()) {
485+
const int high = hex_digit(encoded[i + 1]);
486+
const int low = hex_digit(encoded[i + 2]);
487+
if (high >= 0 && low >= 0) {
488+
decoded += static_cast<char>((high << 4) | low);
489+
i += 2;
490+
continue;
491+
}
492+
}
493+
decoded += encoded[i];
494+
}
495+
// %00 must not truncate the path when canonicalization reaches C filesystem APIs.
496+
if (decoded.find('\0') != std::string::npos) {
497+
return Status::InvalidArgument("local file path must not contain NUL");
498+
}
499+
local_path = std::move(decoded);
500+
}
469501
std::string secure_path;
470502
RETURN_IF_ERROR(canonicalize(dir, &secure_path));
471503
std::string canonical_path;
@@ -489,7 +521,16 @@ Status LocalFileSystem::safe_glob(const std::string& path, std::vector<FileInfo>
489521
}
490522
std::string secure_path;
491523
RETURN_IF_ERROR(canonicalize(config::user_files_secure_path, &secure_path));
492-
std::string full_path = secure_path + "/" + path;
524+
// A symlink root may canonicalize to a directory such as "user_files[prod]".
525+
// The root is literal; only the user-supplied suffix may contain glob operators.
526+
std::string full_path;
527+
for (char c : secure_path) {
528+
if (c == '\\' || c == '*' || c == '?' || c == '[' || c == ']') {
529+
full_path += '\\';
530+
}
531+
full_path += c;
532+
}
533+
full_path += "/" + path;
493534
std::vector<std::string> files;
494535
RETURN_IF_ERROR(_glob(full_path, &files));
495536
for (auto& file : files) {

‎be/test/format_v2/table/lance_reader_test.cpp‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4387,6 +4387,69 @@ TEST(LanceTableReaderLocalPathTest, ScansRelativeDatasetInsideSecureRoot) {
43874387
ASSERT_TRUE(reader.close().ok());
43884388
}
43894389

4390+
TEST(LanceTableReaderLocalPathTest, FileUrisPreserveEscapedDatasetIdentity) {
4391+
const auto original_secure_path = config::user_files_secure_path;
4392+
DEFER(config::user_files_secure_path = original_secure_path);
4393+
const auto root = std::filesystem::temp_directory_path() /
4394+
("lance_escaped_uri_" +
4395+
std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()));
4396+
DEFER({
4397+
std::error_code error;
4398+
std::filesystem::remove_all(root, error);
4399+
});
4400+
const auto fixture =
4401+
std::filesystem::canonical("./be/test/format_v2/table/lance/data/all_types.lance");
4402+
for (const auto* name : {"a b.lance", "a%20b.lance", "plus+percent%.lance"}) {
4403+
std::filesystem::create_directories(root / name);
4404+
std::filesystem::copy(fixture, root / name, std::filesystem::copy_options::recursive);
4405+
}
4406+
config::user_files_secure_path = root.string();
4407+
for (const auto& input :
4408+
{std::make_pair("file://" + (root / "a%20b.lance").string(), root / "a b.lance"),
4409+
std::make_pair(std::string("a%20b.lance"), root / "a%20b.lance"),
4410+
std::make_pair("file://" + (root / "a%20b.lance").string() + "?q=%00#tag",
4411+
root / "a b.lance"),
4412+
std::make_pair("file://" + (root / "a%2520b.lance").string(), root / "a%20b.lance"),
4413+
std::make_pair("file://" + (root / "plus+percent%25.lance").string(),
4414+
root / "plus+percent%.lance")}) {
4415+
TQueryGlobals query_globals;
4416+
RuntimeState state(query_globals);
4417+
RuntimeProfile profile("lance_escaped_file_uri");
4418+
TFileScanRangeParams scan_params;
4419+
LanceTableReader reader;
4420+
const Columns columns = {projected_column("row_id", TYPE_BIGINT, false)};
4421+
ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok());
4422+
auto range = make_latest_lance_range(input.first);
4423+
range.__set_file_type(TFileType::FILE_LOCAL);
4424+
std::vector<std::string> names;
4425+
std::vector<DataTypePtr> types;
4426+
ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok()) << input.first;
4427+
EXPECT_FALSE(names.empty());
4428+
LanceTableReader::DatasetKey key;
4429+
ASSERT_TRUE(reader._dataset_key(range, &key).ok());
4430+
EXPECT_EQ(input.second.string(), key.uri);
4431+
ASSERT_TRUE(prepare_range(&reader, range).ok());
4432+
Block block;
4433+
add_output_columns(&block, columns);
4434+
std::vector<int64_t> row_ids;
4435+
bool eos = false;
4436+
while (!eos) {
4437+
ASSERT_TRUE(reader.get_block(&block, &eos).ok());
4438+
if (!eos) {
4439+
const auto& values =
4440+
assert_cast<const ColumnInt64&>(*block.get_by_position(0).column)
4441+
.get_data();
4442+
for (size_t row = 0; row < block.rows(); ++row) {
4443+
row_ids.push_back(values[row]);
4444+
}
4445+
}
4446+
}
4447+
std::sort(row_ids.begin(), row_ids.end());
4448+
EXPECT_EQ((std::vector<int64_t> {1, 2, 3, 4}), row_ids);
4449+
ASSERT_TRUE(reader.close().ok());
4450+
}
4451+
}
4452+
43904453
TEST(LanceTableReaderLocalPathTest, FixedVersionLocalCatalogKeepsItsUri) {
43914454
TFileScanRangeParams scan_params;
43924455
LanceTableReader reader;

‎be/test/io/fs/local_file_system_test.cpp‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -643,6 +643,17 @@ TEST_F(LocalFileSystemTest, TestSafeGlobSecureRoot) {
643643
ASSERT_EQ(1, files.size());
644644
EXPECT_EQ((root / "secure/inside.csv").string(), files[0].file_name);
645645
}
646+
const auto special_root = root / "secure[prod]*?\\root";
647+
std::filesystem::create_directory(special_root);
648+
std::filesystem::create_directory_symlink(special_root, root / "special-root-link");
649+
ASSERT_TRUE(save_string_file((special_root / "inside.csv").string(), "inside").ok());
650+
for (const auto& secure : {special_root, root / "special-root-link"}) {
651+
config::user_files_secure_path = secure.string();
652+
std::vector<io::FileInfo> matches;
653+
ASSERT_TRUE(fs->safe_glob("*.csv", &matches).ok()) << secure;
654+
ASSERT_EQ(1, matches.size());
655+
EXPECT_EQ((special_root / "inside.csv").string(), matches[0].file_name);
656+
}
646657
config::user_files_secure_path = "";
647658
std::vector<io::FileInfo> files;
648659
EXPECT_FALSE(fs->safe_glob("*.csv", &files).ok());
@@ -675,6 +686,34 @@ TEST_F(LocalFileSystemTest, TestCanonicalizeSecureLocalPath) {
675686
EXPECT_FALSE(status.ok()) << path;
676687
EXPECT_EQ(std::string::npos, status.to_string().find(root.string())) << status;
677688
}
689+
// URI escapes and bare '%' paths may name two different, simultaneously existing datasets.
690+
for (const auto* name : {"a b.lance", "a%20b.lance", "plus+percent%.lance", "bad%2G.lance"}) {
691+
std::filesystem::create_directory(secure / name);
692+
}
693+
for (const auto& input :
694+
{std::make_pair("file://" + (secure / "a%20b.lance").string(), secure / "a b.lance"),
695+
std::make_pair(std::string("a%20b.lance"), secure / "a%20b.lance"),
696+
std::make_pair("file://" + (secure / "a%20b.lance").string() + "#tag",
697+
secure / "a b.lance"),
698+
std::make_pair("file://" + (secure / "a%20b.lance").string() + "?q=%00#tag",
699+
secure / "a b.lance"),
700+
std::make_pair("file://" + (secure / "a%2520b.lance").string(), secure / "a%20b.lance"),
701+
std::make_pair("file://" + (secure / "plus+percent%25.lance").string(),
702+
secure / "plus+percent%.lance"),
703+
std::make_pair("file://" + (secure / "bad%2G.lance").string(),
704+
secure / "bad%2G.lance")}) {
705+
std::string actual;
706+
ASSERT_TRUE(fs->canonicalize_local_file(secure.string(), input.first, &actual).ok())
707+
<< input.first;
708+
EXPECT_EQ(input.second.string(), actual);
709+
}
710+
for (const auto& suffix :
711+
{"/%2e%2e/secure-other", "/nested%2f..%2f..%2fsecure-other", "/a%00b.lance"}) {
712+
std::string actual;
713+
EXPECT_FALSE(fs->canonicalize_local_file(secure.string(),
714+
"file://" + secure.string() + suffix, &actual)
715+
.ok());
716+
}
678717
std::string resolved;
679718
EXPECT_FALSE(fs->canonicalize_local_file("", secure.string(), &resolved).ok());
680719
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@
116116
import org.apache.doris.qe.AutoCloseSessionVariable;
117117
import org.apache.doris.qe.ConnectContext;
118118
import org.apache.doris.qe.SessionVariable;
119+
import org.apache.doris.tablefunction.LocalTableValuedFunction;
119120
import org.apache.doris.thrift.TPartialUpdateNewRowPolicy;
120121

121122
import com.google.common.base.Preconditions;
@@ -659,6 +660,11 @@ private Plan bindTVFTableSink(MatchingContext<UnboundTVFTableSink<Plan>> ctx) {
659660
"INSERT INTO TVF only supports local/s3/hdfs, but got: " + tvfName);
660661
}
661662

663+
// INSERT/EXPLAIN INSERT bind a sink without constructing the LOCAL read TVF.
664+
if (tvfName.equals("local")) {
665+
LocalTableValuedFunction.checkEnabled();
666+
}
667+
662668
// Validate required properties
663669
if (!properties.containsKey("file_path")) {
664670
throw new AnalysisException("TVF sink requires 'file_path' property");

‎fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,7 @@ public class LocalTableValuedFunction extends ExternalFileTableValuedFunction {
6464
private boolean sharedStorage = false;
6565

6666
public LocalTableValuedFunction(Map<String, String> properties) throws AnalysisException {
67-
if (!Config.enable_local_tvf) {
68-
throw new org.apache.doris.nereids.exceptions.AnalysisException(
69-
"LOCAL TVF is disabled. Set enable_local_tvf=true in fe.conf and restart every FE to enable it.");
70-
}
67+
checkEnabled();
7168

7269
// Check privileges before listing files or discovering their schema.
7370
checkAuth(ConnectContext.get());
@@ -146,6 +143,14 @@ private void getFileListFromBackend() throws AnalysisException {
146143
}
147144
}
148145

146+
/** Check read and write admission before properties or backend access. */
147+
public static void checkEnabled() {
148+
if (!Config.enable_local_tvf) {
149+
throw new org.apache.doris.nereids.exceptions.AnalysisException(
150+
"LOCAL TVF is disabled. Set enable_local_tvf=true in fe.conf and restart every FE to enable it.");
151+
}
152+
}
153+
149154
@Override
150155
public void checkAuth(ConnectContext ctx) {
151156
if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.ADMIN)

‎fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030

3131
import com.google.common.collect.ImmutableSet;
3232
import com.google.common.net.HostAndPort;
33+
import com.google.common.net.InetAddresses;
3334

3435
import java.net.URI;
3536
import java.net.URISyntaxException;
@@ -123,10 +124,21 @@ private static boolean isRemoteLanceUri(String path) {
123124
// Lance falls back to a local path if Rust's URL parser rejects the URI. Java's URI
124125
// accepts registry authorities with invalid ports, which Rust rejects. Validate the
125126
// host/port separately without disallowing legitimate bucket names with underscores.
126-
String host = HostAndPort.fromString(authority.substring(authority.lastIndexOf('@') + 1))
127-
.requireBracketsForIPv6().getHost();
128-
// Java accepts IPv6 zone identifiers, while Rust's URL parser rejects them.
129-
return !host.isEmpty() && !(host.contains(":") && host.contains("%"));
127+
String hostPort = authority.substring(authority.lastIndexOf('@') + 1);
128+
HostAndPort parsed = HostAndPort.fromString(hostPort).requireBracketsForIPv6();
129+
if (parsed.hasPort()) {
130+
// Guava parses "-0" as zero, but Rust rejects signs and falls back to local.
131+
String port = hostPort.substring(hostPort.lastIndexOf(':') + 1);
132+
for (int i = 0; i < port.length(); i++) {
133+
if (port.charAt(i) < '0' || port.charAt(i) > '9') {
134+
return false;
135+
}
136+
}
137+
}
138+
String host = parsed.getHost();
139+
// Rust rejects zones and IPv4 tails with leading zeros, such as ::ffff:127.0.0.01.
140+
return !host.isEmpty() && (!host.contains(":")
141+
|| (!host.contains("%") && InetAddresses.isInetAddress(host)));
130142
} catch (URISyntaxException | IllegalArgumentException e) {
131143
return false;
132144
}

‎fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,25 +17,83 @@
1717

1818
package org.apache.doris.nereids.rules.analysis;
1919

20+
import org.apache.doris.common.Config;
21+
import org.apache.doris.nereids.exceptions.AnalysisException;
2022
import org.apache.doris.nereids.parser.NereidsParser;
2123
import org.apache.doris.nereids.trees.expressions.NamedExpression;
2224
import org.apache.doris.nereids.trees.plans.Plan;
25+
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
2326
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTVFCommand;
27+
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
2428
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
2529
import org.apache.doris.nereids.trees.plans.logical.LogicalTVFTableSink;
2630
import org.apache.doris.nereids.util.MemoTestUtils;
2731
import org.apache.doris.nereids.util.PlanChecker;
2832
import org.apache.doris.qe.ConnectContext;
2933

34+
import org.junit.jupiter.api.AfterEach;
3035
import org.junit.jupiter.api.Assertions;
36+
import org.junit.jupiter.api.BeforeEach;
3137
import org.junit.jupiter.api.Test;
3238

39+
import java.util.Arrays;
3340
import java.util.List;
3441

3542
class BindTVFTableSinkTest {
43+
private boolean originalEnableLocalTvf;
44+
45+
@BeforeEach
46+
void setUp() {
47+
originalEnableLocalTvf = Config.enable_local_tvf;
48+
Config.enable_local_tvf = false;
49+
}
50+
51+
@AfterEach
52+
void tearDown() {
53+
Config.enable_local_tvf = originalEnableLocalTvf;
54+
}
55+
56+
@Test
57+
void disabledLocalSinkRejectsInsertAndExplainBeforeProperties() {
58+
for (String prefix : Arrays.asList("", "EXPLAIN ")) {
59+
for (String properties : Arrays.asList("'format'='csv'",
60+
"'file_path'='/tmp/local_gate_', 'backend_id'='1', 'format'='csv'")) {
61+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
62+
() -> analyzeInsert(prefix + "INSERT INTO local(" + properties + ") SELECT 1"));
63+
Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled"));
64+
}
65+
}
66+
}
67+
68+
@Test
69+
void enabledLocalSinkKeepsPropertyValidation() {
70+
Config.enable_local_tvf = true;
71+
AnalysisException exception = Assertions.assertThrows(AnalysisException.class,
72+
() -> analyzeInsert("INSERT INTO local('format'='csv') SELECT 1"));
73+
Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'"));
74+
}
75+
76+
@Test
77+
void disabledLocalDoesNotBlockRemoteSinks() {
78+
for (String name : Arrays.asList("s3", "hdfs")) {
79+
Assertions.assertInstanceOf(LogicalTVFTableSink.class,
80+
analyzeInsert("INSERT INTO " + name + "('file_path'='remote/data', 'format'='csv') SELECT 1"));
81+
}
82+
}
83+
84+
private Plan analyzeInsert(String sql) {
85+
ConnectContext context = MemoTestUtils.createConnectContext();
86+
LogicalPlan parsed = new NereidsParser().parseSingle(sql);
87+
if (parsed instanceof ExplainCommand) {
88+
parsed = ((ExplainCommand) parsed).getLogicalPlan();
89+
}
90+
InsertIntoTVFCommand command = (InsertIntoTVFCommand) parsed;
91+
return PlanChecker.from(context).analyze(command.getExplainPlan(context)).getPlan();
92+
}
3693

3794
@Test
3895
void duplicateOutputNamesKeepDistinctPositions() {
96+
Config.enable_local_tvf = true;
3997
ConnectContext context = MemoTestUtils.createConnectContext();
4098
InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle(
4199
"INSERT INTO local("

‎fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,8 @@ public void testLanceRejectsBrokerLocalUrisBeforeOpeningDataset() {
9292
"file-object-store://ignored/tmp/dataset.lance", "file+uring://ignored/tmp/dataset.lance",
9393
"FILE-OBJECT-STORE://ignored/tmp/dataset.lance", "FILE+URING://ignored/tmp/dataset.lance",
9494
"c://ignored/tmp/dataset.lance", "memory://dataset.lance", "shared-memory://dataset.lance",
95+
"s3://bucket:-0/../../dataset.lance", "s3://bucket:-00/../../dataset.lance",
96+
"s3://bucket:+0/../../dataset.lance", "s3://[::ffff:127.0.0.01]/../../dataset.lance",
9597
"s3://bucket:bad/../../dataset.lance", "s3://bucket:65536/../../dataset.lance",
9698
"s3://:9000/dataset.lance", "s3://user@/dataset.lance",
9799
"s3://[::1%25eth0]/dataset.lance", "https://bucket:bad/../../dataset.lance",
@@ -136,6 +138,9 @@ public void testLanceRemoteUrisReachMetadataLoader() throws Exception {
136138
for (String uri : Arrays.asList("s3://bucket/dataset.lance", "s3+ddb://bucket/dataset.lance",
137139
"oss://bucket/dataset.lance", "gs://bucket/dataset.lance", "az://container/dataset.lance",
138140
"abfss://[email protected]/dataset.lance",
141+
"s3://bucket:0/dataset.lance", "s3://bucket:00080/dataset.lance",
142+
"s3://bucket:65535/dataset.lance", "s3://bucket:/dataset.lance",
143+
"s3://[::1]/dataset.lance", "s3://[::ffff:127.0.0.1]/dataset.lance",
139144
"gs://bucket_with_underscore/dataset.lance", "s3://bucket:9000/dataset.lance",
140145
"goosefs://bucket/dataset.lance", "goosefs:///dataset.lance", "goosefs:/dataset.lance",
141146
"cos://bucket/dataset.lance",

‎regression-test/suites/account_p0/test_local_tvf_enable.groovy‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,17 @@ suite("test_local_tvf_enable", "p0") {
3030
}
3131
}
3232
}
33+
if (!enabled) {
34+
for (def prefix : ["", "EXPLAIN "]) {
35+
test {
36+
sql """${prefix}INSERT INTO LOCAL(
37+
"file_path" = "/tmp/local_enable_test_",
38+
"backend_id" = "-1",
39+
"format" = "csv"
40+
) SELECT 1"""
41+
exception "LOCAL TVF is disabled"
42+
}
43+
}
44+
}
45+
3346
}

0 commit comments

Comments
 (0)