Skip to content

Commit 33cbacc

Browse files
committed
[fix](tvf) Preserve LOCAL file URI dataset identity
### What problem does this PR solve? Issue Number: N/A Related PR: #69527 Problem Summary: LOCAL Lance canonicalization followed symlinks before URL dot-segment normalization, so a valid file URI could open a different in-root dataset. Uppercase file schemes were rejected. Normalize the actual file URI scheme, separators and raw dot segments before one-pass decoding and filesystem containment; preserve bare path behavior. ### Release note LOCAL Lance preserves file URI dataset identity for dot segments, case-insensitive schemes and raw URI separators while retaining secure-root containment. ### Check List (For Author) - Test: 13 selected BE filesystem/Lance tests passed with fresh XML, zero failures/skips and unchanged sources. New regressions fail before the fix, including actual schema and row-value mismatch. Changed-line clang-format16 and git diff --check passed. FE source is unchanged. - Behavior changed: Yes, restore valid file URI identity without changing bare paths, remote TVFs or fixed-version catalogs. - Does this need documentation: The original PR documents the LOCAL startup opt-in; no additional public configuration or syntax is introduced.
1 parent 90a3a93 commit 33cbacc

3 files changed

Lines changed: 182 additions & 2 deletions

File tree

‎be/src/io/fs/local_file_system.cpp‎

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -463,15 +463,68 @@ Status LocalFileSystem::canonicalize_local_file(const std::string& dir,
463463
if (file_path.empty() || file_path.find('\0') != std::string::npos) {
464464
return Status::InvalidArgument("local file path must not be empty or contain NUL");
465465
}
466+
// URI schemes are ASCII case-insensitive; do not alter spelling in bare paths.
467+
const bool is_file_uri = file_path.size() >= 5 &&
468+
(file_path[0] == 'f' || file_path[0] == 'F') &&
469+
(file_path[1] == 'i' || file_path[1] == 'I') &&
470+
(file_path[2] == 'l' || file_path[2] == 'L') &&
471+
(file_path[3] == 'e' || file_path[3] == 'E') && file_path[4] == ':';
472+
std::string local_input = file_path;
473+
if (is_file_uri) {
474+
local_input.replace(0, 4, "file");
475+
// File URLs treat raw backslashes as separators, including before the authority.
476+
// Do this before conversion; percent-encoded backslashes must remain literal.
477+
for (size_t i = 5; i < local_input.size(); ++i) {
478+
if (local_input[i] == '\\') {
479+
local_input[i] = '/';
480+
}
481+
}
482+
}
466483
Path local_path;
467484
// Convert file: URIs to filesystem paths and reject remote schemes before any reader opens them.
468-
RETURN_IF_ERROR(convert_to_abs_path(file_path, local_path));
469-
if (file_path.rfind("file:", 0) == 0) {
485+
RETURN_IF_ERROR(convert_to_abs_path(local_input, local_path));
486+
if (is_file_uri) {
470487
// file:a%20b names "a b", but a bare path containing "%20" must stay literal.
471488
// Decode once, preserve '+' and invalid escapes, matching Lance's file provider.
472489
auto encoded = local_path.string();
473490
// Query and fragment are URI components, not part of the filesystem path.
474491
encoded = encoded.substr(0, encoded.find_first_of("?#"));
492+
// URL dot segments are removed before symlinks are followed and before %2F
493+
// becomes a path separator. Keep empty segments: "a//../b" denotes "a/b".
494+
const bool absolute = !encoded.empty() && encoded[0] == '/';
495+
std::vector<std::string> segments;
496+
for (size_t start = absolute ? 1 : 0;;) {
497+
const size_t end = encoded.find('/', start);
498+
const auto segment =
499+
encoded.substr(start, end == std::string::npos ? end : end - start);
500+
const bool dot = segment == "." || segment == "%2e" || segment == "%2E";
501+
const bool parent = segment == ".." || segment == ".%2e" || segment == ".%2E" ||
502+
segment == "%2e." || segment == "%2E." || segment == "%2e%2e" ||
503+
segment == "%2e%2E" || segment == "%2E%2e" || segment == "%2E%2E";
504+
if (parent) {
505+
if (!segments.empty() && segments.back() != "..") {
506+
segments.pop_back();
507+
} else if (!absolute) {
508+
segments.emplace_back("..");
509+
}
510+
} else if (!dot) {
511+
segments.push_back(segment);
512+
}
513+
if (end == std::string::npos) {
514+
if (dot || parent) {
515+
segments.emplace_back("");
516+
}
517+
break;
518+
}
519+
start = end + 1;
520+
}
521+
encoded = absolute ? "/" : "";
522+
for (size_t i = 0; i < segments.size(); ++i) {
523+
if (i > 0) {
524+
encoded += '/';
525+
}
526+
encoded += segments[i];
527+
}
475528
const auto hex_digit = [](char c) -> int {
476529
if (c >= '0' && c <= '9') return c - '0';
477530
if (c >= 'a' && c <= 'f') return c - 'a' + 10;

‎be/test/format_v2/table/lance_reader_test.cpp‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4450,6 +4450,78 @@ TEST(LanceTableReaderLocalPathTest, FileUrisPreserveEscapedDatasetIdentity) {
44504450
}
44514451
}
44524452

4453+
TEST(LanceTableReaderLocalPathTest, FileUrisNormalizeBeforeFilesystemResolution) {
4454+
const auto original_secure_path = config::user_files_secure_path;
4455+
DEFER(config::user_files_secure_path = original_secure_path);
4456+
const auto root = std::filesystem::temp_directory_path() /
4457+
("lance_uri_normalization_" +
4458+
std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()));
4459+
DEFER({
4460+
std::error_code error;
4461+
std::filesystem::remove_all(root, error);
4462+
});
4463+
std::filesystem::create_directories(root / "nested/child");
4464+
std::filesystem::create_directory_symlink(root / "nested/child", root / "jump");
4465+
const auto backslash_uri = std::string(R"(fIlE:\\localhost)") + root.string() + "/data.lance";
4466+
for (const bool nested : {false, true}) {
4467+
arrow::Int64Builder values_builder;
4468+
ASSERT_TRUE(values_builder.Append(nested ? 42 : 7).ok());
4469+
std::shared_ptr<arrow::Array> values;
4470+
ASSERT_TRUE(values_builder.Finish(&values).ok());
4471+
const auto schema = arrow::schema(
4472+
{arrow::field("row_id", arrow::int64(), false),
4473+
arrow::field(nested ? "nested_only" : "root_only", arrow::int64(), false)});
4474+
ASSERT_TRUE(write_lance_record_batch(root / (nested ? "nested/data.lance" : "data.lance"),
4475+
arrow::RecordBatch::Make(schema, 1, {values, values}))
4476+
.ok());
4477+
}
4478+
config::user_files_secure_path = root.string();
4479+
for (const auto& input :
4480+
{std::make_pair("file://" + root.string() + "/jump/../data.lance", false),
4481+
std::make_pair("file://" + root.string() + "/missing/../data.lance", false),
4482+
std::make_pair("file://" + root.string() + "/jump/%2e%2E/data.lance", false),
4483+
std::make_pair("FILE://" + root.string() + "/data.lance", false),
4484+
std::make_pair(backslash_uri, false),
4485+
std::make_pair("fIlE://" + root.string() + "/jump/.%2e/data.lance", false),
4486+
std::make_pair("file://" + root.string() + "/jump%2f../data.lance", true),
4487+
std::make_pair(std::string("jump/../data.lance"), true)}) {
4488+
SCOPED_TRACE(input.first);
4489+
TQueryGlobals globals;
4490+
RuntimeState state(globals);
4491+
RuntimeProfile profile("lance_uri_normalization");
4492+
TFileScanRangeParams scan_params;
4493+
LanceTableReader reader;
4494+
const Columns columns {projected_column("row_id", TYPE_BIGINT, false)};
4495+
ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok());
4496+
auto range = make_latest_lance_range(input.first);
4497+
range.__set_file_type(TFileType::FILE_LOCAL);
4498+
std::vector<std::string> names;
4499+
std::vector<DataTypePtr> types;
4500+
ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok());
4501+
EXPECT_EQ((std::vector<std::string> {"row_id", input.second ? "nested_only" : "root_only"}),
4502+
names);
4503+
LanceTableReader::DatasetKey key;
4504+
ASSERT_TRUE(reader._dataset_key(range, &key).ok());
4505+
EXPECT_EQ((root / (input.second ? "nested/data.lance" : "data.lance")).string(), key.uri);
4506+
ASSERT_TRUE(prepare_range(&reader, range).ok());
4507+
Block block;
4508+
add_output_columns(&block, columns);
4509+
std::vector<int64_t> row_ids;
4510+
bool eos = false;
4511+
while (!eos) {
4512+
ASSERT_TRUE(reader.get_block(&block, &eos).ok());
4513+
if (!eos) {
4514+
const auto& values =
4515+
assert_cast<const ColumnInt64&>(*block.get_by_position(0).column)
4516+
.get_data();
4517+
row_ids.insert(row_ids.end(), values.begin(), values.end());
4518+
}
4519+
}
4520+
EXPECT_EQ((std::vector<int64_t> {input.second ? 42 : 7}), row_ids);
4521+
ASSERT_TRUE(reader.close().ok());
4522+
}
4523+
}
4524+
44534525
TEST(LanceTableReaderLocalPathTest, FixedVersionLocalCatalogKeepsItsUri) {
44544526
TFileScanRangeParams scan_params;
44554527
LanceTableReader reader;

‎be/test/io/fs/local_file_system_test.cpp‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -718,6 +718,61 @@ TEST_F(LocalFileSystemTest, TestCanonicalizeSecureLocalPath) {
718718
EXPECT_FALSE(fs->canonicalize_local_file("", secure.string(), &resolved).ok());
719719
}
720720

721+
TEST_F(LocalFileSystemTest, TestCanonicalizeFileUriIdentity) {
722+
auto fs = io::global_local_filesystem();
723+
const auto root = std::filesystem::canonical(test_dir);
724+
const auto secure = root / "secure";
725+
std::filesystem::create_directories(secure / "nested/child/data.lance");
726+
std::filesystem::create_directory(secure / "nested/data.lance");
727+
std::filesystem::create_directory(secure / "data.lance");
728+
std::filesystem::create_directory(secure / "a b.lance");
729+
std::filesystem::create_directory(root / "outside");
730+
std::filesystem::create_directory_symlink(secure / "nested/child", secure / "jump");
731+
std::filesystem::create_directory(secure / "literal\\name.lance");
732+
const auto backslash_uri = std::string(R"(fIlE:\\localhost)") + secure.string() + "/data.lance";
733+
for (const auto& input :
734+
{std::make_pair("file://" + secure.string() + "/jump/../data.lance",
735+
secure / "data.lance"),
736+
std::make_pair("file://" + secure.string() + "/missing/../data.lance",
737+
secure / "data.lance"),
738+
std::make_pair("file://" + secure.string() + "/jump/%2e%2E/data.lance",
739+
secure / "data.lance"),
740+
std::make_pair("file://" + secure.string() + "/jump/.%2e/data.lance",
741+
secure / "data.lance"),
742+
std::make_pair("file://" + secure.string() + "/jump/%2E./data.lance",
743+
secure / "data.lance"),
744+
std::make_pair("file://" + secure.string() + "/missing/.././data.lance",
745+
secure / "data.lance"),
746+
std::make_pair("file://" + secure.string() + "/jump//../data.lance",
747+
secure / "nested/child/data.lance"),
748+
std::make_pair("file://" + secure.string() + "/jump%2f../data.lance",
749+
secure / "nested/data.lance"),
750+
std::make_pair("file://" + secure.string() + "/jump\\..\\data.lance",
751+
secure / "data.lance"),
752+
std::make_pair("FILE://" + secure.string() + "/data.lance", secure / "data.lance"),
753+
std::make_pair(backslash_uri, secure / "data.lance"),
754+
std::make_pair("file://" + secure.string() + "/literal%5cname.lance",
755+
secure / "literal\\name.lance"),
756+
std::make_pair(std::string("literal\\name.lance"), secure / "literal\\name.lance"),
757+
std::make_pair("fIlE://" + secure.string() + "/a%20b.lance?q=%00#tag",
758+
secure / "a b.lance"),
759+
std::make_pair(std::string("jump/../data.lance"), secure / "nested/data.lance")}) {
760+
std::string actual;
761+
const auto status = fs->canonicalize_local_file(secure.string(), input.first, &actual);
762+
ASSERT_TRUE(status.ok()) << input.first << " " << status;
763+
EXPECT_EQ(input.second.string(), actual) << input.first;
764+
}
765+
for (const auto& input :
766+
{"file://" + secure.string() + "/jump/../../outside",
767+
"FILE://" + secure.string() + "/%2e%2e/outside",
768+
"fIlE://" + secure.string() + "/a%00b.lance", std::string("HTTP://invalid/dataset")}) {
769+
std::string actual;
770+
const auto status = fs->canonicalize_local_file(secure.string(), input, &actual);
771+
EXPECT_FALSE(status.ok()) << input;
772+
EXPECT_EQ(std::string::npos, status.to_string().find(root.string()));
773+
}
774+
}
775+
721776
TEST_F(LocalFileSystemTest, TestConvertToAbsPath) {
722777
io::Path abs_path;
723778
Status st;

0 commit comments

Comments
 (0)