|
21 | 21 | #include <gtest/gtest.h> |
22 | 22 |
|
23 | 23 | #include <filesystem> |
| 24 | +#include <set> |
24 | 25 |
|
25 | 26 | #include "common/config.h" |
26 | 27 | #include "storage/index/index_file_reader.h" |
@@ -534,6 +535,129 @@ class IndexBuilderTest : public ::testing::Test { |
534 | 535 | EXPECT_FALSE(index_exists); |
535 | 536 | } |
536 | 537 |
|
| 538 | + // Builds a k2 index over a rowset whose schema owns a k1 index that was never |
| 539 | + // written, and returns the index ids present in the output rowset's index |
| 540 | + // file. `leave_empty_index_file` picks between the two shapes a source |
| 541 | + // segment with no index content can have on disk: a zero-byte file (read as |
| 542 | + // INVERTED_INDEX_BYPASS) or no file at all (INVERTED_INDEX_FILE_NOT_FOUND). |
| 543 | + // |
| 544 | + // The rowset is written BEFORE the schema gains k1, so k1 has no content |
| 545 | + // anywhere and nothing can be lost by either shape. That is not an artifact |
| 546 | + // of the test: a zero-byte index file is only ever produced when no logical |
| 547 | + // index opened a directory, and every non-VARIANT index opens its directory |
| 548 | + // eagerly in IndexColumnWriter::create(). A zero-byte file therefore cannot |
| 549 | + // be a former home of some other index's data. |
| 550 | + std::set<int64_t> build_k2_index_over_unwritten_k1(int64_t tablet_id, int64_t rowset_id, |
| 551 | + bool leave_empty_index_file) { |
| 552 | + auto tablet_path = _absolute_dir + "/" + std::to_string(tablet_id); |
| 553 | + _tablet->_tablet_path = tablet_path; |
| 554 | + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tablet_path).ok()); |
| 555 | + EXPECT_TRUE(io::global_local_filesystem()->create_directory(tablet_path).ok()); |
| 556 | + |
| 557 | + RowsetSharedPtr rowset; |
| 558 | + RowsetWriterContext writer_context; |
| 559 | + writer_context.rowset_id.init(rowset_id); |
| 560 | + writer_context.tablet_id = rowset_id; |
| 561 | + writer_context.tablet_schema_hash = 567997577; |
| 562 | + writer_context.partition_id = 10; |
| 563 | + writer_context.rowset_type = BETA_ROWSET; |
| 564 | + writer_context.tablet_path = _absolute_dir + "/" + std::to_string(rowset_id); |
| 565 | + writer_context.rowset_state = VISIBLE; |
| 566 | + writer_context.tablet_schema = _tablet_schema; |
| 567 | + writer_context.version.first = 10; |
| 568 | + writer_context.version.second = 10; |
| 569 | + EXPECT_TRUE( |
| 570 | + io::global_local_filesystem()->create_directory(writer_context.tablet_path).ok()); |
| 571 | + |
| 572 | + auto res = RowsetFactory::create_rowset_writer(*_engine_ref, writer_context, false); |
| 573 | + EXPECT_TRUE(res.has_value()) << res.error(); |
| 574 | + auto rowset_writer = std::move(res).value(); |
| 575 | + { |
| 576 | + Block block = _tablet_schema->create_storage_block(); |
| 577 | + auto columns = std::move(block).mutate_columns(); |
| 578 | + for (int i = 0; i < 1000; ++i) { |
| 579 | + int32_t k1 = i * 10; |
| 580 | + columns[0]->insert_data((const char*)&k1, sizeof(k1)); |
| 581 | + int32_t k2 = i % 100; |
| 582 | + columns[1]->insert_data((const char*)&k2, sizeof(k2)); |
| 583 | + } |
| 584 | + block.set_columns(std::move(columns)); |
| 585 | + EXPECT_TRUE(rowset_writer->add_block(&block).ok()); |
| 586 | + EXPECT_TRUE(rowset_writer->flush().ok()); |
| 587 | + EXPECT_TRUE(rowset_writer->build(rowset).ok()); |
| 588 | + EXPECT_TRUE(_tablet->add_rowset(rowset).ok()); |
| 589 | + } |
| 590 | + |
| 591 | + // The schema gains k1 only now, so the rowset owns an index whose content |
| 592 | + // was never written -- the same state an all-NULL VARIANT column leaves. |
| 593 | + TabletIndex k1_index; |
| 594 | + k1_index._index_id = 1; |
| 595 | + k1_index._index_name = "k1_index"; |
| 596 | + k1_index._index_type = IndexType::INVERTED; |
| 597 | + k1_index._col_unique_ids.push_back(1); |
| 598 | + _tablet_schema->append_index(std::move(k1_index)); |
| 599 | + |
| 600 | + auto segment_path = rowset->segment_path(0); |
| 601 | + EXPECT_TRUE(segment_path.has_value()) << segment_path.error(); |
| 602 | + const std::string source_prefix { |
| 603 | + segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix( |
| 604 | + segment_path.value())}; |
| 605 | + const auto source_index_path = |
| 606 | + segment_v2::InvertedIndexDescriptor::get_index_file_path_v2(source_prefix); |
| 607 | + bool exists = true; |
| 608 | + EXPECT_TRUE(io::global_local_filesystem()->exists(source_index_path, &exists).ok()); |
| 609 | + EXPECT_FALSE(exists) << "an index-less rowset must not have written an index file"; |
| 610 | + if (leave_empty_index_file) { |
| 611 | + io::FileWriterPtr empty; |
| 612 | + EXPECT_TRUE(io::global_local_filesystem()->create_file(source_index_path, &empty).ok()); |
| 613 | + EXPECT_TRUE(empty->close().ok()); |
| 614 | + } |
| 615 | + { |
| 616 | + auto reader = std::make_unique<segment_v2::IndexFileReader>( |
| 617 | + io::global_local_filesystem(), source_prefix, InvertedIndexStorageFormatPB::V2); |
| 618 | + auto st = reader->init(); |
| 619 | + EXPECT_TRUE(leave_empty_index_file ? st.is<ErrorCode::INVERTED_INDEX_BYPASS>() |
| 620 | + : st.is<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>()) |
| 621 | + << st; |
| 622 | + } |
| 623 | + |
| 624 | + TOlapTableIndex k2_index; |
| 625 | + k2_index.index_id = 2; |
| 626 | + k2_index.columns.emplace_back("k2"); |
| 627 | + k2_index.index_name = "k2_index"; |
| 628 | + k2_index.index_type = TIndexType::INVERTED; |
| 629 | + _alter_indexes.clear(); |
| 630 | + _alter_indexes.push_back(k2_index); |
| 631 | + |
| 632 | + IndexBuilder builder(ExecEnv::GetInstance()->storage_engine().to_local(), _tablet, _columns, |
| 633 | + _alter_indexes, false); |
| 634 | + EXPECT_TRUE(builder.init().ok()); |
| 635 | + auto status = builder.do_build_inverted_index(); |
| 636 | + EXPECT_TRUE(status.ok()) << status.to_string(); |
| 637 | + |
| 638 | + std::set<int64_t> output_index_ids; |
| 639 | + EXPECT_EQ(builder._output_rowsets.size(), 1); |
| 640 | + if (builder._output_rowsets.empty()) { |
| 641 | + return output_index_ids; |
| 642 | + } |
| 643 | + auto output_segment_path = builder._output_rowsets[0]->segment_path(0); |
| 644 | + EXPECT_TRUE(output_segment_path.has_value()) << output_segment_path.error(); |
| 645 | + auto reader = std::make_unique<segment_v2::IndexFileReader>( |
| 646 | + io::global_local_filesystem(), |
| 647 | + std::string {segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix( |
| 648 | + output_segment_path.value())}, |
| 649 | + InvertedIndexStorageFormatPB::V2); |
| 650 | + EXPECT_TRUE(reader->init().ok()); |
| 651 | + auto dirs = reader->get_all_directories(); |
| 652 | + EXPECT_TRUE(dirs.has_value()); |
| 653 | + if (dirs.has_value()) { |
| 654 | + for (const auto& [key, _] : dirs.value()) { |
| 655 | + output_index_ids.insert(key.first); |
| 656 | + } |
| 657 | + } |
| 658 | + return output_index_ids; |
| 659 | + } |
| 660 | + |
537 | 661 | StorageEngine* _engine_ref = nullptr; |
538 | 662 | TabletSharedPtr _tablet; |
539 | 663 | TabletMetaSharedPtr _tablet_meta; |
@@ -1027,119 +1151,24 @@ TEST_F(IndexBuilderTest, BuildInvertedIndexAfterWritingDataTest) { |
1027 | 1151 | //EXPECT_TRUE(tablet_schema->has_inverted_index_with_index_id(2)); |
1028 | 1152 | } |
1029 | 1153 |
|
1030 | | -// A schema may legitimately own an inverted index whose container holds nothing. |
1031 | | -// An all-NULL VARIANT column extracts no subcolumn, so no logical index directory |
1032 | | -// is ever opened and the closed container is zero bytes. ALTER on such a rowset |
1033 | | -// must read that exactly like a rowset written before any index existed: there is |
| 1154 | +// A schema can own an inverted index whose index file holds nothing: an all-NULL |
| 1155 | +// VARIANT column extracts no subcolumn, so no logical index directory is ever |
| 1156 | +// opened and the file is closed with nothing in it. ALTER on such a rowset must |
| 1157 | +// read that exactly like a rowset written before any index existed -- there is |
1034 | 1158 | // nothing to carry over, and every requested index is built from the raw columns. |
1035 | 1159 | TEST_F(IndexBuilderTest, BuildIndexOverEmptyIndexFileTest) { |
1036 | | - // 0. prepare tablet path |
1037 | | - auto tablet_path = _absolute_dir + "/" + std::to_string(14695); |
1038 | | - _tablet->_tablet_path = tablet_path; |
1039 | | - ASSERT_TRUE(io::global_local_filesystem()->delete_directory(tablet_path).ok()); |
1040 | | - ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet_path).ok()); |
1041 | | - |
1042 | | - // 1. The schema already owns an index, so the rowset owns a container file |
1043 | | - TabletIndex initial_index; |
1044 | | - initial_index._index_id = 1; |
1045 | | - initial_index._index_name = "k1_index"; |
1046 | | - initial_index._index_type = IndexType::INVERTED; |
1047 | | - initial_index._col_unique_ids.push_back(1); // unique_id for k1 |
1048 | | - _tablet_schema->append_index(std::move(initial_index)); |
1049 | | - |
1050 | | - // 2. Create a rowset writer context |
1051 | | - RowsetSharedPtr rowset; |
1052 | | - const int num_rows = 1000; |
1053 | | - RowsetWriterContext writer_context; |
1054 | | - writer_context.rowset_id.init(15695); |
1055 | | - writer_context.tablet_id = 15695; |
1056 | | - writer_context.tablet_schema_hash = 567997577; |
1057 | | - writer_context.partition_id = 10; |
1058 | | - writer_context.rowset_type = BETA_ROWSET; |
1059 | | - writer_context.tablet_path = _absolute_dir + "/" + std::to_string(15695); |
1060 | | - writer_context.rowset_state = VISIBLE; |
1061 | | - writer_context.tablet_schema = _tablet_schema; |
1062 | | - writer_context.version.first = 10; |
1063 | | - writer_context.version.second = 10; |
1064 | | - |
1065 | | - ASSERT_TRUE(io::global_local_filesystem()->create_directory(writer_context.tablet_path).ok()); |
1066 | | - |
1067 | | - auto res = RowsetFactory::create_rowset_writer(*_engine_ref, writer_context, false); |
1068 | | - ASSERT_TRUE(res.has_value()) << res.error(); |
1069 | | - auto rowset_writer = std::move(res).value(); |
1070 | | - |
1071 | | - // 3. Write data to the rowset |
1072 | | - { |
1073 | | - Block block = _tablet_schema->create_storage_block(); |
1074 | | - auto columns = std::move(block).mutate_columns(); |
1075 | | - for (int i = 0; i < num_rows; ++i) { |
1076 | | - int32_t k1 = i * 10; |
1077 | | - columns[0]->insert_data((const char*)&k1, sizeof(k1)); |
1078 | | - int32_t k2 = i % 100; |
1079 | | - columns[1]->insert_data((const char*)&k2, sizeof(k2)); |
1080 | | - } |
1081 | | - block.set_columns(std::move(columns)); |
1082 | | - ASSERT_TRUE(rowset_writer->add_block(&block).ok()); |
1083 | | - ASSERT_TRUE(rowset_writer->flush().ok()); |
1084 | | - ASSERT_TRUE(rowset_writer->build(rowset).ok()); |
1085 | | - ASSERT_TRUE(_tablet->add_rowset(rowset).ok()); |
1086 | | - } |
1087 | | - |
1088 | | - // 4. Empty the container in place. This is the on-disk shape a segment whose |
1089 | | - // logical indexes wrote nothing leaves behind once the empty file is |
1090 | | - // closed rather than deleted by the local writer's destructor. |
1091 | | - const auto segment_path = rowset->segment_path(0); |
1092 | | - ASSERT_TRUE(segment_path.has_value()) << segment_path.error(); |
1093 | | - const std::string index_path_prefix { |
1094 | | - segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path.value())}; |
1095 | | - const auto index_path = |
1096 | | - segment_v2::InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix); |
1097 | | - std::error_code ec; |
1098 | | - std::filesystem::resize_file(index_path, 0, ec); |
1099 | | - ASSERT_FALSE(ec) << ec.message(); |
1100 | | - { |
1101 | | - auto reader = std::make_unique<segment_v2::IndexFileReader>( |
1102 | | - io::global_local_filesystem(), index_path_prefix, InvertedIndexStorageFormatPB::V2); |
1103 | | - auto st = reader->init(); |
1104 | | - ASSERT_TRUE(st.is<ErrorCode::INVERTED_INDEX_BYPASS>()) << st; |
1105 | | - } |
1106 | | - |
1107 | | - // 5. ADD INDEX over that rowset |
1108 | | - TOlapTableIndex index2; |
1109 | | - index2.index_id = 2; |
1110 | | - index2.columns.emplace_back("k2"); |
1111 | | - index2.index_name = "k2_index"; |
1112 | | - index2.index_type = TIndexType::INVERTED; |
1113 | | - _alter_indexes.push_back(index2); |
1114 | | - |
1115 | | - IndexBuilder builder(ExecEnv::GetInstance()->storage_engine().to_local(), _tablet, _columns, |
1116 | | - _alter_indexes, false); |
1117 | | - ASSERT_TRUE(builder.init().ok()); |
1118 | | - auto status = builder.do_build_inverted_index(); |
1119 | | - // Without tolerating the empty container this fails with |
| 1160 | + // Without tolerating the empty index file this fails the whole ALTER with |
1120 | 1161 | // [E-6004]inverted index file ... is empty. |
1121 | | - EXPECT_TRUE(status.ok()) << status.to_string(); |
| 1162 | + EXPECT_EQ(build_k2_index_over_unwritten_k1(14695, 15695, true), (std::set<int64_t> {2})); |
| 1163 | +} |
1122 | 1164 |
|
1123 | | - // 6. The output rowset carries a real container holding the requested index |
1124 | | - std::vector<io::FileInfo> new_files; |
1125 | | - bool new_dir_exists = false; |
1126 | | - ASSERT_TRUE(io::global_local_filesystem() |
1127 | | - ->list(tablet_path, true, &new_files, &new_dir_exists) |
1128 | | - .ok()); |
1129 | | - ASSERT_TRUE(new_dir_exists); |
1130 | | - int new_idx_file_count = 0; |
1131 | | - for (const auto& file : new_files) { |
1132 | | - if (file.file_name.find(".idx") == std::string::npos) { |
1133 | | - continue; |
1134 | | - } |
1135 | | - new_idx_file_count++; |
1136 | | - int64_t file_size = 0; |
1137 | | - ASSERT_TRUE(io::global_local_filesystem() |
1138 | | - ->file_size(tablet_path + "/" + file.file_name, &file_size) |
1139 | | - .ok()); |
1140 | | - EXPECT_GT(file_size, 0) << file.file_name << " should hold the newly built index"; |
1141 | | - } |
1142 | | - EXPECT_EQ(new_idx_file_count, 1) << "New directory should contain exactly 1 .idx file"; |
| 1165 | +// The same rowset with NO index file at all. This path has tolerated |
| 1166 | +// INVERTED_INDEX_FILE_NOT_FOUND for years, and it produces exactly the same |
| 1167 | +// output: the requested index is built, and an index the source never held is |
| 1168 | +// not invented. Pinning both together is the point -- an empty index file is |
| 1169 | +// being read the way a missing one already was, not given new semantics. |
| 1170 | +TEST_F(IndexBuilderTest, BuildIndexOverMissingIndexFileTest) { |
| 1171 | + EXPECT_EQ(build_k2_index_over_unwritten_k1(14696, 15696, false), (std::set<int64_t> {2})); |
1143 | 1172 | } |
1144 | 1173 |
|
1145 | 1174 | TEST_F(IndexBuilderTest, BuildAnnIndexAfterWritingDataTest) { |
|
0 commit comments