Skip to content

Commit 2be3081

Browse files
airborne12claude
andcommitted
[test](inverted index) Build the empty index file the way production does
The ALTER test built its zero-byte index file by truncating a populated k1 index, which reads as though losing k1 were acceptable. It is not a shape production can produce: every non-VARIANT index opens its directory eagerly in IndexColumnWriter::create(), so a rowset whose schema owns one always has a non-empty index file. Zero bytes means no logical index ever opened, and therefore that nothing was in the file to lose. Build the rowset before the schema gains k1 instead, so k1 has no content anywhere, and assert the two on-disk shapes of a source segment with no index content side by side: a zero-byte file (INVERTED_INDEX_BYPASS) and no file at all (INVERTED_INDEX_FILE_NOT_FOUND). Both produce the same output index set, which is the claim being made -- an empty index file is read the way a missing one already was, not given new semantics. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01Tp6z4cVB4buRW8JwU1KRdM
1 parent a450e0b commit 2be3081

1 file changed

Lines changed: 138 additions & 109 deletions

File tree

‎be/test/storage/index/index_builder_test.cpp‎

Lines changed: 138 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include <gtest/gtest.h>
2222

2323
#include <filesystem>
24+
#include <set>
2425

2526
#include "common/config.h"
2627
#include "storage/index/index_file_reader.h"
@@ -534,6 +535,129 @@ class IndexBuilderTest : public ::testing::Test {
534535
EXPECT_FALSE(index_exists);
535536
}
536537

538+
// Builds a k2 index over a rowset whose schema owns a k1 index that was never
539+
// written, and returns the index ids present in the output rowset's index
540+
// file. `leave_empty_index_file` picks between the two shapes a source
541+
// segment with no index content can have on disk: a zero-byte file (read as
542+
// INVERTED_INDEX_BYPASS) or no file at all (INVERTED_INDEX_FILE_NOT_FOUND).
543+
//
544+
// The rowset is written BEFORE the schema gains k1, so k1 has no content
545+
// anywhere and nothing can be lost by either shape. That is not an artifact
546+
// of the test: a zero-byte index file is only ever produced when no logical
547+
// index opened a directory, and every non-VARIANT index opens its directory
548+
// eagerly in IndexColumnWriter::create(). A zero-byte file therefore cannot
549+
// be a former home of some other index's data.
550+
std::set<int64_t> build_k2_index_over_unwritten_k1(int64_t tablet_id, int64_t rowset_id,
551+
bool leave_empty_index_file) {
552+
auto tablet_path = _absolute_dir + "/" + std::to_string(tablet_id);
553+
_tablet->_tablet_path = tablet_path;
554+
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tablet_path).ok());
555+
EXPECT_TRUE(io::global_local_filesystem()->create_directory(tablet_path).ok());
556+
557+
RowsetSharedPtr rowset;
558+
RowsetWriterContext writer_context;
559+
writer_context.rowset_id.init(rowset_id);
560+
writer_context.tablet_id = rowset_id;
561+
writer_context.tablet_schema_hash = 567997577;
562+
writer_context.partition_id = 10;
563+
writer_context.rowset_type = BETA_ROWSET;
564+
writer_context.tablet_path = _absolute_dir + "/" + std::to_string(rowset_id);
565+
writer_context.rowset_state = VISIBLE;
566+
writer_context.tablet_schema = _tablet_schema;
567+
writer_context.version.first = 10;
568+
writer_context.version.second = 10;
569+
EXPECT_TRUE(
570+
io::global_local_filesystem()->create_directory(writer_context.tablet_path).ok());
571+
572+
auto res = RowsetFactory::create_rowset_writer(*_engine_ref, writer_context, false);
573+
EXPECT_TRUE(res.has_value()) << res.error();
574+
auto rowset_writer = std::move(res).value();
575+
{
576+
Block block = _tablet_schema->create_storage_block();
577+
auto columns = std::move(block).mutate_columns();
578+
for (int i = 0; i < 1000; ++i) {
579+
int32_t k1 = i * 10;
580+
columns[0]->insert_data((const char*)&k1, sizeof(k1));
581+
int32_t k2 = i % 100;
582+
columns[1]->insert_data((const char*)&k2, sizeof(k2));
583+
}
584+
block.set_columns(std::move(columns));
585+
EXPECT_TRUE(rowset_writer->add_block(&block).ok());
586+
EXPECT_TRUE(rowset_writer->flush().ok());
587+
EXPECT_TRUE(rowset_writer->build(rowset).ok());
588+
EXPECT_TRUE(_tablet->add_rowset(rowset).ok());
589+
}
590+
591+
// The schema gains k1 only now, so the rowset owns an index whose content
592+
// was never written -- the same state an all-NULL VARIANT column leaves.
593+
TabletIndex k1_index;
594+
k1_index._index_id = 1;
595+
k1_index._index_name = "k1_index";
596+
k1_index._index_type = IndexType::INVERTED;
597+
k1_index._col_unique_ids.push_back(1);
598+
_tablet_schema->append_index(std::move(k1_index));
599+
600+
auto segment_path = rowset->segment_path(0);
601+
EXPECT_TRUE(segment_path.has_value()) << segment_path.error();
602+
const std::string source_prefix {
603+
segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(
604+
segment_path.value())};
605+
const auto source_index_path =
606+
segment_v2::InvertedIndexDescriptor::get_index_file_path_v2(source_prefix);
607+
bool exists = true;
608+
EXPECT_TRUE(io::global_local_filesystem()->exists(source_index_path, &exists).ok());
609+
EXPECT_FALSE(exists) << "an index-less rowset must not have written an index file";
610+
if (leave_empty_index_file) {
611+
io::FileWriterPtr empty;
612+
EXPECT_TRUE(io::global_local_filesystem()->create_file(source_index_path, &empty).ok());
613+
EXPECT_TRUE(empty->close().ok());
614+
}
615+
{
616+
auto reader = std::make_unique<segment_v2::IndexFileReader>(
617+
io::global_local_filesystem(), source_prefix, InvertedIndexStorageFormatPB::V2);
618+
auto st = reader->init();
619+
EXPECT_TRUE(leave_empty_index_file ? st.is<ErrorCode::INVERTED_INDEX_BYPASS>()
620+
: st.is<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>())
621+
<< st;
622+
}
623+
624+
TOlapTableIndex k2_index;
625+
k2_index.index_id = 2;
626+
k2_index.columns.emplace_back("k2");
627+
k2_index.index_name = "k2_index";
628+
k2_index.index_type = TIndexType::INVERTED;
629+
_alter_indexes.clear();
630+
_alter_indexes.push_back(k2_index);
631+
632+
IndexBuilder builder(ExecEnv::GetInstance()->storage_engine().to_local(), _tablet, _columns,
633+
_alter_indexes, false);
634+
EXPECT_TRUE(builder.init().ok());
635+
auto status = builder.do_build_inverted_index();
636+
EXPECT_TRUE(status.ok()) << status.to_string();
637+
638+
std::set<int64_t> output_index_ids;
639+
EXPECT_EQ(builder._output_rowsets.size(), 1);
640+
if (builder._output_rowsets.empty()) {
641+
return output_index_ids;
642+
}
643+
auto output_segment_path = builder._output_rowsets[0]->segment_path(0);
644+
EXPECT_TRUE(output_segment_path.has_value()) << output_segment_path.error();
645+
auto reader = std::make_unique<segment_v2::IndexFileReader>(
646+
io::global_local_filesystem(),
647+
std::string {segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(
648+
output_segment_path.value())},
649+
InvertedIndexStorageFormatPB::V2);
650+
EXPECT_TRUE(reader->init().ok());
651+
auto dirs = reader->get_all_directories();
652+
EXPECT_TRUE(dirs.has_value());
653+
if (dirs.has_value()) {
654+
for (const auto& [key, _] : dirs.value()) {
655+
output_index_ids.insert(key.first);
656+
}
657+
}
658+
return output_index_ids;
659+
}
660+
537661
StorageEngine* _engine_ref = nullptr;
538662
TabletSharedPtr _tablet;
539663
TabletMetaSharedPtr _tablet_meta;
@@ -1027,119 +1151,24 @@ TEST_F(IndexBuilderTest, BuildInvertedIndexAfterWritingDataTest) {
10271151
//EXPECT_TRUE(tablet_schema->has_inverted_index_with_index_id(2));
10281152
}
10291153

1030-
// A schema may legitimately own an inverted index whose container holds nothing.
1031-
// An all-NULL VARIANT column extracts no subcolumn, so no logical index directory
1032-
// is ever opened and the closed container is zero bytes. ALTER on such a rowset
1033-
// must read that exactly like a rowset written before any index existed: there is
1154+
// A schema can own an inverted index whose index file holds nothing: an all-NULL
1155+
// VARIANT column extracts no subcolumn, so no logical index directory is ever
1156+
// opened and the file is closed with nothing in it. ALTER on such a rowset must
1157+
// read that exactly like a rowset written before any index existed -- there is
10341158
// nothing to carry over, and every requested index is built from the raw columns.
10351159
TEST_F(IndexBuilderTest, BuildIndexOverEmptyIndexFileTest) {
1036-
// 0. prepare tablet path
1037-
auto tablet_path = _absolute_dir + "/" + std::to_string(14695);
1038-
_tablet->_tablet_path = tablet_path;
1039-
ASSERT_TRUE(io::global_local_filesystem()->delete_directory(tablet_path).ok());
1040-
ASSERT_TRUE(io::global_local_filesystem()->create_directory(tablet_path).ok());
1041-
1042-
// 1. The schema already owns an index, so the rowset owns a container file
1043-
TabletIndex initial_index;
1044-
initial_index._index_id = 1;
1045-
initial_index._index_name = "k1_index";
1046-
initial_index._index_type = IndexType::INVERTED;
1047-
initial_index._col_unique_ids.push_back(1); // unique_id for k1
1048-
_tablet_schema->append_index(std::move(initial_index));
1049-
1050-
// 2. Create a rowset writer context
1051-
RowsetSharedPtr rowset;
1052-
const int num_rows = 1000;
1053-
RowsetWriterContext writer_context;
1054-
writer_context.rowset_id.init(15695);
1055-
writer_context.tablet_id = 15695;
1056-
writer_context.tablet_schema_hash = 567997577;
1057-
writer_context.partition_id = 10;
1058-
writer_context.rowset_type = BETA_ROWSET;
1059-
writer_context.tablet_path = _absolute_dir + "/" + std::to_string(15695);
1060-
writer_context.rowset_state = VISIBLE;
1061-
writer_context.tablet_schema = _tablet_schema;
1062-
writer_context.version.first = 10;
1063-
writer_context.version.second = 10;
1064-
1065-
ASSERT_TRUE(io::global_local_filesystem()->create_directory(writer_context.tablet_path).ok());
1066-
1067-
auto res = RowsetFactory::create_rowset_writer(*_engine_ref, writer_context, false);
1068-
ASSERT_TRUE(res.has_value()) << res.error();
1069-
auto rowset_writer = std::move(res).value();
1070-
1071-
// 3. Write data to the rowset
1072-
{
1073-
Block block = _tablet_schema->create_storage_block();
1074-
auto columns = std::move(block).mutate_columns();
1075-
for (int i = 0; i < num_rows; ++i) {
1076-
int32_t k1 = i * 10;
1077-
columns[0]->insert_data((const char*)&k1, sizeof(k1));
1078-
int32_t k2 = i % 100;
1079-
columns[1]->insert_data((const char*)&k2, sizeof(k2));
1080-
}
1081-
block.set_columns(std::move(columns));
1082-
ASSERT_TRUE(rowset_writer->add_block(&block).ok());
1083-
ASSERT_TRUE(rowset_writer->flush().ok());
1084-
ASSERT_TRUE(rowset_writer->build(rowset).ok());
1085-
ASSERT_TRUE(_tablet->add_rowset(rowset).ok());
1086-
}
1087-
1088-
// 4. Empty the container in place. This is the on-disk shape a segment whose
1089-
// logical indexes wrote nothing leaves behind once the empty file is
1090-
// closed rather than deleted by the local writer's destructor.
1091-
const auto segment_path = rowset->segment_path(0);
1092-
ASSERT_TRUE(segment_path.has_value()) << segment_path.error();
1093-
const std::string index_path_prefix {
1094-
segment_v2::InvertedIndexDescriptor::get_index_file_path_prefix(segment_path.value())};
1095-
const auto index_path =
1096-
segment_v2::InvertedIndexDescriptor::get_index_file_path_v2(index_path_prefix);
1097-
std::error_code ec;
1098-
std::filesystem::resize_file(index_path, 0, ec);
1099-
ASSERT_FALSE(ec) << ec.message();
1100-
{
1101-
auto reader = std::make_unique<segment_v2::IndexFileReader>(
1102-
io::global_local_filesystem(), index_path_prefix, InvertedIndexStorageFormatPB::V2);
1103-
auto st = reader->init();
1104-
ASSERT_TRUE(st.is<ErrorCode::INVERTED_INDEX_BYPASS>()) << st;
1105-
}
1106-
1107-
// 5. ADD INDEX over that rowset
1108-
TOlapTableIndex index2;
1109-
index2.index_id = 2;
1110-
index2.columns.emplace_back("k2");
1111-
index2.index_name = "k2_index";
1112-
index2.index_type = TIndexType::INVERTED;
1113-
_alter_indexes.push_back(index2);
1114-
1115-
IndexBuilder builder(ExecEnv::GetInstance()->storage_engine().to_local(), _tablet, _columns,
1116-
_alter_indexes, false);
1117-
ASSERT_TRUE(builder.init().ok());
1118-
auto status = builder.do_build_inverted_index();
1119-
// Without tolerating the empty container this fails with
1160+
// Without tolerating the empty index file this fails the whole ALTER with
11201161
// [E-6004]inverted index file ... is empty.
1121-
EXPECT_TRUE(status.ok()) << status.to_string();
1162+
EXPECT_EQ(build_k2_index_over_unwritten_k1(14695, 15695, true), (std::set<int64_t> {2}));
1163+
}
11221164

1123-
// 6. The output rowset carries a real container holding the requested index
1124-
std::vector<io::FileInfo> new_files;
1125-
bool new_dir_exists = false;
1126-
ASSERT_TRUE(io::global_local_filesystem()
1127-
->list(tablet_path, true, &new_files, &new_dir_exists)
1128-
.ok());
1129-
ASSERT_TRUE(new_dir_exists);
1130-
int new_idx_file_count = 0;
1131-
for (const auto& file : new_files) {
1132-
if (file.file_name.find(".idx") == std::string::npos) {
1133-
continue;
1134-
}
1135-
new_idx_file_count++;
1136-
int64_t file_size = 0;
1137-
ASSERT_TRUE(io::global_local_filesystem()
1138-
->file_size(tablet_path + "/" + file.file_name, &file_size)
1139-
.ok());
1140-
EXPECT_GT(file_size, 0) << file.file_name << " should hold the newly built index";
1141-
}
1142-
EXPECT_EQ(new_idx_file_count, 1) << "New directory should contain exactly 1 .idx file";
1165+
// The same rowset with NO index file at all. This path has tolerated
1166+
// INVERTED_INDEX_FILE_NOT_FOUND for years, and it produces exactly the same
1167+
// output: the requested index is built, and an index the source never held is
1168+
// not invented. Pinning both together is the point -- an empty index file is
1169+
// being read the way a missing one already was, not given new semantics.
1170+
TEST_F(IndexBuilderTest, BuildIndexOverMissingIndexFileTest) {
1171+
EXPECT_EQ(build_k2_index_over_unwritten_k1(14696, 15696, false), (std::set<int64_t> {2}));
11431172
}
11441173

11451174
TEST_F(IndexBuilderTest, BuildAnnIndexAfterWritingDataTest) {

0 commit comments

Comments
 (0)