Skip to content

Commit a303308

Browse files
authored
[fix](cloud) preserve rowset inverted index format in recycler (apache#66793)
Related PR: apache#66475 Problem Summary: This PR separately submits the Meta Service changes from apache#66475. When the shared schema format differs from the physical format used by an existing rowset, recycler and checker must use the rowset format to select the correct inverted-index files. The change persists the physical inverted-index format in metadata and makes recycler/checker prefer the rowset value, with the existing schema and schema-KV fallback retained for old metadata. ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [x] Unit Test - `./run-cloud-ut.sh --run --filter=recycler_test:RecyclerTest.delete_tmp_rowset_data_with_rowset_idx_v3` - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [x] Yes. Recycler and checker select inverted-index files from the persisted rowset format. - Does this need documentation? - [x] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 -->
1 parent cddb396 commit a303308

5 files changed

Lines changed: 120 additions & 9 deletions

File tree

‎be/src/cloud/pb_convert.cpp‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, const RowsetMetaPB& in)
102102
if (in.has___split_schema()) {
103103
out->mutable___split_schema()->CopyFrom(in.__split_schema());
104104
}
105+
if (in.has_inverted_index_storage_format()) {
106+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
107+
}
105108
if (in.has_visible_ts_ms()) {
106109
out->set_visible_ts_ms(in.visible_ts_ms());
107110
}
@@ -198,6 +201,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) {
198201
if (in.has___split_schema()) {
199202
out->mutable___split_schema()->Swap(in.mutable___split_schema());
200203
}
204+
if (in.has_inverted_index_storage_format()) {
205+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
206+
}
201207
if (in.has_visible_ts_ms()) {
202208
out->set_visible_ts_ms(in.visible_ts_ms());
203209
}
@@ -310,6 +316,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, const RowsetMetaCloudPB& in)
310316
if (in.has_reference_instance_id()) {
311317
out->set_reference_instance_id(in.reference_instance_id());
312318
}
319+
if (in.has_inverted_index_storage_format()) {
320+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
321+
}
313322
auto* slice_locations = out->mutable_packed_slice_locations();
314323
slice_locations->clear();
315324
slice_locations->insert(in.packed_slice_locations().begin(), in.packed_slice_locations().end());
@@ -405,6 +414,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in) {
405414
if (in.has_reference_instance_id()) {
406415
out->set_reference_instance_id(in.reference_instance_id());
407416
}
417+
if (in.has_inverted_index_storage_format()) {
418+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
419+
}
408420
auto* slice_locations = out->mutable_packed_slice_locations();
409421
slice_locations->clear();
410422
slice_locations->insert(in.packed_slice_locations().begin(), in.packed_slice_locations().end());
@@ -790,6 +802,9 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, const TabletMetaPB& in)
790802
if (in.has_encryption_algorithm()) {
791803
out->set_encryption_algorithm(in.encryption_algorithm());
792804
}
805+
if (in.has_inverted_index_storage_format()) {
806+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
807+
}
793808
}
794809

795810
void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) {
@@ -875,6 +890,9 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) {
875890
if (in.has_encryption_algorithm()) {
876891
out->set_encryption_algorithm(in.encryption_algorithm());
877892
}
893+
if (in.has_inverted_index_storage_format()) {
894+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
895+
}
878896
}
879897

880898
TabletMetaPB cloud_tablet_meta_to_doris(const TabletMetaCloudPB& in) {
@@ -967,6 +985,9 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, const TabletMetaCloudPB& in)
967985
if (in.has_encryption_algorithm()) {
968986
out->set_encryption_algorithm(in.encryption_algorithm());
969987
}
988+
if (in.has_inverted_index_storage_format()) {
989+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
990+
}
970991
}
971992

972993
void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
@@ -1052,6 +1073,9 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
10521073
if (in.has_encryption_algorithm()) {
10531074
out->set_encryption_algorithm(in.encryption_algorithm());
10541075
}
1076+
if (in.has_inverted_index_storage_format()) {
1077+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
1078+
}
10551079
}
10561080

10571081
} // namespace doris::cloud

‎cloud/src/recycler/checker.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -772,10 +772,13 @@ int InstanceChecker::do_check() {
772772
}
773773
if (!index_ids.empty()) {
774774
const auto& index_map = rs_meta.packed_slice_locations();
775+
const auto index_format =
776+
rs_meta.has_inverted_index_storage_format()
777+
? rs_meta.inverted_index_storage_format()
778+
: rs_meta.tablet_schema().inverted_index_storage_format();
775779
for (int i = 0; i < rs_meta.num_segments(); ++i) {
776780
std::vector<std::string> index_path_v;
777-
if (rs_meta.tablet_schema().inverted_index_storage_format() ==
778-
InvertedIndexStorageFormatPB::V1) {
781+
if (index_format == InvertedIndexStorageFormatPB::V1) {
779782
for (const auto& index_id : index_ids) {
780783
LOG(INFO) << "check inverted index, tablet_id=" << rs_meta.tablet_id()
781784
<< " rowset_id=" << rs_meta.rowset_id_v2() << " segment_id=" << i

‎cloud/src/recycler/recycler.cpp‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3471,8 +3471,9 @@ int InstanceRecycler::delete_rowset_data(const RowsetMetaCloudPB& rs_meta_pb) {
34713471

34723472
// Process inverted indexes
34733473
std::vector<std::pair<int64_t, std::string>> index_ids;
3474-
// default format as v1.
3475-
InvertedIndexStorageFormatPB index_format = InvertedIndexStorageFormatPB::V1;
3474+
InvertedIndexStorageFormatPB index_format = rs_meta_pb.has_inverted_index_storage_format()
3475+
? rs_meta_pb.inverted_index_storage_format()
3476+
: InvertedIndexStorageFormatPB::V1;
34763477
bool delete_rowset_data_by_prefix = false;
34773478
if (rs_meta_pb.rowset_state() == RowsetStatePB::BEGIN_PARTIAL_UPDATE) {
34783479
// if rowset state is RowsetStatePB::BEGIN_PARTIAL_UPDATE, the number of segments data
@@ -3484,7 +3485,8 @@ int InstanceRecycler::delete_rowset_data(const RowsetMetaCloudPB& rs_meta_pb) {
34843485
index_ids.emplace_back(index.index_id(), index.index_suffix_name());
34853486
}
34863487
}
3487-
if (rs_meta_pb.tablet_schema().has_inverted_index_storage_format()) {
3488+
if (!rs_meta_pb.has_inverted_index_storage_format() &&
3489+
rs_meta_pb.tablet_schema().has_inverted_index_storage_format()) {
34883490
index_format = rs_meta_pb.tablet_schema().inverted_index_storage_format();
34893491
}
34903492
} else if (!rs_meta_pb.has_index_id() || !rs_meta_pb.has_schema_version()) {
@@ -3498,7 +3500,9 @@ int InstanceRecycler::delete_rowset_data(const RowsetMetaCloudPB& rs_meta_pb) {
34983500
TEST_SYNC_POINT_CALLBACK("InstanceRecycler::delete_rowset_data.tmp_rowset",
34993501
&inverted_index_get_ret);
35003502
if (inverted_index_get_ret == 0) {
3501-
index_format = index_info.first;
3503+
if (!rs_meta_pb.has_inverted_index_storage_format()) {
3504+
index_format = index_info.first;
3505+
}
35023506
index_ids = index_info.second;
35033507
} else if (inverted_index_get_ret == 1) {
35043508
// 1. Schema kv not found means tablet has been recycled
@@ -4236,15 +4240,18 @@ int InstanceRecycler::delete_rowset_data(
42364240
// Process inverted indexes
42374241
std::vector<std::pair<int64_t, std::string>> index_ids;
42384242
// default format as v1.
4239-
InvertedIndexStorageFormatPB index_format = InvertedIndexStorageFormatPB::V1;
4243+
InvertedIndexStorageFormatPB index_format = rs.has_inverted_index_storage_format()
4244+
? rs.inverted_index_storage_format()
4245+
: InvertedIndexStorageFormatPB::V1;
42404246
int inverted_index_get_ret = 0;
42414247
if (rs.has_tablet_schema()) {
42424248
for (const auto& index : rs.tablet_schema().index()) {
42434249
if (index.has_index_type() && index.index_type() == IndexType::INVERTED) {
42444250
index_ids.emplace_back(index.index_id(), index.index_suffix_name());
42454251
}
42464252
}
4247-
if (rs.tablet_schema().has_inverted_index_storage_format()) {
4253+
if (!rs.has_inverted_index_storage_format() &&
4254+
rs.tablet_schema().has_inverted_index_storage_format()) {
42484255
index_format = rs.tablet_schema().inverted_index_storage_format();
42494256
}
42504257
} else {
@@ -4262,7 +4269,9 @@ int InstanceRecycler::delete_rowset_data(
42624269
TEST_SYNC_POINT_CALLBACK("InstanceRecycler::delete_rowset_data.tmp_rowset",
42634270
&inverted_index_get_ret);
42644271
if (inverted_index_get_ret == 0) {
4265-
index_format = index_info.first;
4272+
if (!rs.has_inverted_index_storage_format()) {
4273+
index_format = index_info.first;
4274+
}
42664275
index_ids = index_info.second;
42674276
} else if (inverted_index_get_ret == 1) {
42684277
// 1. Schema kv not found means tablet has been recycled

‎cloud/test/recycler_test.cpp‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8066,6 +8066,71 @@ TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v2) {
80668066
}
80678067
}
80688068

8069+
TEST(RecyclerTest, delete_tmp_rowset_data_with_rowset_idx_v3) {
8070+
auto txn_kv = std::make_shared<MemTxnKv>();
8071+
ASSERT_EQ(txn_kv->init(), 0);
8072+
8073+
InstanceInfoPB instance;
8074+
instance.set_instance_id(instance_id);
8075+
auto obj_info = instance.add_obj_info();
8076+
obj_info->set_id("delete_tmp_rowset_data_with_rowset_idx_v3");
8077+
obj_info->set_ak(config::test_s3_ak);
8078+
obj_info->set_sk(config::test_s3_sk);
8079+
obj_info->set_endpoint(config::test_s3_endpoint);
8080+
obj_info->set_region(config::test_s3_region);
8081+
obj_info->set_bucket(config::test_s3_bucket);
8082+
obj_info->set_prefix("delete_tmp_rowset_data_with_rowset_idx_v3");
8083+
8084+
doris::TabletSchemaCloudPB schema;
8085+
schema.set_schema_version(1);
8086+
schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1);
8087+
auto index = schema.add_index();
8088+
index->set_index_id(1);
8089+
index->set_index_type(IndexType::INVERTED);
8090+
8091+
{
8092+
InstanceRecycler recycler(txn_kv, instance, thread_group,
8093+
std::make_shared<TxnLazyCommitter>(txn_kv));
8094+
ASSERT_EQ(recycler.init(), 0);
8095+
auto accessor = recycler.accessor_map_.begin()->second;
8096+
std::map<std::string, doris::RowsetMetaCloudPB> rowset_pbs;
8097+
doris::RowsetMetaCloudPB rowset;
8098+
rowset.set_rowset_id(0); // useless but required
8099+
rowset.set_rowset_id_v2("1");
8100+
rowset.set_num_segments(1);
8101+
rowset.set_tablet_id(10000);
8102+
rowset.set_index_id(10001);
8103+
rowset.set_resource_id("delete_tmp_rowset_data_with_rowset_idx_v3");
8104+
rowset.set_schema_version(schema.schema_version());
8105+
rowset.mutable_tablet_schema()->CopyFrom(schema);
8106+
rowset.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3);
8107+
create_tmp_rowset(txn_kv.get(), accessor.get(), rowset, 1, true);
8108+
rowset_pbs.emplace(rowset.rowset_id_v2(), rowset);
8109+
8110+
std::unordered_set<std::string> list_files;
8111+
std::unique_ptr<ListIterator> iter;
8112+
EXPECT_EQ(accessor->list_all(&iter), 0);
8113+
EXPECT_TRUE(iter->has_next());
8114+
for (auto file = iter->next(); file.has_value(); file = iter->next()) {
8115+
list_files.insert(file->path);
8116+
}
8117+
EXPECT_EQ(list_files.size(), 2);
8118+
EXPECT_TRUE(list_files.contains("data/10000/1_0.dat"));
8119+
EXPECT_TRUE(list_files.contains("data/10000/1_0.idx"));
8120+
8121+
ASSERT_EQ(0,
8122+
recycler.delete_rowset_data(rowset_pbs, RowsetRecyclingState::TMP_ROWSET, ctx));
8123+
list_files.clear();
8124+
iter.reset();
8125+
EXPECT_EQ(accessor->list_all(&iter), 0);
8126+
EXPECT_FALSE(iter->has_next());
8127+
for (auto file = iter->next(); file.has_value(); file = iter->next()) {
8128+
list_files.insert(file->path);
8129+
}
8130+
EXPECT_EQ(list_files.size(), 0);
8131+
}
8132+
}
8133+
80698134
TEST(RecyclerTest, delete_tmp_rowset_without_resource_id) {
80708135
auto* sp = SyncPoint::get_instance();
80718136
DORIS_CLOUD_DEFER {

‎gensrc/proto/olap_file.proto‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,9 @@ message RowsetMetaPB {
183183
// For row binlog LSN allocation (FE auto-inc RPC)
184184
optional int64 db_id = 1016;
185185
optional int64 table_id = 1017;
186+
// Actual inverted-index format used by this rowset's files. It is independent
187+
// from the shared (index_id, schema_version) schema key.
188+
optional InvertedIndexStorageFormatPB inverted_index_storage_format = 1018;
186189
}
187190

188191
message SchemaDictKeyList {
@@ -301,6 +304,9 @@ message RowsetMetaCloudPB {
301304
optional int64 db_id = 115;
302305
optional int64 table_id = 116;
303306

307+
// Actual inverted-index format used by this rowset's files. It is independent
308+
// from the shared (index_id, schema_version) schema key.
309+
optional InvertedIndexStorageFormatPB inverted_index_storage_format = 117;
304310
}
305311

306312
message SegmentStatisticsPB {
@@ -733,6 +739,8 @@ message TabletMetaPB {
733739
optional string table_name = 1003;
734740
optional int64 ttl_seconds = 1004;
735741
optional int32 schema_version = 1005; // index_id, schema_version -> schema
742+
// Immutable format of this tablet's inverted-index files.
743+
optional InvertedIndexStorageFormatPB inverted_index_storage_format = 1006;
736744
}
737745

738746
message TabletMetaCloudPB {
@@ -792,6 +800,8 @@ message TabletMetaCloudPB {
792800
optional string table_name = 101;
793801
optional int64 ttl_seconds = 102;
794802
optional int32 schema_version = 103; // index_id, schema_version -> schema
803+
// Immutable format of this tablet's inverted-index files.
804+
optional InvertedIndexStorageFormatPB inverted_index_storage_format = 104;
795805
}
796806

797807
message OLAPRawDeltaHeaderMessage {

0 commit comments

Comments
 (0)