Skip to content

Commit 798eb24

Browse files
authored
[fix](binlog) Fix row binlog recovery for multi-tablet transactions (#67508)
### What problem does this PR solve? Issue Number: close #67091 Related PR: None Problem Summary: A BE restart indexed row-binlog rowsets only by transaction ID, so multiple tablet pairs in one transaction could attach the wrong companion rowset. Persist each base tablet's companion ID and recover using both transaction and tablet IDs. ### Release note Fix incorrect row-binlog companion recovery after a BE restart for multi-tablet transactions. ### Check List (For Author) - Test: Unit Test - Added and ran GroupRowsetBuilderTest.recoverMultipleRowBinlogPairsInOneTxn and GroupRowsetBuilderTest.* - Behavior changed: Yes. BE restart recovery now attaches each base rowset to its persisted companion tablet. - Does this need documentation: No
1 parent e1df149 commit 798eb24

5 files changed

Lines changed: 161 additions & 15 deletions

File tree

‎be/src/storage/data_dir.cpp‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -520,11 +520,11 @@ Status DataDir::load() {
520520
}
521521

522522
// Row binlog rowset is now a normal rowset under its own binlog tablet, loaded above.
523-
// Index them by txn id so each base rowset can re-attach its paired binlog rowset on recovery.
524-
std::map<int64_t, RowsetMetaSharedPtr> txn_id_to_row_binlog_meta;
523+
// Index them by txn and tablet id so each base rowset can re-attach its paired binlog rowset.
524+
std::map<std::pair<int64_t, int64_t>, RowsetMetaSharedPtr> row_binlog_metas;
525525
for (auto&& rowset_meta : dir_rowset_metas) {
526526
if (rowset_meta->is_row_binlog()) {
527-
txn_id_to_row_binlog_meta[rowset_meta->txn_id()] = rowset_meta;
527+
row_binlog_metas[{rowset_meta->txn_id(), rowset_meta->tablet_id()}] = rowset_meta;
528528
}
529529
}
530530

@@ -558,8 +558,9 @@ Status DataDir::load() {
558558
}
559559

560560
RowBinlogTxnInfo attach_row_binlog;
561-
if (auto it = txn_id_to_row_binlog_meta.find(rowset_meta->txn_id());
562-
it != txn_id_to_row_binlog_meta.end()) {
561+
if (auto it = row_binlog_metas.find(
562+
{rowset_meta->txn_id(), tablet->tablet_meta()->binlog_tablet_id()});
563+
it != row_binlog_metas.end()) {
563564
const RowsetMetaSharedPtr& attach_row_binlog_rowset_meta = it->second;
564565
DCHECK_EQ(attach_row_binlog_rowset_meta->rowset_state(), rowset_meta->rowset_state());
565566
TabletSharedPtr binlog_tablet = _engine.tablet_manager()->get_tablet(

‎be/src/storage/tablet/tablet_manager.cpp‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -288,9 +288,11 @@ Status TabletManager::create_tablet(const TCreateTabletReq& request, std::vector
288288
// same) already exist, then just return true(an duplicate request). But if
289289
// tablet_id exist but with different schema_hash, return an error(report task will
290290
// eventually trigger its deletion).
291+
bool tablet_exists = false;
291292
{
292293
SCOPED_TIMER(ADD_TIMER(profile, "GetTabletUnlocked"));
293-
if (_get_tablet_unlocked(tablet_id) != nullptr) {
294+
tablet_exists = _get_tablet_unlocked(tablet_id) != nullptr;
295+
if (tablet_exists && !is_colocated_row_binlog) {
294296
LOG(INFO) << "success to create tablet. tablet already exist. tablet_id=" << tablet_id;
295297
return Status::OK();
296298
}
@@ -331,6 +333,24 @@ Status TabletManager::create_tablet(const TCreateTabletReq& request, std::vector
331333
}
332334
}
333335

336+
auto persist_row_binlog_pair = [&]() {
337+
CHECK(is_colocated_row_binlog);
338+
std::lock_guard base_tablet_wlock(base_tablet->get_header_lock());
339+
CHECK(base_tablet->tablet_meta()->binlog_tablet_id() == 0 ||
340+
base_tablet->tablet_meta()->binlog_tablet_id() == tablet_id)
341+
<< "base tablet " << base_tablet->tablet_id()
342+
<< " is already paired with row-binlog tablet "
343+
<< base_tablet->tablet_meta()->binlog_tablet_id() << ", new row-binlog tablet "
344+
<< tablet_id;
345+
base_tablet->tablet_meta()->set_binlog_tablet_id(tablet_id);
346+
base_tablet->save_meta();
347+
};
348+
if (tablet_exists) {
349+
persist_row_binlog_pair();
350+
LOG(INFO) << "success to create tablet. tablet already exist. tablet_id=" << tablet_id;
351+
return Status::OK();
352+
}
353+
334354
TabletSharedPtr tablet = _internal_create_tablet_unlocked(
335355
request, is_schema_change_or_atomic_restore, is_colocated_row_binlog, base_tablet.get(),
336356
stores, profile);
@@ -339,6 +359,9 @@ Status TabletManager::create_tablet(const TCreateTabletReq& request, std::vector
339359
return Status::Error<CE_CMD_PARAMS_ERROR>("fail to create tablet. tablet_id={}",
340360
request.tablet_id);
341361
}
362+
if (is_colocated_row_binlog) {
363+
persist_row_binlog_pair();
364+
}
342365

343366
LOG(INFO) << "success to create tablet. tablet_id=" << tablet_id
344367
<< ", tablet_path=" << tablet->tablet_path();

‎be/src/storage/tablet/tablet_meta.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,7 @@ TabletMeta::TabletMeta(const TabletMeta& b)
269269
_delete_bitmap(b._delete_bitmap),
270270
_binlog_config(b._binlog_config),
271271
_tablet_role(b._tablet_role),
272+
_binlog_tablet_id(b._binlog_tablet_id),
272273
_compaction_policy(b._compaction_policy),
273274
_time_series_compaction_goal_size_mbytes(b._time_series_compaction_goal_size_mbytes),
274275
_time_series_compaction_file_count_threshold(
@@ -905,6 +906,7 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) {
905906
_binlog_config = tablet_meta_pb.binlog_config();
906907
}
907908
_tablet_role = tablet_meta_pb.tablet_role();
909+
_binlog_tablet_id = tablet_meta_pb.binlog_tablet_id();
908910
_compaction_policy = tablet_meta_pb.compaction_policy();
909911
_time_series_compaction_goal_size_mbytes =
910912
tablet_meta_pb.time_series_compaction_goal_size_mbytes();
@@ -1008,6 +1010,9 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb, bool cloud_get_rowset_
10081010
}
10091011
_binlog_config.to_pb(tablet_meta_pb->mutable_binlog_config());
10101012
tablet_meta_pb->set_tablet_role(_tablet_role);
1013+
if (_binlog_tablet_id > 0) {
1014+
tablet_meta_pb->set_binlog_tablet_id(_binlog_tablet_id);
1015+
}
10111016
tablet_meta_pb->set_compaction_policy(compaction_policy());
10121017
tablet_meta_pb->set_time_series_compaction_goal_size_mbytes(
10131018
time_series_compaction_goal_size_mbytes());
@@ -1252,6 +1257,7 @@ bool operator==(const TabletMeta& a, const TabletMeta& b) {
12521257
if (a._in_restore_mode != b._in_restore_mode) return false;
12531258
if (a._preferred_rowset_type != b._preferred_rowset_type) return false;
12541259
if (a._storage_policy_id != b._storage_policy_id) return false;
1260+
if (a._binlog_tablet_id != b._binlog_tablet_id) return false;
12551261
if (a._compaction_policy != b._compaction_policy) return false;
12561262
if (a._time_series_compaction_goal_size_mbytes != b._time_series_compaction_goal_size_mbytes)
12571263
return false;

‎be/src/storage/tablet/tablet_meta.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -280,6 +280,8 @@ class TabletMeta : public MetadataAdder<TabletMeta> {
280280
return _tablet_role == TabletRolePB::TABLET_ROLE_ROW_BINLOG;
281281
}
282282
void set_tablet_role(TabletRolePB tablet_role) { _tablet_role = tablet_role; }
283+
int64_t binlog_tablet_id() const { return _binlog_tablet_id; }
284+
void set_binlog_tablet_id(int64_t binlog_tablet_id) { _binlog_tablet_id = binlog_tablet_id; }
283285

284286
void set_compaction_policy(std::string compaction_policy) {
285287
_compaction_policy = compaction_policy;
@@ -395,6 +397,7 @@ class TabletMeta : public MetadataAdder<TabletMeta> {
395397
// binlog config
396398
BinlogConfig _binlog_config {};
397399
TabletRolePB _tablet_role = TabletRolePB::TABLET_ROLE_DATA;
400+
int64_t _binlog_tablet_id = 0;
398401

399402
// meta for compaction
400403
std::string _compaction_policy;

‎be/test/olap/rowset/group_rowset_builder_test.cpp‎

Lines changed: 122 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
#include <stdlib.h>
2323
#include <unistd.h>
2424

25+
#include <array>
26+
#include <map>
2527
#include <memory>
2628
#include <string>
2729
#include <vector>
@@ -39,6 +41,7 @@
3941
#include "storage/storage_engine.h"
4042
#include "storage/tablet/tablet.h"
4143
#include "storage/tablet/tablet_manager.h"
44+
#include "storage/tablet/tablet_meta_manager.h"
4245
#include "storage/tablet_info.h"
4346
#include "testutil/creators.h"
4447

@@ -47,14 +50,7 @@ namespace doris {
4750
static const uint32_t MAX_PATH_LEN = 1024;
4851
static StorageEngine* engine_ref = nullptr;
4952

50-
static void set_up() {
51-
char buffer[MAX_PATH_LEN];
52-
EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr);
53-
config::storage_root_path = std::string(buffer) + "/data_test";
54-
auto st = io::global_local_filesystem()->delete_directory(config::storage_root_path);
55-
ASSERT_TRUE(st.ok()) << st;
56-
st = io::global_local_filesystem()->create_directory(config::storage_root_path);
57-
ASSERT_TRUE(st.ok()) << st;
53+
static void open_engine() {
5854
std::vector<StorePath> paths;
5955
paths.emplace_back(config::storage_root_path, -1);
6056

@@ -64,10 +60,26 @@ static void set_up() {
6460
engine_ref = engine.get();
6561
Status s = engine->open();
6662
ASSERT_TRUE(s.ok()) << s;
63+
ExecEnv::GetInstance()->set_storage_engine(std::move(engine));
64+
}
6765

66+
static void set_up() {
67+
char buffer[MAX_PATH_LEN];
68+
EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr);
69+
config::storage_root_path = std::string(buffer) + "/data_test";
70+
auto st = io::global_local_filesystem()->delete_directory(config::storage_root_path);
71+
ASSERT_TRUE(st.ok()) << st;
72+
st = io::global_local_filesystem()->create_directory(config::storage_root_path);
73+
ASSERT_TRUE(st.ok()) << st;
6874
ExecEnv* exec_env = doris::ExecEnv::GetInstance();
6975
exec_env->set_memtable_memory_limiter(new MemTableMemoryLimiter());
70-
exec_env->set_storage_engine(std::move(engine));
76+
open_engine();
77+
}
78+
79+
static void restart_engine() {
80+
engine_ref = nullptr;
81+
ExecEnv::GetInstance()->set_storage_engine(nullptr);
82+
open_engine();
7183
}
7284

7385
static void tear_down() {
@@ -167,4 +179,105 @@ TEST_F(GroupRowsetBuilderTest, buildWithRowBinlogMeta) {
167179
ASSERT_TRUE(res.ok());
168180
}
169181

182+
TEST_F(GroupRowsetBuilderTest, recoverMultipleRowBinlogPairsInOneTxn) {
183+
constexpr int64_t partition_id = 10100;
184+
constexpr int64_t txn_id = 20100;
185+
constexpr int64_t index_id = 30100;
186+
constexpr int64_t row_binlog_index_id = 30101;
187+
constexpr int32_t schema_hash = 40100;
188+
constexpr int32_t row_binlog_schema_hash = 40101;
189+
constexpr std::array<std::pair<int64_t, int64_t>, 2> tablet_pairs = {std::pair {10100, 10101},
190+
std::pair {10200, 10201}};
191+
192+
auto base_request = testutil::create_tablet_request(
193+
0, schema_hash, partition_id, 1, TKeysType::UNIQUE_KEYS,
194+
{{"k1", TPrimitiveType::INT, true}, {"v1", TPrimitiveType::INT, false}});
195+
base_request.__set_enable_unique_key_merge_on_write(true);
196+
testutil::enable_row_binlog(&base_request);
197+
auto row_binlog_schema = testutil::create_row_binlog_tablet_schema(base_request.tablet_schema,
198+
row_binlog_schema_hash);
199+
200+
RuntimeProfile profile("CreateTablet");
201+
for (const auto& [base_tablet_id, row_binlog_tablet_id] : tablet_pairs) {
202+
base_request.tablet_id = base_tablet_id;
203+
ASSERT_TRUE(engine_ref->create_tablet(base_request, &profile).ok());
204+
205+
auto row_binlog_request = base_request;
206+
row_binlog_request.tablet_id = row_binlog_tablet_id;
207+
row_binlog_request.tablet_schema = row_binlog_schema;
208+
row_binlog_request.__set_base_tablet_id(base_tablet_id);
209+
row_binlog_request.__set_tablet_role(TTabletRole::TABLET_ROLE_ROW_BINLOG);
210+
ASSERT_TRUE(engine_ref->create_tablet(row_binlog_request, &profile).ok());
211+
212+
auto base_tablet = engine_ref->tablet_manager()->get_tablet(base_tablet_id);
213+
ASSERT_NE(base_tablet, nullptr);
214+
TabletMetaPB in_memory_meta_pb;
215+
base_tablet->tablet_meta()->to_meta_pb(&in_memory_meta_pb, false);
216+
EXPECT_EQ(in_memory_meta_pb.binlog_tablet_id(), row_binlog_tablet_id);
217+
218+
TabletMetaSharedPtr persisted_meta = std::make_shared<TabletMeta>();
219+
ASSERT_TRUE(TabletMetaManager::get_meta(base_tablet->data_dir(), base_tablet_id,
220+
schema_hash, persisted_meta)
221+
.ok());
222+
TabletMetaPB persisted_meta_pb;
223+
persisted_meta->to_meta_pb(&persisted_meta_pb, false);
224+
EXPECT_EQ(persisted_meta_pb.binlog_tablet_id(), row_binlog_tablet_id);
225+
}
226+
227+
TDescriptorTable tdesc_tbl =
228+
testutil::create_descriptor_table({{TYPE_INT, "k1", false}, {TYPE_INT, "v1", false}});
229+
auto schema_param = testutil::create_table_schema_param(
230+
tdesc_tbl, index_id, schema_hash, base_request.tablet_schema.columns,
231+
row_binlog_index_id, row_binlog_schema_hash, &row_binlog_schema.columns);
232+
ASSERT_NE(schema_param, nullptr);
233+
234+
PUniqueId load_id;
235+
load_id.set_hi(0);
236+
load_id.set_lo(1);
237+
for (const auto& [base_tablet_id, row_binlog_tablet_id] : tablet_pairs) {
238+
WriteRequest data_req;
239+
data_req.tablet_id = base_tablet_id;
240+
data_req.schema_hash = schema_hash;
241+
data_req.txn_id = txn_id;
242+
data_req.partition_id = partition_id;
243+
data_req.index_id = index_id;
244+
data_req.load_id = load_id;
245+
data_req.table_schema_param = schema_param;
246+
data_req.write_req_type = WriteRequestType::DATA;
247+
248+
WriteRequest row_binlog_req = data_req;
249+
row_binlog_req.tablet_id = row_binlog_tablet_id;
250+
row_binlog_req.index_id = row_binlog_index_id;
251+
row_binlog_req.schema_hash = row_binlog_schema_hash;
252+
row_binlog_req.write_req_type = WriteRequestType::ROW_BINLOG;
253+
254+
WriteRequest group_req = data_req;
255+
group_req.write_req_type = WriteRequestType::GROUP;
256+
257+
GroupRowsetBuilder builder(*engine_ref, group_req, data_req, row_binlog_req, &profile);
258+
ASSERT_TRUE(builder.init().ok());
259+
ASSERT_TRUE(builder.rowset_writer()->flush().ok());
260+
ASSERT_TRUE(builder.build_rowset().ok());
261+
ASSERT_TRUE(builder.commit_txn().ok());
262+
}
263+
264+
restart_engine();
265+
266+
std::map<TabletInfo, RowsetSharedPtr> rowsets;
267+
std::map<TabletInfo, std::shared_ptr<TabletTxnInfo>> txn_infos;
268+
engine_ref->txn_manager()->get_txn_related_tablets(txn_id, partition_id, &rowsets, &txn_infos);
269+
ASSERT_EQ(txn_infos.size(), tablet_pairs.size());
270+
for (const auto& [base_tablet_id, row_binlog_tablet_id] : tablet_pairs) {
271+
auto base_tablet = engine_ref->tablet_manager()->get_tablet(base_tablet_id);
272+
ASSERT_NE(base_tablet, nullptr);
273+
auto txn_info = txn_infos.find(base_tablet->get_tablet_info());
274+
ASSERT_NE(txn_info, txn_infos.end());
275+
ASSERT_NE(txn_info->second->attach_row_binlog.tablet, nullptr);
276+
ASSERT_NE(txn_info->second->attach_row_binlog.rowset, nullptr);
277+
EXPECT_EQ(txn_info->second->attach_row_binlog.tablet->tablet_id(), row_binlog_tablet_id);
278+
EXPECT_EQ(txn_info->second->attach_row_binlog.rowset->rowset_meta()->tablet_id(),
279+
row_binlog_tablet_id);
280+
}
281+
}
282+
170283
} // namespace doris

0 commit comments

Comments
 (0)