2222#include < stdlib.h>
2323#include < unistd.h>
2424
25+ #include < array>
26+ #include < map>
2527#include < memory>
2628#include < string>
2729#include < vector>
3941#include " storage/storage_engine.h"
4042#include " storage/tablet/tablet.h"
4143#include " storage/tablet/tablet_manager.h"
44+ #include " storage/tablet/tablet_meta_manager.h"
4245#include " storage/tablet_info.h"
4346#include " testutil/creators.h"
4447
@@ -47,14 +50,7 @@ namespace doris {
4750static const uint32_t MAX_PATH_LEN = 1024 ;
4851static StorageEngine* engine_ref = nullptr ;
4952
50- static void set_up () {
51- char buffer[MAX_PATH_LEN ];
52- EXPECT_NE (getcwd (buffer, MAX_PATH_LEN ), nullptr );
53- config::storage_root_path = std::string (buffer) + " /data_test" ;
54- auto st = io::global_local_filesystem ()->delete_directory (config::storage_root_path);
55- ASSERT_TRUE (st.ok ()) << st;
56- st = io::global_local_filesystem ()->create_directory (config::storage_root_path);
57- ASSERT_TRUE (st.ok ()) << st;
53+ static void open_engine () {
5854 std::vector<StorePath> paths;
5955 paths.emplace_back (config::storage_root_path, -1 );
6056
@@ -64,10 +60,26 @@ static void set_up() {
6460 engine_ref = engine.get ();
6561 Status s = engine->open ();
6662 ASSERT_TRUE (s.ok ()) << s;
63+ ExecEnv::GetInstance ()->set_storage_engine (std::move (engine));
64+ }
6765
66+ static void set_up () {
67+ char buffer[MAX_PATH_LEN ];
68+ EXPECT_NE (getcwd (buffer, MAX_PATH_LEN ), nullptr );
69+ config::storage_root_path = std::string (buffer) + " /data_test" ;
70+ auto st = io::global_local_filesystem ()->delete_directory (config::storage_root_path);
71+ ASSERT_TRUE (st.ok ()) << st;
72+ st = io::global_local_filesystem ()->create_directory (config::storage_root_path);
73+ ASSERT_TRUE (st.ok ()) << st;
6874 ExecEnv* exec_env = doris::ExecEnv::GetInstance ();
6975 exec_env->set_memtable_memory_limiter (new MemTableMemoryLimiter ());
70- exec_env->set_storage_engine (std::move (engine));
76+ open_engine ();
77+ }
78+
79+ static void restart_engine () {
80+ engine_ref = nullptr ;
81+ ExecEnv::GetInstance ()->set_storage_engine (nullptr );
82+ open_engine ();
7183}
7284
7385static void tear_down () {
@@ -167,4 +179,105 @@ TEST_F(GroupRowsetBuilderTest, buildWithRowBinlogMeta) {
167179 ASSERT_TRUE (res.ok ());
168180}
169181
182+ TEST_F (GroupRowsetBuilderTest, recoverMultipleRowBinlogPairsInOneTxn) {
183+ constexpr int64_t partition_id = 10100 ;
184+ constexpr int64_t txn_id = 20100 ;
185+ constexpr int64_t index_id = 30100 ;
186+ constexpr int64_t row_binlog_index_id = 30101 ;
187+ constexpr int32_t schema_hash = 40100 ;
188+ constexpr int32_t row_binlog_schema_hash = 40101 ;
189+ constexpr std::array<std::pair<int64_t , int64_t >, 2 > tablet_pairs = {std::pair {10100 , 10101 },
190+ std::pair {10200 , 10201 }};
191+
192+ auto base_request = testutil::create_tablet_request (
193+ 0 , schema_hash, partition_id, 1 , TKeysType::UNIQUE_KEYS ,
194+ {{" k1" , TPrimitiveType::INT , true }, {" v1" , TPrimitiveType::INT , false }});
195+ base_request.__set_enable_unique_key_merge_on_write (true );
196+ testutil::enable_row_binlog (&base_request);
197+ auto row_binlog_schema = testutil::create_row_binlog_tablet_schema (base_request.tablet_schema ,
198+ row_binlog_schema_hash);
199+
200+ RuntimeProfile profile (" CreateTablet" );
201+ for (const auto & [base_tablet_id, row_binlog_tablet_id] : tablet_pairs) {
202+ base_request.tablet_id = base_tablet_id;
203+ ASSERT_TRUE (engine_ref->create_tablet (base_request, &profile).ok ());
204+
205+ auto row_binlog_request = base_request;
206+ row_binlog_request.tablet_id = row_binlog_tablet_id;
207+ row_binlog_request.tablet_schema = row_binlog_schema;
208+ row_binlog_request.__set_base_tablet_id (base_tablet_id);
209+ row_binlog_request.__set_tablet_role (TTabletRole::TABLET_ROLE_ROW_BINLOG );
210+ ASSERT_TRUE (engine_ref->create_tablet (row_binlog_request, &profile).ok ());
211+
212+ auto base_tablet = engine_ref->tablet_manager ()->get_tablet (base_tablet_id);
213+ ASSERT_NE (base_tablet, nullptr );
214+ TabletMetaPB in_memory_meta_pb;
215+ base_tablet->tablet_meta ()->to_meta_pb (&in_memory_meta_pb, false );
216+ EXPECT_EQ (in_memory_meta_pb.binlog_tablet_id (), row_binlog_tablet_id);
217+
218+ TabletMetaSharedPtr persisted_meta = std::make_shared<TabletMeta>();
219+ ASSERT_TRUE (TabletMetaManager::get_meta (base_tablet->data_dir (), base_tablet_id,
220+ schema_hash, persisted_meta)
221+ .ok ());
222+ TabletMetaPB persisted_meta_pb;
223+ persisted_meta->to_meta_pb (&persisted_meta_pb, false );
224+ EXPECT_EQ (persisted_meta_pb.binlog_tablet_id (), row_binlog_tablet_id);
225+ }
226+
227+ TDescriptorTable tdesc_tbl =
228+ testutil::create_descriptor_table ({{TYPE_INT , " k1" , false }, {TYPE_INT , " v1" , false }});
229+ auto schema_param = testutil::create_table_schema_param (
230+ tdesc_tbl, index_id, schema_hash, base_request.tablet_schema .columns ,
231+ row_binlog_index_id, row_binlog_schema_hash, &row_binlog_schema.columns );
232+ ASSERT_NE (schema_param, nullptr );
233+
234+ PUniqueId load_id;
235+ load_id.set_hi (0 );
236+ load_id.set_lo (1 );
237+ for (const auto & [base_tablet_id, row_binlog_tablet_id] : tablet_pairs) {
238+ WriteRequest data_req;
239+ data_req.tablet_id = base_tablet_id;
240+ data_req.schema_hash = schema_hash;
241+ data_req.txn_id = txn_id;
242+ data_req.partition_id = partition_id;
243+ data_req.index_id = index_id;
244+ data_req.load_id = load_id;
245+ data_req.table_schema_param = schema_param;
246+ data_req.write_req_type = WriteRequestType::DATA ;
247+
248+ WriteRequest row_binlog_req = data_req;
249+ row_binlog_req.tablet_id = row_binlog_tablet_id;
250+ row_binlog_req.index_id = row_binlog_index_id;
251+ row_binlog_req.schema_hash = row_binlog_schema_hash;
252+ row_binlog_req.write_req_type = WriteRequestType::ROW_BINLOG ;
253+
254+ WriteRequest group_req = data_req;
255+ group_req.write_req_type = WriteRequestType::GROUP ;
256+
257+ GroupRowsetBuilder builder (*engine_ref, group_req, data_req, row_binlog_req, &profile);
258+ ASSERT_TRUE (builder.init ().ok ());
259+ ASSERT_TRUE (builder.rowset_writer ()->flush ().ok ());
260+ ASSERT_TRUE (builder.build_rowset ().ok ());
261+ ASSERT_TRUE (builder.commit_txn ().ok ());
262+ }
263+
264+ restart_engine ();
265+
266+ std::map<TabletInfo, RowsetSharedPtr> rowsets;
267+ std::map<TabletInfo, std::shared_ptr<TabletTxnInfo>> txn_infos;
268+ engine_ref->txn_manager ()->get_txn_related_tablets (txn_id, partition_id, &rowsets, &txn_infos);
269+ ASSERT_EQ (txn_infos.size (), tablet_pairs.size ());
270+ for (const auto & [base_tablet_id, row_binlog_tablet_id] : tablet_pairs) {
271+ auto base_tablet = engine_ref->tablet_manager ()->get_tablet (base_tablet_id);
272+ ASSERT_NE (base_tablet, nullptr );
273+ auto txn_info = txn_infos.find (base_tablet->get_tablet_info ());
274+ ASSERT_NE (txn_info, txn_infos.end ());
275+ ASSERT_NE (txn_info->second ->attach_row_binlog .tablet , nullptr );
276+ ASSERT_NE (txn_info->second ->attach_row_binlog .rowset , nullptr );
277+ EXPECT_EQ (txn_info->second ->attach_row_binlog .tablet ->tablet_id (), row_binlog_tablet_id);
278+ EXPECT_EQ (txn_info->second ->attach_row_binlog .rowset ->rowset_meta ()->tablet_id (),
279+ row_binlog_tablet_id);
280+ }
281+ }
282+
170283} // namespace doris
0 commit comments