Skip to content

Commit f8de8ab

Browse files
authored
Pick fix BE UT failure from apache#1661 apache#1678 (apache#1683)
* Fix BE UT failure introduced by apache#1526 in TabletSchemaCache (apache#1661) * Fix be ut * Revert "Fix BE UT failure introduced by apache#1526 in TabletSchemaCache (apache#1647)" This reverts commit bca5d32. * (selectdb-cloud) Modify debug checks for detached schema kv (apache#1678)
1 parent 1cb198e commit f8de8ab

10 files changed

Lines changed: 333 additions & 98 deletions

‎be/src/olap/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ add_library(Olap STATIC
7474
tablet_meta.cpp
7575
tablet_meta_manager.cpp
7676
tablet_schema.cpp
77+
tablet_schema_cache.cpp
7778
txn_manager.cpp
7879
types.cpp
7980
utils.cpp

‎be/src/olap/rowset/rowset_meta.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,6 @@ class RowsetMeta {
378378
int64_t newest_write_timestamp() const { return _rowset_meta_pb.newest_write_timestamp(); }
379379

380380
void set_tablet_schema(const TabletSchemaSPtr& tablet_schema) {
381-
DCHECK(index_id() > 0);
382381
_schema = TabletSchemaCache::instance()->insert(index_id(), tablet_schema);
383382
}
384383

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "olap/tablet_schema_cache.h"
19+
20+
#include <gen_cpp/olap_file.pb.h>
21+
22+
#include "common/sync_point.h"
23+
24+
namespace doris {
25+
26+
void TabletSchemaCache::create_global_schema_cache() {
27+
DCHECK(_s_instance == nullptr);
28+
static TabletSchemaCache instance;
29+
_s_instance = &instance;
30+
std::thread t(&TabletSchemaCache::_recycle, _s_instance);
31+
t.detach();
32+
}
33+
34+
TabletSchemaSPtr TabletSchemaCache::insert(int64_t index_id, const TabletSchemaPB& schema) {
35+
TEST_SYNC_POINT_RETURN_WITH_VALUE("TabletSchemaCache::insert1",
36+
std::make_shared<TabletSchema>());
37+
DCHECK(_s_instance != nullptr);
38+
DCHECK(index_id > 0);
39+
std::lock_guard guard(_mtx);
40+
auto iter = _cache.find({index_id, schema.schema_version()});
41+
if (iter == _cache.end()) {
42+
TabletSchemaSPtr tablet_schema_ptr = std::make_shared<TabletSchema>();
43+
tablet_schema_ptr->init_from_pb(schema);
44+
_cache.insert({{index_id, schema.schema_version()}, tablet_schema_ptr});
45+
return tablet_schema_ptr;
46+
}
47+
return iter->second;
48+
}
49+
50+
TabletSchemaSPtr TabletSchemaCache::insert(int64_t index_id, const TabletSchemaSPtr& schema) {
51+
TEST_SYNC_POINT_RETURN_WITH_VALUE("TabletSchemaCache::insert2",
52+
std::make_shared<TabletSchema>());
53+
DCHECK(_s_instance != nullptr);
54+
DCHECK(index_id > 0);
55+
std::lock_guard guard(_mtx);
56+
auto [it, _] = _cache.insert({{index_id, schema->schema_version()}, schema});
57+
return it->second;
58+
}
59+
60+
/**
61+
* @brief recycle when TabletSchemaSPtr use_count equals 1.
62+
*/
63+
void TabletSchemaCache::_recycle() {
64+
int64_t tablet_schema_cache_recycle_interval = 86400; // s, one day
65+
for (;;) {
66+
std::this_thread::sleep_for(std::chrono::seconds(tablet_schema_cache_recycle_interval));
67+
std::lock_guard guard(_mtx);
68+
for (auto iter = _cache.begin(), last = _cache.end(); iter != last;) {
69+
if (iter->second.unique()) {
70+
iter = _cache.erase(iter);
71+
} else {
72+
++iter;
73+
}
74+
}
75+
}
76+
}
77+
78+
} // namespace doris

‎be/src/olap/tablet_schema_cache.h‎

Lines changed: 5 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -17,68 +17,30 @@
1717

1818
#pragma once
1919

20-
#include <gen_cpp/olap_file.pb.h>
21-
2220
#include <memory>
2321
#include <mutex>
2422
#include <unordered_map>
2523

2624
#include "olap/tablet_schema.h"
2725

2826
namespace doris {
27+
class TabletSchemaPB;
2928

3029
class TabletSchemaCache {
3130
public:
32-
static void create_global_schema_cache() {
33-
DCHECK(_s_instance == nullptr);
34-
static TabletSchemaCache instance;
35-
_s_instance = &instance;
36-
std::thread t(&TabletSchemaCache::_recycle, _s_instance);
37-
t.detach();
38-
}
31+
static void create_global_schema_cache();
3932

4033
static TabletSchemaCache* instance() { return _s_instance; }
4134

42-
TabletSchemaSPtr insert(int64_t index_id, const TabletSchemaPB& schema) {
43-
DCHECK(_s_instance != nullptr);
44-
DCHECK(index_id > 0);
45-
std::lock_guard guard(_mtx);
46-
auto iter = _cache.find({index_id, schema.schema_version()});
47-
if (iter == _cache.end()) {
48-
TabletSchemaSPtr tablet_schema_ptr = std::make_shared<TabletSchema>();
49-
tablet_schema_ptr->init_from_pb(schema);
50-
_cache.insert({{index_id, schema.schema_version()}, tablet_schema_ptr});
51-
return tablet_schema_ptr;
52-
}
53-
return iter->second;
54-
}
35+
TabletSchemaSPtr insert(int64_t index_id, const TabletSchemaPB& schema);
5536

56-
TabletSchemaSPtr insert(int64_t index_id, const TabletSchemaSPtr& schema) {
57-
DCHECK(_s_instance != nullptr);
58-
DCHECK(index_id > 0);
59-
std::lock_guard guard(_mtx);
60-
auto [it, _] = _cache.insert({{index_id, schema->schema_version()}, schema});
61-
return it->second;
62-
}
37+
TabletSchemaSPtr insert(int64_t index_id, const TabletSchemaSPtr& schema);
6338

6439
private:
6540
/**
6641
* @brief recycle when TabletSchemaSPtr use_count equals 1.
6742
*/
68-
void _recycle() {
69-
int64_t tablet_schema_cache_recycle_interval = 86400; // s, one day
70-
for (;;) {
71-
std::this_thread::sleep_for(std::chrono::seconds(tablet_schema_cache_recycle_interval));
72-
std::lock_guard guard(_mtx);
73-
for (auto iter = _cache.begin(), last = _cache.end(); iter != last;) {
74-
if (iter->second.unique()) {
75-
iter = _cache.erase(iter);
76-
} else {
77-
++iter;
78-
}
79-
}
80-
}
81-
}
43+
void _recycle();
8244

8345
private:
8446
static inline TabletSchemaCache* _s_instance = nullptr;

‎be/test/CMakeLists.txt‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -352,10 +352,7 @@ if (CLOUD_MODE)
352352
olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp
353353
olap/rowset/segment_v2/zone_map_index_test.cpp
354354
olap/rowset/segment_v2/inverted_index_searcher_cache_test.cpp
355-
olap/tablet_meta_test.cpp
356-
olap/rowset/rowset_meta_manager_test.cpp
357-
olap/rowset/rowset_meta_test.cpp
358-
olap/rowset/rowset_tree_test.cpp
355+
#olap/rowset/rowset_tree_test.cpp
359356
olap/generic_iterators_test.cpp
360357
olap/key_coder_test.cpp
361358
olap/short_key_index_test.cpp
@@ -368,7 +365,6 @@ if (CLOUD_MODE)
368365
olap/common_test.cpp
369366
#olap/segcompaction_test.cpp
370367
#olap/ordered_data_compaction_test.cpp
371-
olap/cumulative_compaction_policy_test.cpp
372368
)
373369
else()
374370
set(OLAP_TEST_FILES
@@ -471,6 +467,7 @@ set(CLOUD_TEST_FILES
471467
cloud/cloud_tablet_test.cpp
472468
cloud/cloud_prefetch_test.cpp
473469
cloud/cloud_file_cache_test.cpp
470+
cloud/cumulative_compaction_policy_test.cpp
474471
)
475472

476473
if (CLOUD_MODE)

‎be/test/cloud/cloud_tablet_mgr_test.cpp‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,10 @@ class MockMetaMgr final : public MetaMgr {
2929

3030
TEST(CloudTabletMgrTest, normal) {
3131
auto sp = SyncPoint::get_instance();
32-
sp->clear_all_call_backs();
32+
Defer defer {[sp] {
33+
sp->clear_call_back("CloudTabletMgr::get_tablet");
34+
sp->clear_call_back("meta_mgr");
35+
}};
3336
sp->enable_processing();
3437

3538
MockMetaMgr mock_meta_mgr;
@@ -86,7 +89,10 @@ TEST(CloudTabletMgrTest, normal) {
8689

8790
TEST(CloudTabletMgrTest, concurrent) {
8891
auto sp = SyncPoint::get_instance();
89-
sp->clear_all_call_backs();
92+
Defer defer {[sp] {
93+
sp->clear_call_back("meta_mgr");
94+
sp->clear_call_back("MockMetaMgr::get_tablet_meta");
95+
}};
9096
sp->enable_processing();
9197

9298
MockMetaMgr mock_meta_mgr;
Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "olap/cumulative_compaction_policy.h"
19+
20+
#include <gtest/gtest.h>
21+
22+
#include <sstream>
23+
24+
#include "olap/rowset/rowset_factory.h"
25+
#include "olap/rowset/rowset_meta.h"
26+
#include "olap/tablet_meta.h"
27+
28+
namespace doris {
29+
30+
TEST(TestSizeBasedCumulativeCompactionPolicy, level_size) {
31+
constexpr int MB = 1024 * 1024;
32+
SizeBasedCumulativeCompactionPolicy policy(1024 * MB);
33+
EXPECT_EQ(512 * MB, policy._level_size(1200 * MB));
34+
EXPECT_EQ(0, policy._level_size(1000));
35+
EXPECT_EQ(2 * MB, policy._level_size(2 * MB + 100));
36+
EXPECT_EQ(1 * MB, policy._level_size(2 * MB - 100));
37+
}
38+
39+
static RowsetSharedPtr create_rowset(Version version, int num_segments, bool overlapping,
40+
int data_size) {
41+
auto rs_meta = std::make_shared<RowsetMeta>();
42+
rs_meta->set_rowset_type(BETA_ROWSET); // important
43+
rs_meta->set_start_version(version.first);
44+
rs_meta->set_end_version(version.second);
45+
rs_meta->set_num_segments(num_segments);
46+
rs_meta->set_segments_overlap(overlapping ? OVERLAPPING : NONOVERLAPPING);
47+
rs_meta->set_total_disk_size(data_size);
48+
RowsetSharedPtr rowset;
49+
RowsetFactory::create_rowset(nullptr, "", std::move(rs_meta), &rowset);
50+
return rowset;
51+
}
52+
53+
TEST(TestSizeBasedCumulativeCompactionPolicy, pick_input_rowsets) {
54+
constexpr int MB = 1024 * 1024;
55+
SizeBasedCumulativeCompactionPolicy policy(1024 * MB, 0.05, 64 * MB, 64 * MB);
56+
constexpr int min_compaction_score = 2;
57+
constexpr int max_compaction_score = 100;
58+
Version last_delete_version = {-1, -1};
59+
size_t compaction_score = 0;
60+
{
61+
// [2-2] score=1 size=1, [3-3] score=1 size=1, [4-4] score=1 size=1
62+
Tablet tablet(std::make_shared<TabletMeta>(), nullptr);
63+
std::vector<RowsetSharedPtr> rowsets;
64+
rowsets.push_back(create_rowset({2, 2}, 1, false, MB));
65+
rowsets.push_back(create_rowset({3, 3}, 1, false, MB));
66+
rowsets.push_back(create_rowset({4, 4}, 1, false, MB));
67+
tablet.cloud_add_rowsets(rowsets, false);
68+
std::vector<RowsetSharedPtr> input_rowsets;
69+
policy.pick_input_rowsets(&tablet, rowsets, max_compaction_score, min_compaction_score,
70+
&input_rowsets, &last_delete_version, &compaction_score);
71+
ASSERT_EQ(input_rowsets.size(), 3);
72+
EXPECT_EQ(input_rowsets[0]->end_version(), 2);
73+
EXPECT_EQ(input_rowsets[1]->end_version(), 3);
74+
EXPECT_EQ(input_rowsets[2]->end_version(), 4);
75+
EXPECT_EQ(compaction_score, 3);
76+
}
77+
{
78+
// [2-2] score=1 size=4, [3-3] score=1 size=1, [4-4] score=1 size=1
79+
Tablet tablet(std::make_shared<TabletMeta>(), nullptr);
80+
std::vector<RowsetSharedPtr> rowsets;
81+
rowsets.push_back(create_rowset({2, 2}, 1, false, 4 * MB));
82+
rowsets.push_back(create_rowset({3, 3}, 1, false, MB));
83+
rowsets.push_back(create_rowset({4, 4}, 1, false, MB));
84+
tablet.cloud_add_rowsets(rowsets, false);
85+
std::vector<RowsetSharedPtr> input_rowsets;
86+
policy.pick_input_rowsets(&tablet, rowsets, max_compaction_score, min_compaction_score,
87+
&input_rowsets, &last_delete_version, &compaction_score);
88+
ASSERT_EQ(input_rowsets.size(), 2);
89+
EXPECT_EQ(input_rowsets[0]->end_version(), 3);
90+
EXPECT_EQ(input_rowsets[1]->end_version(), 4);
91+
EXPECT_EQ(compaction_score, 2);
92+
}
93+
{
94+
// [2-2] score=1 size=4, [3-3] score=1 size=2, [4-4] score=1 size=1
95+
Tablet tablet(std::make_shared<TabletMeta>(), nullptr);
96+
std::vector<RowsetSharedPtr> rowsets;
97+
rowsets.push_back(create_rowset({2, 2}, 1, false, 4 * MB));
98+
rowsets.push_back(create_rowset({3, 3}, 1, false, 2 * MB));
99+
rowsets.push_back(create_rowset({4, 4}, 1, false, MB));
100+
tablet.cloud_add_rowsets(rowsets, false);
101+
std::vector<RowsetSharedPtr> input_rowsets;
102+
policy.pick_input_rowsets(&tablet, rowsets, max_compaction_score, min_compaction_score,
103+
&input_rowsets, &last_delete_version, &compaction_score);
104+
ASSERT_EQ(input_rowsets.size(), 0);
105+
EXPECT_EQ(compaction_score, 0);
106+
}
107+
{
108+
// [2-2] score=90 size=100, [3-3] score=1 size=1, [4-4] score=1 size=1
109+
Tablet tablet(std::make_shared<TabletMeta>(), nullptr);
110+
std::vector<RowsetSharedPtr> rowsets;
111+
rowsets.push_back(create_rowset({2, 2}, 90, true, 100 * MB));
112+
rowsets.push_back(create_rowset({3, 3}, 1, false, MB));
113+
rowsets.push_back(create_rowset({4, 4}, 1, false, MB));
114+
tablet.cloud_add_rowsets(rowsets, false);
115+
std::vector<RowsetSharedPtr> input_rowsets;
116+
policy.pick_input_rowsets(&tablet, rowsets, max_compaction_score, min_compaction_score,
117+
&input_rowsets, &last_delete_version, &compaction_score);
118+
ASSERT_EQ(input_rowsets.size(), 2);
119+
EXPECT_EQ(input_rowsets[0]->end_version(), 3);
120+
EXPECT_EQ(input_rowsets[1]->end_version(), 4);
121+
EXPECT_EQ(compaction_score, 2);
122+
}
123+
{
124+
// [2-2] score=100 size=100, [3-3] score=1 size=1, [4-4] score=1 size=1
125+
Tablet tablet(std::make_shared<TabletMeta>(), nullptr);
126+
std::vector<RowsetSharedPtr> rowsets;
127+
rowsets.push_back(create_rowset({2, 2}, 100, true, 100 * MB));
128+
rowsets.push_back(create_rowset({3, 3}, 1, false, MB));
129+
rowsets.push_back(create_rowset({4, 4}, 1, false, MB));
130+
tablet.cloud_add_rowsets(rowsets, false);
131+
std::vector<RowsetSharedPtr> input_rowsets;
132+
policy.pick_input_rowsets(&tablet, rowsets, max_compaction_score, min_compaction_score,
133+
&input_rowsets, &last_delete_version, &compaction_score);
134+
ASSERT_EQ(input_rowsets.size(), 1);
135+
EXPECT_EQ(input_rowsets[0]->end_version(), 2);
136+
EXPECT_EQ(compaction_score, 100);
137+
}
138+
{
139+
// [2-2] score=50 size=100, [3-3] score=60 size=120, [4-4] score=1 size=1
140+
Tablet tablet(std::make_shared<TabletMeta>(), nullptr);
141+
std::vector<RowsetSharedPtr> rowsets;
142+
rowsets.push_back(create_rowset({2, 2}, 50, true, 100 * MB));
143+
rowsets.push_back(create_rowset({3, 3}, 60, true, 120 * MB));
144+
rowsets.push_back(create_rowset({4, 4}, 1, false, MB));
145+
tablet.cloud_add_rowsets(rowsets, false);
146+
std::vector<RowsetSharedPtr> input_rowsets;
147+
policy.pick_input_rowsets(&tablet, rowsets, max_compaction_score, min_compaction_score,
148+
&input_rowsets, &last_delete_version, &compaction_score);
149+
ASSERT_EQ(input_rowsets.size(), 2);
150+
EXPECT_EQ(input_rowsets[0]->end_version(), 2);
151+
EXPECT_EQ(input_rowsets[1]->end_version(), 3);
152+
EXPECT_EQ(compaction_score, 110);
153+
}
154+
{
155+
// [2-2] score=1 size=200, [3-3] score=100 size=100, [4-4] score=1 size=1
156+
Tablet tablet(std::make_shared<TabletMeta>(), nullptr);
157+
std::vector<RowsetSharedPtr> rowsets;
158+
rowsets.push_back(create_rowset({2, 2}, 10, false, 200 * MB));
159+
rowsets.push_back(create_rowset({3, 3}, 100, true, 100 * MB));
160+
rowsets.push_back(create_rowset({4, 4}, 1, false, MB));
161+
tablet.cloud_add_rowsets(rowsets, false);
162+
std::vector<RowsetSharedPtr> input_rowsets;
163+
policy.pick_input_rowsets(&tablet, rowsets, max_compaction_score, min_compaction_score,
164+
&input_rowsets, &last_delete_version, &compaction_score);
165+
ASSERT_EQ(input_rowsets.size(), 1);
166+
EXPECT_EQ(input_rowsets[0]->end_version(), 3);
167+
EXPECT_EQ(compaction_score, 100);
168+
}
169+
}
170+
171+
} // namespace doris
172+
173+
// @brief Test Stub

0 commit comments

Comments
 (0)