Skip to content

Commit d34fbed

Browse files
committed
[feature](cloud) support event driven or periodic warm up (apache#4101)
1 parent c31394b commit d34fbed

55 files changed

Lines changed: 5103 additions & 270 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/cloud/cloud_backend_service.cpp‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@
3636

3737
namespace doris {
3838

39+
bvar::Adder<uint64_t> g_file_cache_warm_up_cache_async_submitted_segment_num(
40+
"file_cache_warm_up_cache_async_submitted_segment_num");
41+
3942
CloudBackendService::CloudBackendService(CloudStorageEngine& engine, ExecEnv* exec_env)
4043
: BaseBackendService(exec_env), _engine(engine) {}
4144

@@ -95,8 +98,15 @@ void CloudBackendService::warm_up_tablets(TWarmUpTabletsResponse& response,
9598
LOG_INFO("receive the warm up request.")
9699
.tag("request_type", "SET_JOB")
97100
.tag("job_id", request.job_id);
98-
st = manager.check_and_set_job_id(request.job_id);
99-
if (!st) {
101+
if (request.__isset.event) {
102+
st = manager.set_event(request.job_id, request.event);
103+
if (st.ok()) {
104+
break;
105+
}
106+
} else {
107+
st = manager.check_and_set_job_id(request.job_id);
108+
}
109+
if (!st.ok()) {
100110
LOG_WARNING("SET_JOB failed.").error(st);
101111
break;
102112
}
@@ -142,7 +152,11 @@ void CloudBackendService::warm_up_tablets(TWarmUpTabletsResponse& response,
142152
LOG_INFO("receive the warm up request.")
143153
.tag("request_type", "CLEAR_JOB")
144154
.tag("job_id", request.job_id);
145-
st = manager.clear_job(request.job_id);
155+
if (request.__isset.event) {
156+
st = manager.set_event(request.job_id, request.event, /* clear: */ true);
157+
} else {
158+
st = manager.clear_job(request.job_id);
159+
}
146160
break;
147161
}
148162
default:
@@ -181,6 +195,8 @@ void CloudBackendService::warm_up_cache_async(TWarmUpCacheAsyncResponse& respons
181195
PGetFileCacheMetaResponse brpc_response;
182196
brpc_stub->get_file_cache_meta_by_tablet_id(&cntl, &brpc_request, &brpc_response, nullptr);
183197
if (!cntl.Failed()) {
198+
g_file_cache_warm_up_cache_async_submitted_segment_num
199+
<< brpc_response.file_cache_block_metas().size();
184200
_engine.file_cache_block_downloader().submit_download_task(
185201
std::move(*brpc_response.mutable_file_cache_block_metas()));
186202
} else {

‎be/src/cloud/cloud_internal_service.cpp‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "cloud/cloud_storage_engine.h"
2121
#include "cloud/cloud_tablet_mgr.h"
2222
#include "io/cache/block_file_cache.h"
23+
#include "io/cache/block_file_cache_downloader.h"
2324
#include "io/cache/block_file_cache_factory.h"
2425

2526
namespace doris {
@@ -105,4 +106,187 @@ void CloudInternalServiceImpl::get_file_cache_meta_by_tablet_id(
105106
}
106107
}
107108

109+
bvar::Adder<uint64_t> g_file_cache_event_driven_warm_up_submitted_segment_num(
110+
"file_cache_event_driven_warm_up_submitted_segment_num");
111+
bvar::Adder<uint64_t> g_file_cache_event_driven_warm_up_finished_segment_num(
112+
"file_cache_event_driven_warm_up_finished_segment_num");
113+
bvar::Adder<uint64_t> g_file_cache_event_driven_warm_up_failed_segment_num(
114+
"file_cache_event_driven_warm_up_failed_segment_num");
115+
bvar::Adder<uint64_t> g_file_cache_event_driven_warm_up_submitted_segment_size(
116+
"file_cache_event_driven_warm_up_submitted_segment_size");
117+
bvar::Adder<uint64_t> g_file_cache_event_driven_warm_up_finished_segment_size(
118+
"file_cache_event_driven_warm_up_finished_segment_size");
119+
bvar::Adder<uint64_t> g_file_cache_event_driven_warm_up_failed_segment_size(
120+
"file_cache_event_driven_warm_up_failed_segment_size");
121+
bvar::Adder<uint64_t> g_file_cache_event_driven_warm_up_submitted_index_num(
122+
"file_cache_event_driven_warm_up_submitted_index_num");
123+
bvar::Adder<uint64_t> g_file_cache_event_driven_warm_up_finished_index_num(
124+
"file_cache_event_driven_warm_up_finished_index_num");
125+
bvar::Adder<uint64_t> g_file_cache_event_driven_warm_up_failed_index_num(
126+
"file_cache_event_driven_warm_up_failed_index_num");
127+
bvar::Adder<uint64_t> g_file_cache_event_driven_warm_up_submitted_index_size(
128+
"file_cache_event_driven_warm_up_submitted_index_size");
129+
bvar::Adder<uint64_t> g_file_cache_event_driven_warm_up_finished_index_size(
130+
"file_cache_event_driven_warm_up_finished_index_size");
131+
bvar::Adder<uint64_t> g_file_cache_event_driven_warm_up_failed_index_size(
132+
"file_cache_event_driven_warm_up_failed_index_size");
133+
134+
void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* controller
135+
[[maybe_unused]],
136+
const PWarmUpRowsetRequest* request,
137+
PWarmUpRowsetResponse* response,
138+
google::protobuf::Closure* done) {
139+
brpc::ClosureGuard closure_guard(done);
140+
for (auto& rs_meta_pb : request->rowset_metas()) {
141+
RowsetMeta rs_meta;
142+
rs_meta.init_from_pb(rs_meta_pb);
143+
auto storage_resource = rs_meta.remote_storage_resource();
144+
if (!storage_resource) {
145+
LOG(WARNING) << storage_resource.error();
146+
continue;
147+
}
148+
int64_t tablet_id = rs_meta.tablet_id();
149+
auto res = _engine.tablet_mgr().get_tablet(tablet_id);
150+
if (!res.has_value()) {
151+
LOG_WARNING("Warm up error ").tag("tablet_id", tablet_id).error(res.error());
152+
continue;
153+
}
154+
auto tablet = res.value();
155+
auto tablet_meta = tablet->tablet_meta();
156+
157+
int64_t expiration_time =
158+
tablet_meta->ttl_seconds() == 0 || rs_meta.newest_write_timestamp() <= 0
159+
? 0
160+
: rs_meta.newest_write_timestamp() + tablet_meta->ttl_seconds();
161+
if (expiration_time <= UnixSeconds()) {
162+
expiration_time = 0;
163+
}
164+
165+
for (int64_t segment_id = 0; segment_id < rs_meta.num_segments(); segment_id++) {
166+
auto download_done = [&, tablet_id, rowset_id = rs_meta.rowset_id().to_string(),
167+
segment_size = rs_meta.segment_file_size(segment_id)](Status st) {
168+
if (st.ok()) {
169+
g_file_cache_event_driven_warm_up_finished_segment_num << 1;
170+
g_file_cache_event_driven_warm_up_finished_segment_size << segment_size;
171+
} else {
172+
g_file_cache_event_driven_warm_up_failed_segment_num << 1;
173+
g_file_cache_event_driven_warm_up_failed_segment_size << segment_size;
174+
LOG(WARNING) << "download segment failed, tablet_id: " << tablet_id
175+
<< " rowset_id: " << rowset_id << ", error: " << st;
176+
}
177+
};
178+
179+
io::DownloadFileMeta download_meta {
180+
.path = storage_resource.value()->remote_segment_path(rs_meta, segment_id),
181+
.file_size = rs_meta.segment_file_size(segment_id),
182+
.offset = 0,
183+
.download_size = rs_meta.segment_file_size(segment_id),
184+
.file_system = storage_resource.value()->fs,
185+
.ctx =
186+
{
187+
.is_index_data = false,
188+
.expiration_time = expiration_time,
189+
.is_dryrun =
190+
config::enable_reader_dryrun_when_download_file_cache,
191+
},
192+
.download_done = std::move(download_done),
193+
};
194+
g_file_cache_event_driven_warm_up_submitted_segment_num << 1;
195+
g_file_cache_event_driven_warm_up_submitted_segment_size
196+
<< rs_meta.segment_file_size(segment_id);
197+
_engine.file_cache_block_downloader().submit_download_task(download_meta);
198+
199+
auto download_inverted_index = [&](std::string index_path, uint64_t idx_size) {
200+
auto storage_resource = rs_meta.remote_storage_resource();
201+
auto download_done = [=, tablet_id = rs_meta.tablet_id(),
202+
rowset_id = rs_meta.rowset_id().to_string()](Status st) {
203+
if (st.ok()) {
204+
g_file_cache_event_driven_warm_up_finished_index_num << 1;
205+
g_file_cache_event_driven_warm_up_finished_index_size << idx_size;
206+
} else {
207+
g_file_cache_event_driven_warm_up_failed_index_num << 1;
208+
g_file_cache_event_driven_warm_up_failed_index_size << idx_size;
209+
LOG(WARNING) << "download inverted index failed, tablet_id: " << tablet_id
210+
<< " rowset_id: " << rowset_id << ", error: " << st;
211+
}
212+
};
213+
io::DownloadFileMeta download_meta {
214+
.path = io::Path(index_path),
215+
.file_size = static_cast<int64_t>(idx_size),
216+
.file_system = storage_resource.value()->fs,
217+
.ctx =
218+
{
219+
.is_index_data = true,
220+
.expiration_time = expiration_time,
221+
.is_dryrun = config::
222+
enable_reader_dryrun_when_download_file_cache,
223+
},
224+
.download_done = std::move(download_done),
225+
};
226+
g_file_cache_event_driven_warm_up_submitted_index_num << 1;
227+
g_file_cache_event_driven_warm_up_submitted_index_size << idx_size;
228+
_engine.file_cache_block_downloader().submit_download_task(download_meta);
229+
};
230+
231+
// inverted index
232+
auto schema_ptr = rs_meta.tablet_schema();
233+
auto idx_version = schema_ptr->get_inverted_index_storage_format();
234+
bool has_inverted_index = schema_ptr->has_inverted_index();
235+
236+
if (has_inverted_index) {
237+
if (idx_version == InvertedIndexStorageFormatPB::V1) {
238+
auto&& inverted_index_info = rs_meta.inverted_index_file_info(segment_id);
239+
std::unordered_map<int64_t, int64_t> index_size_map;
240+
for (const auto& info : inverted_index_info.index_info()) {
241+
if (info.index_file_size() != -1) {
242+
index_size_map[info.index_id()] = info.index_file_size();
243+
} else {
244+
VLOG_DEBUG << "Invalid index_file_size for segment_id " << segment_id
245+
<< ", index_id " << info.index_id();
246+
}
247+
}
248+
for (const auto& index : schema_ptr->inverted_indexes()) {
249+
auto idx_path = storage_resource.value()->remote_idx_v1_path(
250+
rs_meta, segment_id, index->index_id(), index->get_index_suffix());
251+
download_inverted_index(idx_path, index_size_map[index->index_id()]);
252+
}
253+
} else { // InvertedIndexStorageFormatPB::V2
254+
auto&& inverted_index_info = rs_meta.inverted_index_file_info(segment_id);
255+
int64_t idx_size = 0;
256+
if (inverted_index_info.has_index_size()) {
257+
idx_size = inverted_index_info.index_size();
258+
} else {
259+
VLOG_DEBUG << "index_size is not set for segment " << segment_id;
260+
}
261+
auto idx_path =
262+
storage_resource.value()->remote_idx_v2_path(rs_meta, segment_id);
263+
download_inverted_index(idx_path, idx_size);
264+
}
265+
}
266+
}
267+
}
268+
}
269+
270+
bvar::Adder<uint64_t> g_file_cache_recycle_cache_finished_segment_num(
271+
"file_cache_recycle_cache_finished_segment_num");
272+
273+
void CloudInternalServiceImpl::recycle_cache(google::protobuf::RpcController* controller
274+
[[maybe_unused]],
275+
const PRecycleCacheRequest* request,
276+
PRecycleCacheResponse* response,
277+
google::protobuf::Closure* done) {
278+
brpc::ClosureGuard closure_guard(done);
279+
280+
if (config::enable_file_cache) {
281+
for (const auto& meta : request->cache_metas()) {
282+
for (int64_t segment_id = 0; segment_id < meta.num_segments(); segment_id++) {
283+
auto file_key = Segment::file_cache_key(meta.rowset_id(), segment_id);
284+
auto* file_cache = io::FileCacheFactory::instance()->get_by_path(file_key);
285+
file_cache->remove_if_cached_async(file_key);
286+
g_file_cache_recycle_cache_finished_segment_num << 1;
287+
}
288+
}
289+
}
290+
}
291+
108292
} // namespace doris

‎be/src/cloud/cloud_internal_service.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,14 @@ class CloudInternalServiceImpl final : public PInternalService {
4040
PGetFileCacheMetaResponse* response,
4141
google::protobuf::Closure* done) override;
4242

43+
void warm_up_rowset(google::protobuf::RpcController* controller,
44+
const PWarmUpRowsetRequest* request, PWarmUpRowsetResponse* response,
45+
google::protobuf::Closure* done) override;
46+
47+
void recycle_cache(google::protobuf::RpcController* controller,
48+
const PRecycleCacheRequest* request, PRecycleCacheResponse* response,
49+
google::protobuf::Closure* done) override;
50+
4351
private:
4452
CloudStorageEngine& _engine;
4553
};

‎be/src/cloud/cloud_meta_mgr.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939

4040
#include "cloud/cloud_storage_engine.h"
4141
#include "cloud/cloud_tablet.h"
42+
#include "cloud/cloud_warm_up_manager.h"
4243
#include "cloud/config.h"
4344
#include "cloud/pb_convert.h"
4445
#include "cloud/schema_cloud_dictionary_cache.h"
@@ -1009,7 +1010,7 @@ Status CloudMetaMgr::prepare_rowset(const RowsetMeta& rs_meta, const std::string
10091010
return st;
10101011
}
10111012

1012-
Status CloudMetaMgr::commit_rowset(const RowsetMeta& rs_meta, const std::string& job_id,
1013+
Status CloudMetaMgr::commit_rowset(RowsetMeta& rs_meta, const std::string& job_id,
10131014
RowsetMetaSharedPtr* existed_rs_meta) {
10141015
VLOG_DEBUG << "commit rowset, tablet_id: " << rs_meta.tablet_id()
10151016
<< ", rowset_id: " << rs_meta.rowset_id() << " txn_id: " << rs_meta.txn_id();
@@ -1051,6 +1052,8 @@ Status CloudMetaMgr::commit_rowset(const RowsetMeta& rs_meta, const std::string&
10511052
RETURN_IF_ERROR(
10521053
engine.get_schema_cloud_dictionary_cache().refresh_dict(rs_meta_pb.index_id()));
10531054
}
1055+
auto& manager = ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager();
1056+
manager.warm_up_rowset(rs_meta);
10541057
return st;
10551058
}
10561059

‎be/src/cloud/cloud_meta_mgr.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ class CloudMetaMgr {
7272
Status prepare_rowset(const RowsetMeta& rs_meta, const std::string& job_id,
7373
std::shared_ptr<RowsetMeta>* existed_rs_meta = nullptr);
7474

75-
Status commit_rowset(const RowsetMeta& rs_meta, const std::string& job_id,
75+
Status commit_rowset(RowsetMeta& rs_meta, const std::string& job_id,
7676
std::shared_ptr<RowsetMeta>* existed_rs_meta = nullptr);
7777

7878
Status update_tmp_rowset(const RowsetMeta& rs_meta);

0 commit comments

Comments
 (0)