Skip to content

Commit d5c427f

Browse files
committed
[refactor](be) Keep read-ahead operations with their owning objects
### What problem does this PR solve? Issue Number: apache#67611 (tracking) Related PR: apache#67579, apache#67292, apache#67612 Problem Summary: Plan reset and timing aggregation, asynchronous read result construction and terminal-state checks, and partial-block request validation were implemented outside the objects whose state they describe. Move these operations to ColumnReadAheadPlan, FileRangeReadSubmitResult, FileRangeRead, and PartialBlockWritebackRequest. Keep partial-block buffer creation and remote reads on Task, while the manager retains queue admission, worker scheduling, and lifecycle accounting. Preserve vector capacity during plan reset, per-call statistics, lock boundaries, and the join of accepted remote reads before task cleanup. ### Release note None ### Check List (For Author) - Test: Unit Test — 198 tests from 11 suites passed through run-be-ut.sh with ASAN and -j100. Added plan-reset capacity/ownership, empty-plan timing aggregation, and rejected-result coverage. Existing read-ahead, scheduler, cache-writeback and hole-fill tests passed, including concurrent reads, failed submission/read cleanup, and shutdown. clang-format v16, check-format, build hygiene, and git diff --check passed. No cluster tests or clang-tidy were run. - Behavior changed: No — move object-local operations without changing read-ahead, IO, or writeback mechanisms. - Does this need documentation: No — internal responsibility cleanup; configuration and metric meanings are unchanged.
1 parent 503e43a commit d5c427f

10 files changed

Lines changed: 162 additions & 78 deletions

‎be/src/io/cache/partial_block_writeback_manager.cpp‎

Lines changed: 43 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ size_t PartialBlockWritebackManager::BlockKeyHash::operator()(const BlockKey& ke
5252
}
5353

5454
struct PartialBlockWritebackManager::Task {
55+
// Allocate one tracked block buffer and copy the first fragment into it.
56+
static TaskPtr create(PartialBlockWritebackRequest request, const BlockKey& task_key,
57+
size_t block_size);
58+
5559
// A queued task may merge while the manager lock is free. Once activate() wins, later
5660
// fragments are deduplicated and the worker obtains their bytes from the source read instead.
5761
std::optional<PartialBlockSubmitResult> try_merge(size_t fragment_offset, Slice data) {
@@ -80,6 +84,10 @@ struct PartialBlockWritebackManager::Task {
8084
return HoleFillPlanner::plan(block, covered_intervals, options, read_ranges);
8185
}
8286

87+
// Read a single range on the block worker; submit multiple ranges through its reusable token
88+
// and join them, including on failure.
89+
Status read_holes(const std::vector<FileRange>& read_ranges, ThreadPoolToken& token);
90+
8391
void activate() {
8492
bool expected = false;
8593
DORIS_CHECK(active.compare_exchange_strong(expected, true, std::memory_order_acq_rel));
@@ -114,6 +122,10 @@ struct PartialBlockWritebackManager::Task {
114122
// A queued task may absorb foreground fragments while unrelated queue operations proceed.
115123
std::mutex fragment_mutex;
116124
std::atomic<bool> active {false};
125+
126+
private:
127+
// Read one buffer slice with per-call IO statistics on a block worker or remote-read thread.
128+
Status _read_hole(const FileRange& range);
117129
};
118130

119131
class PartialBlockWritebackManager::Worker : public std::enable_shared_from_this<Worker> {
@@ -232,7 +244,7 @@ Status PartialBlockWritebackManager::_start() {
232244

233245
PartialBlockSubmitResult PartialBlockWritebackManager::try_submit(
234246
PartialBlockWritebackRequest request) {
235-
_validate_request(request);
247+
request.sanity_check(_options.block_size);
236248

237249
if (!request.write_manager->accepting()) {
238250
return PartialBlockSubmitResult::REJECTED;
@@ -272,7 +284,7 @@ PartialBlockSubmitResult PartialBlockWritebackManager::try_submit(
272284
}
273285
}
274286

275-
auto candidate = _create_task(std::move(request), key);
287+
auto candidate = Task::create(std::move(request), key, _options.block_size);
276288
if (candidate == nullptr) {
277289
return PartialBlockSubmitResult::BUFFER_ALLOCATION_FAILED;
278290
}
@@ -294,32 +306,31 @@ PartialBlockSubmitResult PartialBlockWritebackManager::try_submit(
294306
}
295307
}
296308

297-
void PartialBlockWritebackManager::_validate_request(
298-
const PartialBlockWritebackRequest& request) const {
299-
DORIS_CHECK(request.write_manager != nullptr);
300-
DORIS_CHECK(request.source_reader != nullptr);
301-
DORIS_CHECK(request.write_epoch.key_token != nullptr);
302-
DORIS_CHECK(request.block_offset % _options.block_size == 0);
303-
DORIS_CHECK(request.block_valid_size > 0);
304-
DORIS_CHECK(request.block_valid_size <= _options.block_size);
305-
DORIS_CHECK(request.block_offset <= request.source_reader->size());
306-
DORIS_CHECK(request.block_valid_size <= request.source_reader->size() - request.block_offset);
307-
DORIS_CHECK(request.data.data != nullptr);
308-
DORIS_CHECK(request.data.size > 0);
309-
DORIS_CHECK(request.fragment_offset < request.block_valid_size);
310-
DORIS_CHECK(request.data.size <= request.block_valid_size - request.fragment_offset);
309+
void PartialBlockWritebackRequest::sanity_check(size_t block_size) const {
310+
DORIS_CHECK(write_manager != nullptr);
311+
DORIS_CHECK(source_reader != nullptr);
312+
DORIS_CHECK(write_epoch.key_token != nullptr);
313+
DORIS_CHECK(block_offset % block_size == 0);
314+
DORIS_CHECK(block_valid_size > 0);
315+
DORIS_CHECK(block_valid_size <= block_size);
316+
DORIS_CHECK(block_offset <= source_reader->size());
317+
DORIS_CHECK(block_valid_size <= source_reader->size() - block_offset);
318+
DORIS_CHECK(data.data != nullptr);
319+
DORIS_CHECK(data.size > 0);
320+
DORIS_CHECK(fragment_offset < block_valid_size);
321+
DORIS_CHECK(data.size <= block_valid_size - fragment_offset);
311322
}
312323

313-
PartialBlockWritebackManager::TaskPtr PartialBlockWritebackManager::_create_task(
314-
PartialBlockWritebackRequest request, const BlockKey& key) {
324+
PartialBlockWritebackManager::TaskPtr PartialBlockWritebackManager::Task::create(
325+
PartialBlockWritebackRequest request, const BlockKey& task_key, size_t block_size) {
315326
AsyncCacheWriteBufferPtr buffer;
316-
if (!request.write_manager->allocate_tracked_buffer(_options.block_size, &buffer).ok()) {
327+
if (!request.write_manager->allocate_tracked_buffer(block_size, &buffer).ok()) {
317328
return nullptr;
318329
}
319330
std::memcpy(buffer->data() + request.fragment_offset, request.data.data, request.data.size);
320331

321332
auto task = std::make_shared<Task>();
322-
task->key = key;
333+
task->key = task_key;
323334
task->inflight_index = request.inflight_index;
324335
task->source_reader = std::move(request.source_reader);
325336
task->block_valid_size = request.block_valid_size;
@@ -644,7 +655,7 @@ void PartialBlockWritebackManager::_process_task(const TaskPtr& task, ThreadPool
644655
return;
645656
}
646657

647-
status = _read_holes(task, read_ranges, token);
658+
status = task->read_holes(read_ranges, token);
648659
if (!status.ok()) {
649660
read_ahead_bvars().hole_fill_failed_blocks << 1;
650661
LOG(WARNING) << "Read partial block holes failed, hash=" << task->key.cache_hash.to_string()
@@ -669,19 +680,18 @@ void PartialBlockWritebackManager::_process_task(const TaskPtr& task, ThreadPool
669680
}
670681
}
671682

672-
Status PartialBlockWritebackManager::_read_holes(const TaskPtr& task,
673-
const std::vector<FileRange>& read_ranges,
674-
ThreadPoolToken& token) {
683+
Status PartialBlockWritebackManager::Task::read_holes(const std::vector<FileRange>& read_ranges,
684+
ThreadPoolToken& token) {
675685
if (read_ranges.empty()) {
676686
return Status::OK();
677687
}
678688
if (read_ranges.size() == 1) {
679-
return _read_hole(task, read_ranges.front());
689+
return _read_hole(read_ranges.front());
680690
}
681691
std::vector<Status> results(read_ranges.size());
682692
for (size_t index = 0; index < read_ranges.size(); ++index) {
683693
Status status = token.submit_func(
684-
[&, index]() { results[index] = _read_hole(task, read_ranges[index]); });
694+
[&, index]() { results[index] = _read_hole(read_ranges[index]); });
685695
if (!status.ok()) {
686696
results[index] = std::move(status);
687697
break;
@@ -695,22 +705,21 @@ Status PartialBlockWritebackManager::_read_holes(const TaskPtr& task,
695705
return Status::OK();
696706
}
697707

698-
Status PartialBlockWritebackManager::_read_hole(const TaskPtr& task, const FileRange& range) {
708+
Status PartialBlockWritebackManager::Task::_read_hole(const FileRange& range) {
699709
FileCacheStatistics file_cache_stats;
700710
FileReaderStats file_reader_stats;
701-
auto io_context = task->io_context;
702-
io_context.io_context.file_cache_stats = &file_cache_stats;
703-
io_context.io_context.file_reader_stats = &file_reader_stats;
711+
auto read_context = io_context;
712+
read_context.io_context.file_cache_stats = &file_cache_stats;
713+
read_context.io_context.file_reader_stats = &file_reader_stats;
704714
size_t bytes_read = 0;
705715
int64_t read_ns = 0;
706716
Status status;
707717
read_ahead_bvars().hole_fill_remote_requests << 1;
708718
{
709719
SCOPED_RAW_TIMER(&read_ns);
710-
status = task->source_reader->read_at(
711-
range.offset,
712-
Slice(task->buffer->data() + range.offset - task->key.block_offset, range.size),
713-
&bytes_read, &io_context.io_context);
720+
status = source_reader->read_at(
721+
range.offset, Slice(buffer->data() + range.offset - key.block_offset, range.size),
722+
&bytes_read, &read_context.io_context);
714723
}
715724
read_ahead_bvars().hole_fill_remote_read_time_ns << read_ns;
716725
read_ahead_bvars().hole_fill_remote_bytes << static_cast<int64_t>(bytes_read);

‎be/src/io/cache/partial_block_writeback_manager.h‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,9 @@ struct PartialBlockWritebackRequest {
9696
CacheAdmissionContext admission_ctx;
9797
AsyncCacheWriteEpoch write_epoch;
9898
FileRangeReadIOContext io_context;
99+
100+
/// Assert request invariants using the block size from validated manager options.
101+
void sanity_check(size_t block_size) const;
99102
};
100103

101104
/// BE-level bounded queue, block workers, and a shared remote-read pool for completing partial
@@ -180,9 +183,6 @@ class PartialBlockWritebackManager {
180183
Status _resize_workers_locked(size_t worker_count);
181184
/// Stop and join workers in `[keep_worker_count, _workers.size())` under the lifecycle lock.
182185
void _stop_workers_locked(size_t keep_worker_count);
183-
void _validate_request(const PartialBlockWritebackRequest& request) const;
184-
/// Allocate one tracked block buffer and copy the first fragment into it.
185-
TaskPtr _create_task(PartialBlockWritebackRequest request, const BlockKey& key);
186186
/// Atomically install `candidate`, replace a stale queued task, or return a current same-key
187187
/// task. Destruction of a displaced task occurs after the manager lock is released.
188188
EnqueueResult _enqueue_or_get_existing(const TaskPtr& candidate, TaskPtr* existing);
@@ -198,12 +198,6 @@ class PartialBlockWritebackManager {
198198
/// Plan holes, wait for parallel source reads, and hand the completed buffer to the cache
199199
/// writer. Every failure drops this best-effort task after its reads finish.
200200
void _process_task(const TaskPtr& task, ThreadPoolToken& token);
201-
/// Read a single range on the block worker; submit multiple ranges through its reusable token
202-
/// and join them, including on failure.
203-
Status _read_holes(const TaskPtr& task, const std::vector<FileRange>& read_ranges,
204-
ThreadPoolToken& token);
205-
/// Read one buffer slice with per-call IO statistics on a block worker or remote-read thread.
206-
Status _read_hole(const TaskPtr& task, const FileRange& range);
207201
/// Remove an active task and release its per-cache-writer slot accounting.
208202
void _complete_task(const TaskPtr& task);
209203

‎be/src/io/fs/file_range_read_scheduler.cpp‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -42,21 +42,17 @@ bool add_overflows(size_t left, size_t right) {
4242
return right > std::numeric_limits<size_t>::max() - left;
4343
}
4444

45-
bool is_terminal(FileRangeRead::State state) {
46-
return state == FileRangeRead::State::READY || state == FileRangeRead::State::FAILED ||
47-
state == FileRangeRead::State::CANCELLED;
48-
}
45+
} // namespace
4946

50-
FileRangeReadSubmitResult rejected(FileRangeReadRejectReason reason, Status status = Status::OK()) {
47+
FileRangeReadSubmitResult FileRangeReadSubmitResult::rejected(FileRangeReadRejectReason reason,
48+
Status status) {
5149
return FileRangeReadSubmitResult {
5250
.reads = {},
5351
.reject_reason = reason,
5452
.status = std::move(status),
5553
};
5654
}
5755

58-
} // namespace
59-
6056
FileRangeReadBudget::FileRangeReadBudget(size_t max_bytes) : _max_bytes(max_bytes) {
6157
DORIS_CHECK(_max_bytes > 0);
6258
}
@@ -203,20 +199,24 @@ Status FileRangeRead::create(FileRange range, std::shared_ptr<MemTrackerLimiter>
203199
return Status::OK();
204200
}
205201

202+
bool FileRangeRead::_is_terminal(State state) {
203+
return state == State::READY || state == State::FAILED || state == State::CANCELLED;
204+
}
205+
206206
Status FileRangeRead::wait(bool* ready_on_entry) {
207207
std::unique_lock lock(_mutex);
208208
if (ready_on_entry != nullptr) {
209209
*ready_on_entry = _state == State::READY;
210210
}
211-
_cv.wait(lock, [this]() { return is_terminal(_state); });
211+
_cv.wait(lock, [this]() { return _is_terminal(_state); });
212212
return _status;
213213
}
214214

215215
void FileRangeRead::request_cancel() {
216216
bool publish_cancelled = false;
217217
{
218218
std::lock_guard lock(_mutex);
219-
if (is_terminal(_state)) {
219+
if (_is_terminal(_state)) {
220220
return;
221221
}
222222
_cancel_requested = true;
@@ -301,7 +301,7 @@ void FileRangeRead::_publish_from_running(State state, Status status, FileRangeR
301301
{
302302
std::lock_guard lock(_mutex);
303303
DORIS_CHECK(_state == State::RUNNING);
304-
DORIS_CHECK(is_terminal(state));
304+
DORIS_CHECK(_is_terminal(state));
305305
if (_cancel_requested) {
306306
state = State::CANCELLED;
307307
status = Status::Cancelled("asynchronous file range read cancelled");
@@ -404,7 +404,7 @@ FileRangeReadSubmitResult FileRangeReadScheduler::try_submit(
404404
COUNTER_UPDATE(&io_context.statistics->be_budget_rejected_batches, 1);
405405
}
406406
}
407-
return rejected(reason, std::move(status));
407+
return FileRangeReadSubmitResult::rejected(reason, std::move(status));
408408
};
409409
size_t total_bytes = 0;
410410
Status validation_status = _validate_request(ranges, reader, context, &total_bytes);

‎be/src/io/fs/file_range_read_scheduler.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ class FileRangeRead {
187187
std::shared_ptr<FileRangeRead>* output,
188188
std::shared_ptr<ReadAheadStatistics> statistics = nullptr);
189189

190+
static bool _is_terminal(State state);
190191
bool _mark_running();
191192
bool _is_cancel_requested() const;
192193
void _publish_ready(FileRangeReadStats stats);
@@ -222,6 +223,8 @@ struct FileRangeReadSubmitResult {
222223
FileRangeReadRejectReason reject_reason {FileRangeReadRejectReason::NONE};
223224
Status status;
224225

226+
static FileRangeReadSubmitResult rejected(FileRangeReadRejectReason reason,
227+
Status status = Status::OK());
225228
bool accepted() const {
226229
return reject_reason == FileRangeReadRejectReason::NONE && status.ok();
227230
}

‎be/src/storage/segment/column_read_ahead.cpp‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,28 @@
2323

2424
#include "common/cast_set.h"
2525
#include "common/logging.h"
26+
#include "io/fs/read_ahead_metrics.h"
2627
#include "runtime/runtime_profile.h"
2728

2829
namespace doris::segment_v2 {
2930

31+
void ColumnReadAheadPlan::reset(ColumnReadAhead* owner) {
32+
column = owner;
33+
new_pages.clear();
34+
released_pages.clear();
35+
window_discard_ns = 0;
36+
current_batch_plan_ns = 0;
37+
window_extend_ns = 0;
38+
}
39+
40+
void ColumnReadAheadPlan::update_statistics(io::ReadAheadStatistics* statistics) const {
41+
if (statistics != nullptr) {
42+
COUNTER_UPDATE(&statistics->window_discard_time, window_discard_ns);
43+
COUNTER_UPDATE(&statistics->current_batch_plan_time, current_batch_plan_ns);
44+
COUNTER_UPDATE(&statistics->window_extend_time, window_extend_ns);
45+
}
46+
}
47+
3048
const ColumnReadAheadOptions& ColumnReadAheadContext::options(ColumnReadAheadRole role) const {
3149
return role == ColumnReadAheadRole::EAGER ? eager_options : lazy_options;
3250
}
@@ -90,7 +108,7 @@ void ColumnReadAhead::plan(const rowid_t* current_rowids, size_t count,
90108
DORIS_CHECK(count > 0);
91109
DORIS_CHECK(output != nullptr);
92110
DCHECK(std::is_sorted(current_rowids, current_rowids + count));
93-
_reset_plan(output);
111+
output->reset(this);
94112
_select_candidate_pages(scan_rowids);
95113
_discard_passed_pages(_page_for_ordinal(current_rowids[_reverse ? count - 1 : 0]).page_index,
96114
output);
@@ -109,15 +127,6 @@ void ColumnReadAhead::plan(const rowid_t* current_rowids, size_t count,
109127
}
110128
}
111129

112-
void ColumnReadAhead::_reset_plan(ColumnReadAheadPlan* output) {
113-
output->column = this;
114-
output->new_pages.clear();
115-
output->released_pages.clear();
116-
output->window_discard_ns = 0;
117-
output->current_batch_plan_ns = 0;
118-
output->window_extend_ns = 0;
119-
}
120-
121130
void ColumnReadAhead::_select_candidate_pages(const roaring::Roaring& scan_rowids) {
122131
_candidate_pages.clear();
123132
for (auto row = scan_rowids.begin(); row != scan_rowids.end();) {
@@ -139,7 +148,7 @@ void ColumnReadAhead::start(const roaring::Roaring& scan_rowids, ColumnReadAhead
139148
void ColumnReadAhead::advance(int32_t page_index, ColumnReadAheadPlan* output) {
140149
DCHECK_GE(page_index, 0);
141150
DCHECK_LT(page_index, _pages.size());
142-
_reset_plan(output);
151+
output->reset(this);
143152
if (!_reverse) {
144153
_discard_passed_pages(page_index, output);
145154
} else if (page_index < _reverse_low_page_index) {

‎be/src/storage/segment/column_read_ahead.h‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@
2828
#include "io/fs/file_range_coalescer.h"
2929
#include "storage/segment/common.h"
3030

31+
namespace doris::io {
32+
struct ReadAheadStatistics;
33+
}
34+
3135
namespace doris::segment_v2 {
3236

3337
struct ColumnReadAheadOptions {
@@ -65,6 +69,10 @@ struct ColumnReadAheadPlan {
6569
int64_t current_batch_plan_ns {0};
6670
int64_t window_extend_ns {0};
6771

72+
/// Reuse this plan for another call, retaining the page vectors' allocated capacity.
73+
void reset(ColumnReadAhead* owner);
74+
/// Accumulate per-call timings even for an empty plan; statistics may be absent.
75+
void update_statistics(io::ReadAheadStatistics* statistics) const;
6876
bool empty() const { return new_pages.empty() && released_pages.empty(); }
6977
};
7078

@@ -143,7 +151,6 @@ class ColumnReadAhead {
143151

144152
const ColumnReadAheadPage& _page_for_ordinal(rowid_t ordinal) const;
145153
void _select_candidate_pages(const roaring::Roaring& scan_rowids);
146-
void _reset_plan(ColumnReadAheadPlan* output);
147154
void _plan_page(int32_t page_index, ColumnReadAheadPlan* output);
148155
/// Add one page only if the window has not already seen it.
149156
void _add_page(const ColumnReadAheadPage& page, ColumnReadAheadPlan* output);

0 commit comments

Comments
 (0)