Skip to content

Commit 06c0770

Browse files
committed
[improvement](be) Add page read-ahead and hole-fill observability
### What problem does this PR solve? Issue Number: apache#67611 (tracking; keep open) Related PR: apache#67292 Problem Summary: Page read-ahead, IO coalescing and partial-block writeback lack query-level attribution and BE-wide capacity and IO signals. Add named RuntimeProfile counters for planning, compressed candidate and cache-hit bytes, submission outcomes, source IO, page consumption and foreground writeback. Distinguish Query/BE budget rejection and executor rejection from failures of accepted reads, and account queued cancellations only for accepted tasks. Add successful-source and BE cumulative IO/consumption bytes so asynchronous tail work remains observable after the final Profile snapshot without delaying query completion. Record synchronous data-page fallback attempts, requested bytes and time at PageIO, including decode retries while excluding Page Cache hits and index/dictionary reads. Add aggregate hole-fill failure/read-time metrics and keep detailed failure causes in logs. Use COUNTER_UPDATE and scoped timers, with statistics owned independently of the reporting profile so outstanding reads can finish safely. Append point-query statistics to the existing sampled diagnostic log. ### Release note Expose PageReadAhead query profile counters and BE-wide metrics for read-ahead IO, data-page fallback, buffer capacity and background cache-block hole filling. ### Check List (For Author) - Test: Unit Test - 131 targeted ASAN tests from 18 suites passed via run-be-ut.sh -j100 with PCH enabled, as recorded in the original accounting commit. - clang-format 16, check-format, build hygiene and git diff --check passed before squashing. - The historical results above predate removal of TopN-specific collection; current branch validation is recorded in apache#67611. - Behavior changed: Yes (observability only; read scheduling, cancellation, fallback and writeback policies are unchanged) - Does this need documentation: Yes (doris-io/phase2_page_prefetch_and_io_coalescing/observability.md)
1 parent 9a6bcd5 commit 06c0770

22 files changed

Lines changed: 1003 additions & 67 deletions

‎be/src/exec/scan/olap_scanner.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
#include "exprs/vexpr.h"
4747
#include "exprs/vexpr_context.h"
4848
#include "io/cache/block_file_cache_profile.h"
49+
#include "io/fs/read_ahead_metrics.h"
4950
#include "io/io_common.h"
5051
#include "runtime/descriptors.h"
5152
#include "runtime/exec_env.h"
@@ -722,6 +723,9 @@ void OlapScanner::update_realtime_counters() {
722723
}
723724
OlapScanLocalState* local_state = static_cast<OlapScanLocalState*>(_local_state);
724725
const OlapReaderStatistics& stats = _tablet_reader->stats();
726+
if (stats.read_ahead_stats != nullptr) {
727+
stats.read_ahead_stats->update_profile(local_state->_segment_profile.get());
728+
}
725729
COUNTER_UPDATE(local_state->_read_compressed_counter, stats.compressed_bytes_read);
726730
COUNTER_UPDATE(local_state->_read_uncompressed_counter, stats.uncompressed_bytes_read);
727731
COUNTER_UPDATE(local_state->_scan_bytes, stats.uncompressed_bytes_read);
@@ -918,6 +922,10 @@ void OlapScanner::_collect_profile_before_close() {
918922
inverted_index_profile.update(local_state->_index_filter_profile.get(),
919923
&stats.inverted_index_stats);
920924

925+
if (stats.read_ahead_stats != nullptr) {
926+
stats.read_ahead_stats->update_profile(local_state->_segment_profile.get());
927+
}
928+
921929
if (has_file_cache_statistics(stats.file_cache_stats)) {
922930
io::FileCacheProfileReporter cache_profile(local_state->_segment_profile.get());
923931
cache_profile.update(&stats.file_cache_stats);

‎be/src/io/cache/file_range_planner.cpp‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include <utility>
2424

2525
#include "common/logging.h"
26+
#include "io/fs/read_ahead_metrics.h"
2627

2728
namespace doris::io {
2829

@@ -219,15 +220,38 @@ FileRangePlan build_final_plan(const std::vector<FileRange>& input_ranges,
219220
} // namespace
220221

221222
Status FileRangePlanner::plan(const std::vector<FileRange>& input_ranges, size_t file_size,
222-
const FileRangePlanOptions& options,
223-
FileRangePlan* const output_plan) {
223+
const FileRangePlanOptions& options, FileRangePlan* const output_plan,
224+
ReadAheadStatistics* statistics) {
224225
DORIS_CHECK(output_plan != nullptr);
225226
RETURN_IF_ERROR(options.validate());
226227
RETURN_IF_ERROR(validate_input_ranges(input_ranges, file_size));
227228

228229
const auto base_ranges = FileRangeCoalescer::coalesce(input_ranges, options.coalesce_options);
229230
auto components = choose_components(input_ranges, base_ranges, file_size, options);
230231
*output_plan = build_final_plan(input_ranges, std::move(components));
232+
if (statistics != nullptr) {
233+
size_t input_bytes = 0;
234+
size_t coalesced_bytes = 0;
235+
size_t planned_bytes = 0;
236+
for (const auto& range : input_ranges) {
237+
input_bytes += range.size;
238+
}
239+
for (const auto& range : base_ranges) {
240+
coalesced_bytes += range.size;
241+
}
242+
for (const auto& range : output_plan->ranges) {
243+
planned_bytes += range.size;
244+
}
245+
COUNTER_UPDATE(&statistics->input_pages, static_cast<int64_t>(input_ranges.size()));
246+
COUNTER_UPDATE(&statistics->input_bytes, static_cast<int64_t>(input_bytes));
247+
COUNTER_UPDATE(&statistics->coalesced_ranges, static_cast<int64_t>(base_ranges.size()));
248+
COUNTER_UPDATE(&statistics->coalesced_bytes, static_cast<int64_t>(coalesced_bytes));
249+
COUNTER_UPDATE(&statistics->block_fill_bytes,
250+
static_cast<int64_t>(planned_bytes - coalesced_bytes));
251+
COUNTER_UPDATE(&statistics->planned_ranges,
252+
static_cast<int64_t>(output_plan->ranges.size()));
253+
COUNTER_UPDATE(&statistics->planned_bytes, static_cast<int64_t>(planned_bytes));
254+
}
231255
return Status::OK();
232256
}
233257

‎be/src/io/cache/file_range_planner.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525

2626
namespace doris::io {
2727

28+
struct ReadAheadStatistics;
29+
2830
struct FileRangePlanOptions {
2931
/// Limits for the initial range coalescing pass.
3032
FileRangeCoalesceOptions coalesce_options;
@@ -63,7 +65,8 @@ class FileRangePlanner {
6365
/// within `max_range_bytes`. One input larger than that limit remains a standalone read. The
6466
/// physical EOF block uses its valid file bytes as the coverage denominator.
6567
static Status plan(const std::vector<FileRange>& input_ranges, size_t file_size,
66-
const FileRangePlanOptions& options, FileRangePlan* output_plan);
68+
const FileRangePlanOptions& options, FileRangePlan* output_plan,
69+
ReadAheadStatistics* statistics = nullptr);
6770
};
6871

6972
} // namespace doris::io

‎be/src/io/cache/partial_block_writeback_manager.cpp‎

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include "cpp/sync_point.h"
2929
#include "io/cache/hole_fill_planner.h"
3030
#include "io/cache/inflight_write_buffer_index.h"
31+
#include "io/fs/read_ahead_metrics.h"
3132
#include "util/countdown_latch.h"
3233
#include "util/defer_op.h"
3334
#include "util/threadpool.h"
@@ -175,6 +176,7 @@ PartialBlockWritebackManager::PartialBlockWritebackManager(PartialBlockWriteback
175176
_max_pending_tasks(_options.max_pending_bytes / _options.block_size),
176177
_configured_worker_count(_options.worker_count) {
177178
DORIS_CHECK(_max_pending_tasks > 0);
179+
read_ahead_bvars().hole_fill_pending_bytes << 0;
178180
}
179181

180182
PartialBlockWritebackManager::~PartialBlockWritebackManager() {
@@ -363,6 +365,11 @@ PartialBlockWritebackManager::EnqueueResult PartialBlockWritebackManager::_enque
363365
DORIS_CHECK(inserted);
364366
static_cast<void>(entry);
365367
}
368+
if (discarded_task != nullptr) {
369+
read_ahead_bvars().hole_fill_dropped_blocks << 1;
370+
} else {
371+
read_ahead_bvars().hole_fill_pending_bytes << static_cast<int64_t>(_options.block_size);
372+
}
366373
}
367374
_queue_cv.notify_one();
368375
return EnqueueResult::QUEUED;
@@ -384,6 +391,9 @@ void PartialBlockWritebackManager::shutdown() {
384391
const size_t erased = _tasks.erase(task->key);
385392
DORIS_CHECK(erased == 1);
386393
}
394+
read_ahead_bvars().hole_fill_pending_bytes
395+
<< -static_cast<int64_t>(queued.size() * _options.block_size);
396+
read_ahead_bvars().hole_fill_dropped_blocks << static_cast<int64_t>(queued.size());
387397
}
388398
queued.clear();
389399
_queue_cv.notify_all();
@@ -530,6 +540,7 @@ PartialBlockWritebackManager::TaskPtr PartialBlockWritebackManager::_take_runnab
530540
task->activate();
531541
_queue.erase(iterator);
532542
++_active_hole_fill_slots_by_writer[task->key.write_manager];
543+
read_ahead_bvars().hole_fill_active_blocks << 1;
533544
return task;
534545
}
535546
++iterator;
@@ -544,17 +555,26 @@ void PartialBlockWritebackManager::_discard_queued_task_locked(Queue::iterator i
544555
const size_t erased = _tasks.erase(task->key);
545556
DORIS_CHECK(erased == 1);
546557
discarded_tasks->splice(discarded_tasks->end(), _queue, iterator);
558+
read_ahead_bvars().hole_fill_pending_bytes << -static_cast<int64_t>(_options.block_size);
559+
read_ahead_bvars().hole_fill_dropped_blocks << 1;
547560
}
548561

549562
void PartialBlockWritebackManager::_process_task(const TaskPtr& task) {
550-
Defer complete {[&]() { _complete_task(task); }};
563+
bool write_submitted = false;
564+
Defer complete {[&]() {
565+
if (!write_submitted) {
566+
read_ahead_bvars().hole_fill_dropped_blocks << 1;
567+
}
568+
_complete_task(task);
569+
}};
551570
if (!task->key.write_manager->check_write_epoch(task->write_epoch)) {
552571
return;
553572
}
554573

555574
std::vector<FileRange> read_ranges;
556575
Status status = task->plan_hole_reads(_options.hole_fill_coalesce, &read_ranges);
557576
if (!status.ok()) {
577+
read_ahead_bvars().hole_fill_failed_blocks << 1;
558578
LOG(WARNING) << "Plan partial block hole-fill reads failed, hash="
559579
<< task->key.cache_hash.to_string() << ", offset=" << task->key.block_offset
560580
<< ", status=" << status;
@@ -568,11 +588,19 @@ void PartialBlockWritebackManager::_process_task(const TaskPtr& task) {
568588
io_context.io_context.file_reader_stats = &file_reader_stats;
569589
for (const auto& range : read_ranges) {
570590
size_t bytes_read = 0;
571-
status = task->source_reader->read_at(
572-
range.offset,
573-
Slice(task->buffer->data() + range.offset - task->key.block_offset, range.size),
574-
&bytes_read, &io_context.io_context);
591+
read_ahead_bvars().hole_fill_remote_requests << 1;
592+
int64_t read_ns = 0;
593+
{
594+
SCOPED_RAW_TIMER(&read_ns);
595+
status = task->source_reader->read_at(
596+
range.offset,
597+
Slice(task->buffer->data() + range.offset - task->key.block_offset, range.size),
598+
&bytes_read, &io_context.io_context);
599+
}
600+
read_ahead_bvars().hole_fill_remote_read_time_ns << read_ns;
601+
read_ahead_bvars().hole_fill_remote_bytes << static_cast<int64_t>(bytes_read);
575602
if (!status.ok() || bytes_read != range.size) {
603+
read_ahead_bvars().hole_fill_failed_blocks << 1;
576604
LOG(WARNING) << "Read partial block hole failed, hash="
577605
<< task->key.cache_hash.to_string()
578606
<< ", block_offset=" << task->key.block_offset
@@ -582,7 +610,7 @@ void PartialBlockWritebackManager::_process_task(const TaskPtr& task) {
582610
}
583611
}
584612

585-
static_cast<void>(
613+
const auto result =
586614
task->key.write_manager->try_submit_owned_block(AsyncCacheWriteOwnedBlockRequest {
587615
.cache_hash = task->key.cache_hash,
588616
.file_offset = task->key.block_offset,
@@ -592,7 +620,11 @@ void PartialBlockWritebackManager::_process_task(const TaskPtr& task) {
592620
.write_epoch = std::move(task->write_epoch),
593621
.admission_mode = AsyncCacheWriteAdmissionMode::REQUIRE_SPARE_CAPACITY,
594622
.inflight_index = task->inflight_index,
595-
}));
623+
});
624+
write_submitted = result == AsyncCacheWriteBlockSubmitResult::SUBMITTED;
625+
if (write_submitted) {
626+
read_ahead_bvars().hole_fill_write_submitted_blocks << 1;
627+
}
596628
}
597629

598630
void PartialBlockWritebackManager::_complete_task(const TaskPtr& task) {
@@ -607,6 +639,8 @@ void PartialBlockWritebackManager::_complete_task(const TaskPtr& task) {
607639
if (--active_entry->second == 0) {
608640
_active_hole_fill_slots_by_writer.erase(active_entry);
609641
}
642+
read_ahead_bvars().hole_fill_pending_bytes << -static_cast<int64_t>(_options.block_size);
643+
read_ahead_bvars().hole_fill_active_blocks << -1;
610644
}
611645
_queue_cv.notify_one();
612646
}

‎be/src/io/cache/range_cache_writeback.cpp‎

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
#include "common/logging.h"
2323
#include "io/cache/partial_block_writeback_manager.h"
24+
#include "io/fs/read_ahead_metrics.h"
2425

2526
namespace doris::io {
2627

@@ -42,24 +43,27 @@ std::optional<AsyncCacheWriteEpoch> RangeCacheWriteback::capture_write_epoch() c
4243
}
4344

4445
RangeWritebackDispatchResult RangeCacheWriteback::submit_consumed_range(
45-
const FileRange& range, Slice data, const AsyncCacheWriteEpoch& write_epoch) {
46+
const FileRange& range, Slice data, const AsyncCacheWriteEpoch& write_epoch,
47+
ReadAheadStatistics* statistics) {
4648
DORIS_CHECK(write_epoch.key_token != nullptr);
49+
SCOPED_TIMER(statistics != nullptr ? &statistics->writeback_time : nullptr);
4750
if (!_options.write_manager->accepting()) {
4851
return {};
4952
}
5053
RangeWritebackDispatcher dispatcher(
5154
_options.file_size, _options.block_size,
5255
[&](const FileCacheBlockFragment& fragment) {
53-
return _submit_complete_block(fragment, write_epoch);
56+
return _submit_complete_block(fragment, write_epoch, statistics);
5457
},
5558
[&](const FileCacheBlockFragment& fragment) {
56-
return _submit_partial_block(fragment, write_epoch);
59+
return _submit_partial_block(fragment, write_epoch, statistics);
5760
});
5861
return dispatcher.dispatch(range, data);
5962
}
6063

6164
bool RangeCacheWriteback::_submit_complete_block(const FileCacheBlockFragment& fragment,
62-
const AsyncCacheWriteEpoch& write_epoch) {
65+
const AsyncCacheWriteEpoch& write_epoch,
66+
ReadAheadStatistics* statistics) {
6367
DORIS_CHECK(fragment.complete());
6468
const auto result = _options.write_manager->try_submit_block(AsyncCacheWriteBlockRequest {
6569
.cache_hash = _options.cache_hash,
@@ -70,12 +74,23 @@ bool RangeCacheWriteback::_submit_complete_block(const FileCacheBlockFragment& f
7074
.write_epoch = write_epoch,
7175
.inflight_index = _options.inflight_index,
7276
});
77+
if (statistics != nullptr) {
78+
if (result == AsyncCacheWriteBlockSubmitResult::SUBMITTED) {
79+
COUNTER_UPDATE(&statistics->complete_blocks_submitted, 1);
80+
COUNTER_UPDATE(&statistics->complete_block_bytes, fragment.data.size);
81+
} else if (result == AsyncCacheWriteBlockSubmitResult::ALREADY_INFLIGHT) {
82+
COUNTER_UPDATE(&statistics->writeback_deduplicated_blocks, 1);
83+
} else {
84+
COUNTER_UPDATE(&statistics->writeback_rejected_blocks, 1);
85+
}
86+
}
7387
return result == AsyncCacheWriteBlockSubmitResult::SUBMITTED ||
7488
result == AsyncCacheWriteBlockSubmitResult::ALREADY_INFLIGHT;
7589
}
7690

7791
bool RangeCacheWriteback::_submit_partial_block(const FileCacheBlockFragment& fragment,
78-
const AsyncCacheWriteEpoch& write_epoch) {
92+
const AsyncCacheWriteEpoch& write_epoch,
93+
ReadAheadStatistics* statistics) {
7994
DORIS_CHECK(!fragment.complete());
8095
const auto result = _options.partial_block_manager->try_submit(PartialBlockWritebackRequest {
8196
.write_manager = _options.write_manager,
@@ -90,6 +105,21 @@ bool RangeCacheWriteback::_submit_partial_block(const FileCacheBlockFragment& fr
90105
.write_epoch = write_epoch,
91106
.io_context = _options.io_context,
92107
});
108+
if (statistics != nullptr) {
109+
if (result == PartialBlockSubmitResult::QUEUED ||
110+
result == PartialBlockSubmitResult::MERGED) {
111+
auto* counter = result == PartialBlockSubmitResult::QUEUED
112+
? &statistics->partial_blocks_queued
113+
: &statistics->partial_blocks_merged;
114+
COUNTER_UPDATE(counter, 1);
115+
COUNTER_UPDATE(&statistics->partial_fragment_bytes, fragment.data.size);
116+
} else if (result == PartialBlockSubmitResult::ACTIVE_DEDUPLICATED ||
117+
result == PartialBlockSubmitResult::CACHE_WRITE_INFLIGHT) {
118+
COUNTER_UPDATE(&statistics->writeback_deduplicated_blocks, 1);
119+
} else {
120+
COUNTER_UPDATE(&statistics->writeback_rejected_blocks, 1);
121+
}
122+
}
93123
return result == PartialBlockSubmitResult::QUEUED ||
94124
result == PartialBlockSubmitResult::MERGED ||
95125
result == PartialBlockSubmitResult::ACTIVE_DEDUPLICATED ||

‎be/src/io/cache/range_cache_writeback.h‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,15 +60,18 @@ class RangeCacheWriteback {
6060
/// Copy each block fragment into its owning asynchronous path. The range buffer only needs to
6161
/// remain valid for this call.
6262
RangeWritebackDispatchResult submit_consumed_range(const FileRange& range, Slice data,
63-
const AsyncCacheWriteEpoch& write_epoch);
63+
const AsyncCacheWriteEpoch& write_epoch,
64+
ReadAheadStatistics* statistics = nullptr);
6465

6566
private:
6667
/// Copy a complete fragment directly into the asynchronous cache writer.
6768
bool _submit_complete_block(const FileCacheBlockFragment& fragment,
68-
const AsyncCacheWriteEpoch& write_epoch);
69+
const AsyncCacheWriteEpoch& write_epoch,
70+
ReadAheadStatistics* statistics);
6971
/// Copy a partial fragment into the background hole-fill manager.
7072
bool _submit_partial_block(const FileCacheBlockFragment& fragment,
71-
const AsyncCacheWriteEpoch& write_epoch);
73+
const AsyncCacheWriteEpoch& write_epoch,
74+
ReadAheadStatistics* statistics);
7275

7376
const RangeCacheWritebackOptions _options;
7477
};

0 commit comments

Comments
 (0)