Skip to content

Commit b9396d9

Browse files
committed
[feature](be) Connect read-ahead range writeback
Issue Number: None Related PR: apache#67292 Problem Summary: Query read-ahead must publish only buffers that were actually consumed, preserve caller-visible IO statistics, and honor the original cache-write policy. Centralize query read-ahead construction so SegmentIterator and exact row-id readers share the same byte windows, range planner, page-cache probe, scheduler, and writeback setup. Each asynchronous submission owns its statistics and File Cache epoch; the first consumed page publishes the completed range once, while no-write and remote-only requests skip cache writeback. Enable consumed query read-ahead ranges to populate File Cache when asynchronous cache writeback is enabled. - Test: Unit Test - ./run-be-ut.sh --run --filter=AsyncCachedRemoteFileReaderTest.*:SegmentReadAheadTest.*:SegmentIteratorReadAheadTest.*:ColumnReaderTest.*ReadAhead* -j100 - Behavior changed: Yes; consumed successful query read-ahead ranges now contribute IO statistics and enter cache writeback when policy permits. - Does this need documentation: Yes; covered by the read-ahead and hole-fill design document.
1 parent e0870fa commit b9396d9

9 files changed

Lines changed: 275 additions & 59 deletions

‎be/src/io/cache/cached_remote_file_reader.cpp‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
#include "io/cache/file_block.h"
5151
#include "io/cache/inflight_write_buffer_index.h"
5252
#include "io/cache/peer_file_cache_reader.h"
53+
#include "io/cache/range_cache_writeback.h"
5354
#include "io/cache/remote_scan_cache_write_limiter.h"
5455
#include "io/fs/file_reader.h"
5556
#include "io/fs/local_file_system.h"
@@ -133,6 +134,32 @@ CachedRemoteFileReader::CachedRemoteFileReader(FileReaderSPtr remote_file_reader
133134
}
134135
}
135136

137+
std::shared_ptr<RangeCacheWriteback> CachedRemoteFileReader::make_range_cache_writeback(
138+
const IOContext& io_context, PartialBlockWritebackManager* partial_block_manager) const {
139+
DORIS_CHECK(_cache != nullptr);
140+
DORIS_CHECK(partial_block_manager != nullptr);
141+
if (_resolve_cache_write_mode(&io_context) == CacheWriteMode::NO_WRITE ||
142+
use_remote_only_on_cache_miss(&io_context)) {
143+
return nullptr;
144+
}
145+
auto* write_manager = _cache->async_write_manager();
146+
DORIS_CHECK(write_manager != nullptr);
147+
CacheContext cache_context(&io_context);
148+
return std::make_shared<RangeCacheWriteback>(RangeCacheWritebackOptions {
149+
.write_manager = write_manager,
150+
.partial_block_manager = partial_block_manager,
151+
.inflight_index = config::enable_async_file_cache_write_inflight_write_buffer_index
152+
? _cache->inflight_write_buffer_index()
153+
: nullptr,
154+
.source_reader = _remote_file_reader,
155+
.cache_hash = _cache_hash,
156+
.file_size = size(),
157+
.block_size = static_cast<size_t>(config::file_cache_each_block_size),
158+
.admission_ctx = CacheAdmissionContext::from_cache_context(cache_context, _tablet_id),
159+
.io_context = FileRangeReadIOContext::from_caller(io_context),
160+
});
161+
}
162+
136163
void CachedRemoteFileReader::_init_doris_table_cache() {
137164
_cache_hash = BlockFileCache::hash(path().filename().native());
138165
_cache = FileCacheFactory::instance()->get_by_path(_cache_hash);

‎be/src/io/cache/cached_remote_file_reader.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ struct AsyncCacheWriteEpoch;
3939
struct IOContext;
4040
struct FileCacheStatistics;
4141
struct PeerFetchResult;
42+
class PartialBlockWritebackManager;
43+
class RangeCacheWriteback;
4244

4345
} // namespace doris::io
4446

@@ -87,6 +89,12 @@ class CachedRemoteFileReader final : public FileReader,
8789
/// @return Raw pointer to the underlying reader owned by this object.
8890
FileReader* get_remote_reader() { return _remote_file_reader.get(); }
8991

92+
/// Build the immutable cache metadata used to write back consumed query read-ahead ranges.
93+
/// The returned object retains the underlying remote reader for background hole fills. A
94+
/// no-write request returns nullptr.
95+
std::shared_ptr<RangeCacheWriteback> make_range_cache_writeback(
96+
const IOContext& io_context, PartialBlockWritebackManager* partial_block_manager) const;
97+
9098
/// Align a read range to file-cache block boundaries.
9199
/// @param[in] offset Requested read offset in bytes.
92100
/// @param[in] size Requested read size in bytes.

‎be/src/io/io_common.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,13 @@ struct FileReaderStats {
5555
size_t read_bytes = 0;
5656
int64_t read_time_ns = 0;
5757
size_t read_rows = 0;
58+
59+
void merge_from(const FileReaderStats& other) {
60+
read_calls += other.read_calls;
61+
read_bytes += other.read_bytes;
62+
read_time_ns += other.read_time_ns;
63+
read_rows += other.read_rows;
64+
}
5865
};
5966

6067
struct FileCacheStatistics {

‎be/src/storage/segment/segment_iterator.cpp‎

Lines changed: 10 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,6 @@
6868
#include "exprs/vliteral.h"
6969
#include "exprs/vmatch_predicate.h"
7070
#include "exprs/vslot_ref.h"
71-
#include "io/cache/cached_remote_file_reader.h"
7271
#include "io/fs/file_reader.h"
7372
#include "io/io_common.h"
7473
#include "runtime/exec_env.h"
@@ -387,8 +386,7 @@ Status SegmentIterator::init(const StorageReadOptions& opts) {
387386
}
388387

389388
Status SegmentIterator::_init_query_read_ahead() {
390-
if (!config::enable_query_read_ahead || !config::is_cloud_mode() ||
391-
_opts.io_ctx.reader_type != ReaderType::READER_QUERY) {
389+
if (!SegmentReadAhead::enabled() || _opts.io_ctx.reader_type != ReaderType::READER_QUERY) {
392390
return Status::OK();
393391
}
394392

@@ -399,48 +397,18 @@ Status SegmentIterator::_init_query_read_ahead() {
399397
DORIS_CHECK(scheduler != nullptr);
400398
auto* query_context = _opts.runtime_state->get_query_ctx();
401399
DORIS_CHECK(query_context != nullptr);
402-
403-
ColumnReadAheadOptions eager_options {
404-
.high_watermark_bytes = cast_set<size_t>(config::read_ahead_eager_high_watermark_bytes),
405-
.low_watermark_bytes = cast_set<size_t>(config::read_ahead_eager_low_watermark_bytes),
406-
};
407-
ColumnReadAheadOptions lazy_options {
408-
.high_watermark_bytes = cast_set<size_t>(config::read_ahead_lazy_high_watermark_bytes),
409-
.low_watermark_bytes = cast_set<size_t>(config::read_ahead_lazy_low_watermark_bytes),
410-
};
411-
const auto eager_status = eager_options.validate();
412-
const auto lazy_status = lazy_options.validate();
413-
if (!eager_status.ok() || !lazy_status.ok()) {
414-
LOG_EVERY_N(WARNING, 100) << "invalid query read-ahead byte windows; feature disabled for "
415-
"this segment: eager="
416-
<< eager_status << ", lazy=" << lazy_status;
400+
auto read_context = query_context->get_or_create_file_range_read_context(scheduler);
401+
const auto status = SegmentReadAhead::create_for_query(
402+
_file_reader, exec_env, std::move(read_context), _opts, &_segment_read_ahead);
403+
if (!status.ok()) {
404+
LOG_EVERY_N(WARNING, 100)
405+
<< "failed to initialize query read-ahead; use the original read path: " << status;
417406
return Status::OK();
418407
}
419-
420-
SegmentReadAheadOptions options {
421-
.range_plan = {.coalesce_options =
422-
{.max_gap_bytes =
423-
cast_set<size_t>(config::read_ahead_max_gap_bytes),
424-
.max_range_bytes =
425-
cast_set<size_t>(config::read_ahead_max_range_bytes),
426-
.max_read_amplification_ratio =
427-
config::read_ahead_max_read_amplification_ratio},
428-
.cache_block_size = cast_set<size_t>(config::file_cache_each_block_size),
429-
.block_fill_min_coverage = config::read_ahead_block_fill_min_coverage},
430-
.page_cache_probe = _opts.use_page_cache ? make_storage_page_cache_probe(_file_reader)
431-
: ReadAheadPageCacheProbe {},
432-
.range_consumer_factory = {},
433-
};
434-
auto read_context = query_context->get_or_create_file_range_read_context(scheduler);
435-
_segment_read_ahead = std::make_unique<SegmentReadAhead>(
436-
_file_reader, scheduler, std::move(read_context),
437-
io::FileRangeReadIOContext::from_caller(_opts.io_ctx), std::move(options));
408+
DORIS_CHECK(_segment_read_ahead != nullptr);
438409
_file_reader = _segment_read_ahead->file_reader();
439-
_column_read_ahead_context = std::make_unique<ColumnReadAheadContext>(ColumnReadAheadContext {
440-
.eager_options = eager_options,
441-
.lazy_options = lazy_options,
442-
.segment = _segment_read_ahead.get(),
443-
});
410+
_column_read_ahead_context =
411+
std::make_unique<ColumnReadAheadContext>(_segment_read_ahead->column_context());
444412
return Status::OK();
445413
}
446414

‎be/src/storage/segment/segment_read_ahead.cpp‎

Lines changed: 100 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,15 @@
2323
#include <cstring>
2424
#include <utility>
2525

26+
#include "common/cast_set.h"
27+
#include "common/config.h"
2628
#include "common/logging.h"
29+
#include "io/cache/cached_remote_file_reader.h"
30+
#include "io/cache/range_cache_writeback.h"
31+
#include "runtime/exec_env.h"
2732
#include "storage/cache/page_cache.h"
33+
#include "storage/iterators.h"
34+
#include "storage/segment/column_reader.h"
2835

2936
namespace doris::segment_v2 {
3037
namespace {
@@ -130,7 +137,7 @@ void SegmentReadAheadFileReader::_finish_page(const PageKey& key, bool consumed)
130137
}
131138
if (consumed_range != nullptr && consumed_range->consumer) {
132139
auto consumer = std::move(consumed_range->consumer);
133-
consumer(consumed_range->read->range(), consumed_range->read->data());
140+
consumer(*consumed_range->read);
134141
}
135142
}
136143

@@ -156,19 +163,109 @@ SegmentReadAhead::SegmentReadAhead(io::FileReaderSPtr source_reader,
156163
io::FileRangeReadScheduler* scheduler,
157164
std::shared_ptr<io::FileRangeReadContext> context,
158165
io::FileRangeReadIOContext io_context,
159-
SegmentReadAheadOptions options)
166+
SegmentReadAheadOptions options,
167+
ColumnReadAheadOptions eager_options,
168+
ColumnReadAheadOptions lazy_options)
160169
: _source_reader(std::move(source_reader)),
161170
_scheduler(scheduler),
162171
_context(std::move(context)),
163172
_io_context(std::move(io_context)),
164173
_options(std::move(options)),
165174
_reader(std::shared_ptr<SegmentReadAheadFileReader>(
166-
new SegmentReadAheadFileReader(_source_reader))) {
175+
new SegmentReadAheadFileReader(_source_reader))),
176+
_column_context {
177+
.eager_options = eager_options, .lazy_options = lazy_options, .segment = this} {
167178
DORIS_CHECK(_source_reader != nullptr);
168179
DORIS_CHECK(_scheduler != nullptr);
169180
DORIS_CHECK(_context != nullptr);
170181
}
171182

183+
bool SegmentReadAhead::enabled() {
184+
return config::enable_query_read_ahead && config::is_cloud_mode();
185+
}
186+
187+
Status SegmentReadAhead::create_for_query(io::FileReaderSPtr source_reader, ExecEnv* exec_env,
188+
std::shared_ptr<io::FileRangeReadContext> context,
189+
const StorageReadOptions& read_options,
190+
std::unique_ptr<SegmentReadAhead>* output) {
191+
DORIS_CHECK(source_reader != nullptr);
192+
DORIS_CHECK(exec_env != nullptr);
193+
DORIS_CHECK(output != nullptr);
194+
output->reset();
195+
if (!enabled() || read_options.io_ctx.reader_type != ReaderType::READER_QUERY) {
196+
return Status::OK();
197+
}
198+
DORIS_CHECK(context != nullptr);
199+
200+
ColumnReadAheadOptions eager_options {
201+
.high_watermark_bytes = cast_set<size_t>(config::read_ahead_eager_high_watermark_bytes),
202+
.low_watermark_bytes = cast_set<size_t>(config::read_ahead_eager_low_watermark_bytes),
203+
};
204+
ColumnReadAheadOptions lazy_options {
205+
.high_watermark_bytes = cast_set<size_t>(config::read_ahead_lazy_high_watermark_bytes),
206+
.low_watermark_bytes = cast_set<size_t>(config::read_ahead_lazy_low_watermark_bytes),
207+
};
208+
RETURN_IF_ERROR(eager_options.validate());
209+
RETURN_IF_ERROR(lazy_options.validate());
210+
211+
auto* file_cache_stats = read_options.io_ctx.file_cache_stats;
212+
auto* file_reader_stats = read_options.io_ctx.file_reader_stats;
213+
std::shared_ptr<io::RangeCacheWriteback> range_writeback;
214+
auto cached_reader = std::dynamic_pointer_cast<io::CachedRemoteFileReader>(source_reader);
215+
if (cached_reader != nullptr) {
216+
auto* partial_block_manager = exec_env->partial_block_writeback_manager();
217+
DORIS_CHECK(partial_block_manager != nullptr);
218+
range_writeback = cached_reader->make_range_cache_writeback(read_options.io_ctx,
219+
partial_block_manager);
220+
}
221+
ReadAheadRangeConsumerFactory range_consumer_factory;
222+
if (file_cache_stats != nullptr || file_reader_stats != nullptr || range_writeback != nullptr) {
223+
range_consumer_factory = [file_cache_stats, file_reader_stats,
224+
range_writeback]() -> ReadAheadRangeConsumer {
225+
std::optional<io::AsyncCacheWriteEpoch> write_epoch;
226+
if (range_writeback != nullptr) {
227+
write_epoch = range_writeback->capture_write_epoch();
228+
}
229+
return [file_cache_stats, file_reader_stats, range_writeback,
230+
write_epoch = std::move(write_epoch)](const io::FileRangeRead& read) {
231+
const auto stats = read.stats();
232+
if (file_cache_stats != nullptr) {
233+
file_cache_stats->merge_from(stats.file_cache);
234+
}
235+
if (file_reader_stats != nullptr) {
236+
file_reader_stats->merge_from(stats.file_reader);
237+
}
238+
if (write_epoch.has_value()) {
239+
static_cast<void>(range_writeback->submit_consumed_range(
240+
read.range(), read.data(), *write_epoch));
241+
}
242+
};
243+
};
244+
}
245+
246+
SegmentReadAheadOptions options {
247+
.range_plan = {.coalesce_options =
248+
{.max_gap_bytes =
249+
cast_set<size_t>(config::read_ahead_max_gap_bytes),
250+
.max_range_bytes =
251+
cast_set<size_t>(config::read_ahead_max_range_bytes),
252+
.max_read_amplification_ratio =
253+
config::read_ahead_max_read_amplification_ratio},
254+
.cache_block_size = cast_set<size_t>(config::file_cache_each_block_size),
255+
.block_fill_min_coverage = config::read_ahead_block_fill_min_coverage},
256+
.page_cache_probe = read_options.use_page_cache
257+
? make_storage_page_cache_probe(source_reader)
258+
: ReadAheadPageCacheProbe {},
259+
.range_consumer_factory = std::move(range_consumer_factory),
260+
};
261+
auto* scheduler = exec_env->file_range_read_scheduler();
262+
DORIS_CHECK(scheduler != nullptr);
263+
output->reset(new SegmentReadAhead(std::move(source_reader), scheduler, std::move(context),
264+
io::FileRangeReadIOContext::from_caller(read_options.io_ctx),
265+
std::move(options), eager_options, lazy_options));
266+
return Status::OK();
267+
}
268+
172269
SegmentReadAheadResult SegmentReadAhead::apply_plans(std::vector<ColumnReadAheadPlan> plans) {
173270
SegmentReadAheadResult result;
174271
std::vector<PendingPage> misses;

‎be/src/storage/segment/segment_read_ahead.h‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,18 @@
2929
#include "io/fs/file_reader.h"
3030
#include "storage/segment/column_read_ahead.h"
3131

32-
namespace doris::segment_v2 {
32+
namespace doris {
33+
34+
class ExecEnv;
35+
class StorageReadOptions;
36+
37+
namespace segment_v2 {
38+
39+
class ColumnIterator;
3340

3441
/// Invoked with the whole physical range at most once, when any registered page successfully
3542
/// consumes bytes from it.
36-
using ReadAheadRangeConsumer = std::function<void(const io::FileRange&, Slice)>;
43+
using ReadAheadRangeConsumer = std::function<void(const io::FileRangeRead&)>;
3744
/// Creates submission-scoped state immediately before asynchronous range tasks are admitted.
3845
using ReadAheadRangeConsumerFactory = std::function<ReadAheadRangeConsumer()>;
3946
/// Returns true when the exact compressed data-page range is already in StoragePageCache.
@@ -143,7 +150,19 @@ class SegmentReadAhead final {
143150
public:
144151
SegmentReadAhead(io::FileReaderSPtr source_reader, io::FileRangeReadScheduler* scheduler,
145152
std::shared_ptr<io::FileRangeReadContext> context,
146-
io::FileRangeReadIOContext io_context, SegmentReadAheadOptions options);
153+
io::FileRangeReadIOContext io_context, SegmentReadAheadOptions options,
154+
ColumnReadAheadOptions eager_options = {},
155+
ColumnReadAheadOptions lazy_options = {});
156+
157+
/// Read the dynamic query read-ahead switch and cloud-mode requirement.
158+
static bool enabled();
159+
160+
/// Create query read-ahead state when enabled for a query reader. On an inapplicable request,
161+
/// return OK with `output` cleared so callers continue with the original reader.
162+
static Status create_for_query(io::FileReaderSPtr source_reader, ExecEnv* exec_env,
163+
std::shared_ptr<io::FileRangeReadContext> context,
164+
const StorageReadOptions& read_options,
165+
std::unique_ptr<SegmentReadAhead>* output);
147166

148167
SegmentReadAhead(const SegmentReadAhead&) = delete;
149168
SegmentReadAhead& operator=(const SegmentReadAhead&) = delete;
@@ -155,15 +174,19 @@ class SegmentReadAhead final {
155174

156175
const std::shared_ptr<SegmentReadAheadFileReader>& file_reader() const { return _reader; }
157176

177+
const ColumnReadAheadContext& column_context() const { return _column_context; }
178+
158179
private:
159180
const io::FileReaderSPtr _source_reader;
160181
io::FileRangeReadScheduler* const _scheduler;
161182
const std::shared_ptr<io::FileRangeReadContext> _context;
162183
const io::FileRangeReadIOContext _io_context;
163184
const SegmentReadAheadOptions _options;
164185
const std::shared_ptr<SegmentReadAheadFileReader> _reader;
186+
const ColumnReadAheadContext _column_context;
165187
};
166188

167189
ReadAheadPageCacheProbe make_storage_page_cache_probe(const io::FileReaderSPtr& file_reader);
168190

169-
} // namespace doris::segment_v2
191+
} // namespace segment_v2
192+
} // namespace doris

0 commit comments

Comments
 (0)