Skip to content

Commit 4f48ccf

Browse files
Gabriel39Gabriel39
authored
[improvement](be) Improve FileScannerV2 profiling and pruning (#65449)
### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: FileScannerV2 collected file-cache statistics but did not publish the FileCache profile subtree or account for cache-write bytes in the query resource context. It also lacked V1-compatible split pruning for late-arriving runtime filters on partition columns and always propagated missing-file errors, even when `ignore_not_found_file_in_external_table` was enabled. This PR: - publishes the shared FileCache profile counters from FileScannerV2 and records cache-write bytes; - passes the latest scanner conjunct snapshot into `TableReader::prepare_split`, where partition-only runtime filters are evaluated before a concrete reader is opened; - reports `FileScannerRuntimeFilterPartitionPruningTime` and `RuntimeFilterPartitionPrunedRangeNum`; - skips and counts `NOT_FOUND` splits when the external-table missing-file option is enabled; - resets native, JNI, Hudi hybrid, and Paimon hybrid split state before continuing; - adds focused unit coverage for profile reporting, partition pruning, and split cleanup after `NOT_FOUND`. ### Release note FileScannerV2 profiles now expose FileCache statistics, runtime-filter partition range pruning statistics, and ignored missing-file counts. FileScannerV2 can prune partition ranges before opening a reader and can continue after missing external files when the existing ignore option is enabled. ### Check List (For Author) - Test: Manual test / Unit Test attempted - clang-format 16 dry-run passed for all changed C++ files - `git diff --check` passed - targeted BE unit tests were started, but CMake configuration was blocked because `thirdparty/installed` is missing Protobuf; tests did not reach compilation - Behavior changed: Yes. FileScannerV2 publishes additional profile counters, prunes partition ranges using runtime filters, and skips configured ignorable `NOT_FOUND` splits. - Does this need documentation: Yes. The existing FileScanner V1/V2 profile document is updated in place.
1 parent 5aa3c6c commit 4f48ccf

13 files changed

Lines changed: 351 additions & 23 deletions

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 78 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
#include "format_v2/table/paimon_reader.h"
5959
#include "format_v2/table/remote_doris_reader.h"
6060
#include "format_v2/table_reader.h"
61+
#include "io/cache/block_file_cache_profile.h"
6162
#include "io/fs/file_meta_cache.h"
6263
#include "io/io_common.h"
6364
#include "runtime/descriptors.h"
@@ -243,6 +244,15 @@ FileScannerV2::RealtimeCounterDeltas FileScannerV2::TEST_collect_realtime_counte
243244
last_read_rows, last_bytes_read_from_local,
244245
last_bytes_read_from_remote);
245246
}
247+
248+
void FileScannerV2::TEST_report_file_cache_profile(
249+
RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics) {
250+
_report_file_cache_profile(profile, file_cache_statistics);
251+
}
252+
253+
bool FileScannerV2::TEST_should_skip_not_found(const Status& status, bool ignore_not_found) {
254+
return _should_skip_not_found(status, ignore_not_found);
255+
}
246256
#endif
247257

248258
bool FileScannerV2::is_supported(const TFileScanRangeParams& params, const TFileRangeDesc& range) {
@@ -283,6 +293,8 @@ Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjunc
283293
RETURN_IF_ERROR(Scanner::init(state, conjuncts));
284294
_get_block_timer =
285295
ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileScannerV2GetBlockTime", 1);
296+
_not_found_file_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(),
297+
"NotFoundFileNum", TUnit::UNIT, 1);
286298
_file_counter =
287299
ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), "FileNumber", TUnit::UNIT, 1);
288300
_file_read_bytes_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(),
@@ -334,7 +346,17 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e
334346
if (_should_run_adaptive_batch_size()) {
335347
_table_reader->set_batch_size(_predict_reader_batch_rows());
336348
}
337-
RETURN_IF_ERROR(_table_reader->get_block(block, eof));
349+
const auto status = _table_reader->get_block(block, eof);
350+
if (_should_skip_not_found(status, config::ignore_not_found_file_in_external_table)) {
351+
RETURN_IF_ERROR(_table_reader->abort_split());
352+
COUNTER_UPDATE(_not_found_file_counter, 1);
353+
_state->update_num_finished_scan_range(1);
354+
_has_prepared_split = false;
355+
block->clear_column_data(cast_set<int64_t>(_projected_columns.size()));
356+
*eof = false;
357+
continue;
358+
}
359+
RETURN_IF_ERROR(status);
338360
}
339361
if (*eof) {
340362
_state->update_num_finished_scan_range(1);
@@ -348,23 +370,40 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e
348370
}
349371

350372
Status FileScannerV2::_prepare_next_split(bool* eos) {
351-
bool has_next = _first_scan_range;
352-
if (!_first_scan_range) {
353-
RETURN_IF_ERROR(_split_source->get_next(&has_next, &_current_range));
354-
}
355-
_first_scan_range = false;
356-
if (!has_next || _should_stop) {
357-
*eos = true;
373+
while (true) {
374+
bool has_next = _first_scan_range;
375+
if (!_first_scan_range) {
376+
RETURN_IF_ERROR(_split_source->get_next(&has_next, &_current_range));
377+
}
378+
_first_scan_range = false;
379+
if (!has_next || _should_stop) {
380+
*eos = true;
381+
return Status::OK();
382+
}
383+
DORIS_CHECK(_table_reader != nullptr);
384+
_current_range_path = _current_range.path;
385+
386+
std::map<std::string, Field> partition_values;
387+
RETURN_IF_ERROR(_generate_partition_values(_current_range, &partition_values));
388+
const auto status =
389+
_prepare_table_reader_split(_current_range, std::move(partition_values));
390+
if (_should_skip_not_found(status, config::ignore_not_found_file_in_external_table)) {
391+
RETURN_IF_ERROR(_table_reader->abort_split());
392+
COUNTER_UPDATE(_not_found_file_counter, 1);
393+
_state->update_num_finished_scan_range(1);
394+
continue;
395+
}
396+
RETURN_IF_ERROR(status);
397+
if (_table_reader->current_split_pruned()) {
398+
_state->update_num_finished_scan_range(1);
399+
continue;
400+
}
401+
_init_adaptive_batch_size_state(get_range_format_type(*_params, _current_range));
402+
COUNTER_UPDATE(_file_counter, 1);
403+
_has_prepared_split = true;
404+
*eos = false;
358405
return Status::OK();
359406
}
360-
DORIS_CHECK(_table_reader != nullptr);
361-
_current_range_path = _current_range.path;
362-
_init_adaptive_batch_size_state(get_range_format_type(*_params, _current_range));
363-
RETURN_IF_ERROR(_prepare_table_reader_split(_current_range));
364-
COUNTER_UPDATE(_file_counter, 1);
365-
_has_prepared_split = true;
366-
*eos = false;
367-
return Status::OK();
368407
}
369408

370409
Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) {
@@ -425,13 +464,17 @@ Status FileScannerV2::_create_table_reader_for_format(
425464
return Status::OK();
426465
}
427466

428-
Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range) {
429-
std::map<std::string, Field> partition_values;
430-
RETURN_IF_ERROR(_generate_partition_values(range, &partition_values));
467+
Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range,
468+
std::map<std::string, Field> partition_values) {
431469
format::FileFormat current_split_format;
432470
RETURN_IF_ERROR(_to_file_format(get_range_format_type(*_params, range), &current_split_format));
471+
VExprContextSPtrs partition_prune_conjuncts;
472+
if (_state->query_options().enable_runtime_filter_partition_prune) {
473+
RETURN_IF_ERROR(_build_table_conjuncts(&partition_prune_conjuncts));
474+
}
433475
RETURN_IF_ERROR(_table_reader->prepare_split({
434476
.partition_values = std::move(partition_values),
477+
.partition_prune_conjuncts = std::move(partition_prune_conjuncts),
435478
.cache = _kv_cache,
436479
.current_range = range,
437480
.current_split_format = current_split_format,
@@ -440,6 +483,10 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range) {
440483
return Status::OK();
441484
}
442485

486+
bool FileScannerV2::_should_skip_not_found(const Status& status, bool ignore_not_found) {
487+
return ignore_not_found && status.is<ErrorCode::NOT_FOUND>();
488+
}
489+
443490
bool FileScannerV2::_should_enable_file_meta_cache() const {
444491
return ExecEnv::GetInstance()->file_meta_cache()->enabled() &&
445492
_split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3;
@@ -874,6 +921,12 @@ FileScannerV2::UncachedReaderBytesStorage FileScannerV2::_uncached_reader_bytes_
874921
void FileScannerV2::_collect_profile_before_close() {
875922
_report_file_reader_predicate_filtered_rows();
876923
Scanner::_collect_profile_before_close();
924+
if (config::enable_file_cache && _state->query_options().enable_file_cache &&
925+
_profile != nullptr) {
926+
_report_file_cache_profile(_profile, *_file_cache_statistics);
927+
_state->get_query_ctx()->resource_ctx()->io_context()->update_bytes_write_into_cache(
928+
_file_cache_statistics->bytes_write_into_cache);
929+
}
877930
if (_file_reader_stats != nullptr) {
878931
COUNTER_SET(_file_read_bytes_counter, cast_set<int64_t>(_file_reader_stats->read_bytes));
879932
COUNTER_SET(_file_read_calls_counter, cast_set<int64_t>(_file_reader_stats->read_calls));
@@ -884,6 +937,12 @@ void FileScannerV2::_collect_profile_before_close() {
884937
_report_condition_cache_profile();
885938
}
886939

940+
void FileScannerV2::_report_file_cache_profile(
941+
RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics) {
942+
io::FileCacheProfileReporter cache_profile(profile);
943+
cache_profile.update(&file_cache_statistics);
944+
}
945+
887946
bool FileScannerV2::_should_update_load_counters() const {
888947
if (_is_load) {
889948
return true;

‎be/src/exec/scan/file_scanner_v2.h‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@ class FileScannerV2 final : public Scanner {
8080
UncachedReaderBytesStorage uncached_reader_bytes_storage, int64_t* last_read_bytes,
8181
int64_t* last_read_rows, int64_t* last_bytes_read_from_local,
8282
int64_t* last_bytes_read_from_remote);
83+
static void TEST_report_file_cache_profile(
84+
RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics);
85+
static bool TEST_should_skip_not_found(const Status& status, bool ignore_not_found);
8386
#endif
8487

8588
FileScannerV2(RuntimeState* state, FileScanLocalState* parent, int64_t limit,
@@ -108,7 +111,9 @@ class FileScannerV2 final : public Scanner {
108111
Status _init_table_reader(const TFileRangeDesc& range);
109112
Status _create_table_reader_for_format(const TFileRangeDesc& range,
110113
std::unique_ptr<format::TableReader>* reader) const;
111-
Status _prepare_table_reader_split(const TFileRangeDesc& range);
114+
Status _prepare_table_reader_split(const TFileRangeDesc& range,
115+
std::map<std::string, Field> partition_values);
116+
static bool _should_skip_not_found(const Status& status, bool ignore_not_found);
112117
bool _should_enable_file_meta_cache() const;
113118
std::optional<format::GlobalRowIdContext> _create_global_rowid_context(
114119
const TFileRangeDesc& range) const;
@@ -135,6 +140,8 @@ class FileScannerV2 final : public Scanner {
135140
int64_t* last_read_rows, int64_t* last_bytes_read_from_local,
136141
int64_t* last_bytes_read_from_remote);
137142
static UncachedReaderBytesStorage _uncached_reader_bytes_storage(TFileType::type file_type);
143+
static void _report_file_cache_profile(RuntimeProfile* profile,
144+
const io::FileCacheStatistics& file_cache_statistics);
138145
void _report_file_reader_predicate_filtered_rows();
139146
void _report_condition_cache_profile();
140147

@@ -167,6 +174,7 @@ class FileScannerV2 final : public Scanner {
167174
ShardedKVCache* _kv_cache = nullptr;
168175

169176
RuntimeProfile::Counter* _get_block_timer = nullptr;
177+
RuntimeProfile::Counter* _not_found_file_counter = nullptr;
170178
RuntimeProfile::Counter* _file_counter = nullptr;
171179
RuntimeProfile::Counter* _file_read_bytes_counter = nullptr;
172180
RuntimeProfile::Counter* _file_read_calls_counter = nullptr;

‎be/src/format_v2/jni/jni_table_reader.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ Status JniTableReader::prepare_split(const SplitReadOptions& options) {
4747
_current_range = options.current_range;
4848
RETURN_IF_ERROR(validate_scan_range(options.current_range));
4949
RETURN_IF_ERROR(TableReader::prepare_split(options));
50+
if (current_split_pruned()) {
51+
return Status::OK();
52+
}
5053
DORIS_CHECK(!_closed);
5154
DORIS_CHECK(!_scanner_opened);
5255
if (_is_table_level_count_active()) {
@@ -95,6 +98,11 @@ Status JniTableReader::get_block(Block* output_block, bool* eos) {
9598
}
9699
}
97100

101+
Status JniTableReader::abort_split() {
102+
RETURN_IF_ERROR(_close_jni_scanner());
103+
return TableReader::abort_split();
104+
}
105+
98106
Status JniTableReader::_get_next_jni_block(size_t* rows, bool* eof) {
99107
DORIS_CHECK(rows != nullptr);
100108
DORIS_CHECK(eof != nullptr);

‎be/src/format_v2/jni/jni_table_reader.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ class JniTableReader : public TableReader {
4949
Status init(TableReadOptions&& options) override;
5050
Status prepare_split(const SplitReadOptions& options) override;
5151
Status get_block(Block* block, bool* eos) override;
52+
Status abort_split() override;
5253
Status close() override;
5354

5455
protected:

‎be/src/format_v2/table/hudi_reader.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,16 @@ Status HudiHybridReader::get_block(Block* block, bool* eos) {
6666
return _current_split_reader->get_block(block, eos);
6767
}
6868

69+
bool HudiHybridReader::current_split_pruned() const {
70+
DORIS_CHECK(_current_split_reader != nullptr);
71+
return _current_split_reader->current_split_pruned();
72+
}
73+
74+
Status HudiHybridReader::abort_split() {
75+
DORIS_CHECK(_current_split_reader != nullptr);
76+
return _current_split_reader->abort_split();
77+
}
78+
6979
Status HudiHybridReader::close() {
7080
Status close_status = Status::OK();
7181
if (_native_reader != nullptr) {

‎be/src/format_v2/table/hudi_reader.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ class HudiHybridReader final : public format::TableReader {
5858
Status init(format::TableReadOptions&& options) override;
5959
Status prepare_split(const format::SplitReadOptions& options) override;
6060
Status get_block(Block* block, bool* eos) override;
61+
bool current_split_pruned() const override;
62+
Status abort_split() override;
6163
Status close() override;
6264

6365
private:

‎be/src/format_v2/table/iceberg_reader.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,9 @@ Status IcebergTableReader::prepare_split(const format::SplitReadOptions& options
193193
}
194194
}
195195
RETURN_IF_ERROR(TableReader::prepare_split(options));
196+
if (current_split_pruned()) {
197+
return Status::OK();
198+
}
196199
if (_is_table_level_count_active()) {
197200
return Status::OK();
198201
}

‎be/src/format_v2/table/paimon_reader.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,16 @@ Status PaimonHybridReader::get_block(Block* block, bool* eos) {
9191
return _current_split_reader->get_block(block, eos);
9292
}
9393

94+
bool PaimonHybridReader::current_split_pruned() const {
95+
DORIS_CHECK(_current_split_reader != nullptr);
96+
return _current_split_reader->current_split_pruned();
97+
}
98+
99+
Status PaimonHybridReader::abort_split() {
100+
DORIS_CHECK(_current_split_reader != nullptr);
101+
return _current_split_reader->abort_split();
102+
}
103+
94104
Status PaimonHybridReader::close() {
95105
Status close_status = Status::OK();
96106
if (_native_reader != nullptr) {

‎be/src/format_v2/table/paimon_reader.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@ class PaimonHybridReader final : public format::TableReader {
6363
Status init(format::TableReadOptions&& options) override;
6464
Status prepare_split(const format::SplitReadOptions& options) override;
6565
Status get_block(Block* block, bool* eos) override;
66+
bool current_split_pruned() const override;
67+
Status abort_split() override;
6668
Status close() override;
6769

6870
#ifdef BE_TEST

0 commit comments

Comments
 (0)