Skip to content

Commit a3bf75b

Browse files
committed
[improvement](external) Refine Parquet splits by row group in BE
1 parent daec798 commit a3bf75b

27 files changed

Lines changed: 1452 additions & 69 deletions

‎be/src/exec/operator/file_scan_operator.cpp‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -258,9 +258,54 @@ void FileScanLocalState::set_scan_ranges(RuntimeState* state,
258258
if (_split_source == nullptr) {
259259
_split_source = std::make_shared<LocalSplitSourceConnector>(scan_ranges, _max_scanners);
260260
}
261-
// currently the total number of splits in the bach split mode cannot be accurately obtained,
262-
// so we don't do it in the batch split mode.
263-
_max_scanners = std::min(_max_scanners, _split_source->num_scan_ranges());
261+
// A single FE Parquet split can publish many row-group children after its footer is read.
262+
// Keep the requested scanner concurrency in that case; capping it to the initial range
263+
// count would leave the generated children serial even though they share one footer.
264+
bool can_generate_parquet_splits = false;
265+
const TFileScanRangeParams* common_params = nullptr;
266+
if (state->get_query_ctx() != nullptr &&
267+
state->get_query_ctx()->file_scan_range_params_map.contains(parent_id())) {
268+
common_params = &state->get_query_ctx()->file_scan_range_params_map[parent_id()];
269+
}
270+
for (const auto& scan_range_params : scan_ranges) {
271+
const auto& file_scan_range =
272+
scan_range_params.scan_range.ext_scan_range.file_scan_range;
273+
const auto* params =
274+
file_scan_range.__isset.params ? &file_scan_range.params : common_params;
275+
if (params == nullptr) {
276+
continue;
277+
}
278+
const bool is_load =
279+
state->desc_tbl().get_tuple_descriptor(params->src_tuple_id) != nullptr;
280+
if (!_should_use_file_scanner_v2(state->query_options(), is_load, *params)) {
281+
continue;
282+
}
283+
can_generate_parquet_splits =
284+
std::ranges::any_of(file_scan_range.ranges, [&](const auto& range) {
285+
const auto format =
286+
range.__isset.format_type ? range.format_type : params->format_type;
287+
if (format == TFileFormatType::FORMAT_PARQUET) {
288+
return true;
289+
}
290+
if (format != TFileFormatType::FORMAT_JNI ||
291+
!range.__isset.table_format_params ||
292+
range.table_format_params.table_format_type != "paimon" ||
293+
!range.table_format_params.__isset.paimon_params) {
294+
return false;
295+
}
296+
const auto& paimon = range.table_format_params.paimon_params;
297+
return paimon.__isset.file_format && paimon.file_format == "parquet" &&
298+
!paimon.__isset.paimon_split;
299+
});
300+
if (can_generate_parquet_splits) {
301+
break;
302+
}
303+
}
304+
// Currently the total number of remote splits cannot be accurately obtained, so batch
305+
// mode already skips this cap.
306+
if (!can_generate_parquet_splits) {
307+
_max_scanners = std::min(_max_scanners, _split_source->num_scan_ranges());
308+
}
264309
}
265310

266311
if (!scan_ranges.empty() &&

‎be/src/exec/operator/file_scan_operator.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ class FileScanLocalState final : public ScanLocalState<FileScanLocalState> {
9494
// 2. parquet file meta
9595
// KVCache<std::string> _kv_cache;
9696
std::unique_ptr<ShardedKVCache> _kv_cache;
97+
FileContextRegistry _file_context_registry;
9798
TupleId _output_tuple_id = -1;
9899
RuntimeProfile::Counter* _condition_cache_hit_counter = nullptr;
99100
RuntimeProfile::Counter* _condition_cache_filtered_rows_counter = nullptr;

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 148 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,27 @@ Status adapt_runtime_filter_for_table_reader(VExprSPtr* expr) {
281281

282282
} // namespace
283283

284+
int64_t FileScannerV2::_cumulative_profile_delta(int64_t current, int64_t* reported) {
285+
DORIS_CHECK(reported != nullptr);
286+
DORIS_CHECK(current >= *reported);
287+
const int64_t delta = current - *reported;
288+
*reported = current;
289+
return delta;
290+
}
291+
292+
FileScannerV2::FileReaderProfileDeltas FileScannerV2::_collect_file_reader_profile_deltas(
293+
const io::FileReaderStats& stats, int64_t* reported_bytes, int64_t* reported_calls,
294+
int64_t* reported_time) {
295+
return {
296+
.read_bytes =
297+
_cumulative_profile_delta(cast_set<int64_t>(stats.read_bytes), reported_bytes),
298+
.read_calls =
299+
_cumulative_profile_delta(cast_set<int64_t>(stats.read_calls), reported_calls),
300+
.read_time_ns =
301+
_cumulative_profile_delta(cast_set<int64_t>(stats.read_time_ns), reported_time),
302+
};
303+
}
304+
284305
#ifdef BE_TEST
285306
FileScannerV2::FileScannerV2(RuntimeState* state, RuntimeProfile* profile,
286307
std::unique_ptr<format::TableReader> table_reader)
@@ -380,7 +401,8 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat
380401
const std::unordered_map<std::string, int>* colname_to_slot_id)
381402
: Scanner(state, local_state, limit, profile),
382403
_split_source(std::move(split_source)),
383-
_kv_cache(kv_cache) {
404+
_kv_cache(kv_cache),
405+
_file_context_registry(&local_state->_file_context_registry) {
384406
(void)colname_to_slot_id;
385407
if (state->get_query_ctx() != nullptr &&
386408
state->get_query_ctx()->file_scan_range_params_map.count(local_state->parent_id()) > 0) {
@@ -448,16 +470,50 @@ Status FileScannerV2::_open_impl(RuntimeState* state) {
448470
DORIS_CHECK(_table_reader != nullptr);
449471
RETURN_IF_ERROR(_init_expr_ctxes());
450472
RETURN_IF_ERROR(_init_table_reader(_current_range));
473+
// Refine the first source split before yielding the scanner worker. Other scanners may be
474+
// waiting for its row-group children, so deferring publication until a later get_block()
475+
// turn could let those waiters occupy the scan thread pool ahead of the producer.
476+
bool eos = false;
477+
RETURN_IF_ERROR(_prepare_next_split(&eos));
451478
}
452479
return Status::OK();
453480
}
454481

455482
Status FileScannerV2::_get_next_scan_range(bool* has_next) {
456483
SCOPED_TIMER(_get_next_range_timer);
457484
DORIS_CHECK(has_next != nullptr);
458-
RETURN_IF_ERROR(_split_source->get_next(has_next, &_current_range));
485+
RETURN_IF_ERROR(_split_source->get_next_split(has_next, &_current_split));
459486
if (*has_next) {
487+
_current_range = _current_split.materialize_range();
460488
RETURN_IF_ERROR(_validate_scan_range(*_params, _current_range));
489+
} else {
490+
_current_split = {};
491+
_current_range = {};
492+
}
493+
return Status::OK();
494+
}
495+
496+
Status FileScannerV2::_retire_current_source_split(std::vector<FileScanSplit> generated_splits) {
497+
if (!_current_split.is_source_split) {
498+
return Status::OK();
499+
}
500+
const bool transferred_progress = !generated_splits.empty();
501+
RETURN_IF_ERROR(
502+
_split_source->finish_source_split(_current_split, std::move(generated_splits)));
503+
_current_split.is_source_split = false;
504+
if (transferred_progress) {
505+
_current_split.source_progress.reset();
506+
}
507+
return Status::OK();
508+
}
509+
510+
Status FileScannerV2::_complete_current_split() {
511+
RETURN_IF_ERROR(_retire_current_source_split());
512+
if (_current_split.source_progress != nullptr) {
513+
if (_current_split.source_progress->complete_one()) {
514+
_state->update_num_finished_scan_range(1);
515+
}
516+
_current_split.source_progress.reset();
461517
}
462518
return Status::OK();
463519
}
@@ -488,7 +544,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e
488544
if (_should_skip_not_found(status, config::ignore_not_found_file_in_external_table)) {
489545
RETURN_IF_ERROR(_table_reader->abort_split());
490546
COUNTER_UPDATE(_not_found_file_counter, 1);
491-
_state->update_num_finished_scan_range(1);
547+
RETURN_IF_ERROR(_complete_current_split());
492548
_has_prepared_split = false;
493549
block->clear_column_data(cast_set<int64_t>(_projected_columns.size()));
494550
*eof = false;
@@ -502,7 +558,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e
502558
// discard partial reader state, and let the loop fetch the next split.
503559
RETURN_IF_ERROR(_table_reader->abort_split());
504560
COUNTER_UPDATE(_empty_file_counter, 1);
505-
_state->update_num_finished_scan_range(1);
561+
RETURN_IF_ERROR(_complete_current_split());
506562
_has_prepared_split = false;
507563
block->clear_column_data(cast_set<int64_t>(_projected_columns.size()));
508564
*eof = false;
@@ -511,7 +567,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e
511567
RETURN_IF_ERROR(status);
512568
}
513569
if (*eof) {
514-
_state->update_num_finished_scan_range(1);
570+
RETURN_IF_ERROR(_complete_current_split());
515571
_has_prepared_split = false;
516572
*eof = false;
517573
continue;
@@ -552,6 +608,7 @@ Status FileScannerV2::_prepare_next_split(bool* eos) {
552608
}
553609
_first_scan_range = false;
554610
if (!has_next || _should_stop) {
611+
RETURN_IF_ERROR(_complete_current_split());
555612
*eos = true;
556613
return Status::OK();
557614
}
@@ -575,7 +632,7 @@ Status FileScannerV2::_prepare_next_split(bool* eos) {
575632
if (_should_skip_not_found(status, config::ignore_not_found_file_in_external_table)) {
576633
RETURN_IF_ERROR(_table_reader->abort_split());
577634
COUNTER_UPDATE(_not_found_file_counter, 1);
578-
_state->update_num_finished_scan_range(1);
635+
RETURN_IF_ERROR(_complete_current_split());
579636
continue;
580637
}
581638
if (_should_skip_empty(status, _should_stop || _io_ctx->should_stop)) {
@@ -585,14 +642,51 @@ Status FileScannerV2::_prepare_next_split(bool* eos) {
585642
// advance exactly one scan range and preserve later files in the same scan.
586643
RETURN_IF_ERROR(_table_reader->abort_split());
587644
COUNTER_UPDATE(_empty_file_counter, 1);
588-
_state->update_num_finished_scan_range(1);
645+
RETURN_IF_ERROR(_complete_current_split());
589646
continue;
590647
}
591648
RETURN_IF_ERROR(status);
592649
if (_table_reader->current_split_pruned()) {
593-
_state->update_num_finished_scan_range(1);
650+
RETURN_IF_ERROR(_complete_current_split());
594651
continue;
595652
}
653+
if (_current_split.is_source_split && _can_refine_source_split(_current_range)) {
654+
std::vector<FileScanSplit> generated_splits;
655+
bool was_split = false;
656+
const auto split_status = _table_reader->build_physical_splits(
657+
_current_split, &generated_splits, &was_split);
658+
const auto ignored_split_status = _classify_ignored_split_status(
659+
split_status, config::ignore_not_found_file_in_external_table,
660+
_should_stop || _io_ctx->should_stop);
661+
if (ignored_split_status == IgnoredSplitStatus::NOT_FOUND) {
662+
RETURN_IF_ERROR(_table_reader->abort_split());
663+
COUNTER_UPDATE(_not_found_file_counter, 1);
664+
RETURN_IF_ERROR(_complete_current_split());
665+
continue;
666+
}
667+
if (ignored_split_status == IgnoredSplitStatus::EMPTY) {
668+
RETURN_IF_ERROR(_table_reader->abort_split());
669+
COUNTER_UPDATE(_empty_file_counter, 1);
670+
RETURN_IF_ERROR(_complete_current_split());
671+
continue;
672+
}
673+
RETURN_IF_ERROR(split_status);
674+
if (was_split) {
675+
RETURN_IF_ERROR(_table_reader->abort_split());
676+
const bool has_children = !generated_splits.empty();
677+
RETURN_IF_ERROR(_retire_current_source_split(std::move(generated_splits)));
678+
if (!has_children) {
679+
RETURN_IF_ERROR(_complete_current_split());
680+
}
681+
continue;
682+
}
683+
}
684+
if (_current_split.is_source_split) {
685+
// Non-Parquet and metadata-only source splits are already fully prepared and cannot
686+
// publish children. Release the source reservation before yielding the scanner worker
687+
// so waiters never depend on a later get_block() turn to observe completion.
688+
RETURN_IF_ERROR(_retire_current_source_split());
689+
}
596690
COUNTER_UPDATE(_file_counter, 1);
597691
_has_prepared_split = true;
598692
_table_reader_rf_num = _applied_rf_num;
@@ -636,6 +730,7 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) {
636730
.push_down_agg_type = _local_state->get_push_down_agg_type(),
637731
.push_down_count_columns = std::move(push_down_count_columns),
638732
.condition_cache_digest = _local_state->get_condition_cache_digest(),
733+
.file_context_registry = _file_context_registry,
639734
}));
640735
return Status::OK();
641736
}
@@ -709,6 +804,8 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range,
709804
.cache = _kv_cache,
710805
.current_range = range,
711806
.current_split_format = current_split_format,
807+
.file_context = _current_split.file_context,
808+
.format_split_id = _current_split.format_split_id,
712809
.global_rowid_context = _create_global_rowid_context(range),
713810
}));
714811
return Status::OK();
@@ -726,6 +823,27 @@ bool FileScannerV2::_should_skip_empty(const Status& status, bool stopped) {
726823
return !stopped && status.is<ErrorCode::END_OF_FILE>();
727824
}
728825

826+
FileScannerV2::IgnoredSplitStatus FileScannerV2::_classify_ignored_split_status(
827+
const Status& status, bool ignore_not_found, bool stopped) {
828+
if (_should_skip_not_found(status, ignore_not_found)) {
829+
return IgnoredSplitStatus::NOT_FOUND;
830+
}
831+
if (_should_skip_empty(status, stopped)) {
832+
return IgnoredSplitStatus::EMPTY;
833+
}
834+
return IgnoredSplitStatus::NONE;
835+
}
836+
837+
bool FileScannerV2::_can_refine_source_split(const TFileRangeDesc& range) {
838+
if (!range.__isset.table_format_params || !range.table_format_params.__isset.iceberg_params) {
839+
return true;
840+
}
841+
const auto& iceberg = range.table_format_params.iceberg_params;
842+
// Iceberg delete readers build split-local delete state. Refining the data file would rebuild
843+
// that state for every row-group child, so keep the already prepared source reader instead.
844+
return !iceberg.__isset.delete_files || iceberg.delete_files.empty();
845+
}
846+
729847
bool FileScannerV2::_should_enable_file_meta_cache() const {
730848
return ExecEnv::GetInstance()->file_meta_cache()->enabled() &&
731849
_split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3;
@@ -1053,16 +1171,24 @@ Status FileScannerV2::close(RuntimeState* state) {
10531171
if (_table_reader != nullptr) {
10541172
const auto close_status = _table_reader->close();
10551173
if (!close_status.ok()) {
1174+
// A failed reader close remains retryable, but this scanner can no longer publish
1175+
// useful children. Retire its source reservation so peer scanners are not stranded.
1176+
const auto split_status = _complete_current_split();
1177+
if (!split_status.ok()) {
1178+
return split_status;
1179+
}
10561180
return close_status;
10571181
}
10581182
_report_condition_cache_profile();
10591183
_table_reader.reset();
10601184
}
1185+
RETURN_IF_ERROR(_complete_current_split());
10611186
return Scanner::close(state);
10621187
}
10631188

10641189
void FileScannerV2::try_stop() {
10651190
Scanner::try_stop();
1191+
_split_source->stop();
10661192
if (_io_ctx) {
10671193
_io_ctx->should_stop = true;
10681194
}
@@ -1073,7 +1199,6 @@ void FileScannerV2::update_realtime_counters() {
10731199
return;
10741200
}
10751201
DORIS_CHECK(_file_cache_statistics != nullptr);
1076-
const int64_t bytes_read = cast_set<int64_t>(_file_reader_stats->read_bytes);
10771202
auto* local_state = static_cast<FileScanLocalState*>(_local_state);
10781203
const auto file_type =
10791204
_current_range.__isset.file_type
@@ -1095,9 +1220,14 @@ void FileScannerV2::update_realtime_counters() {
10951220
_state->get_query_ctx()->resource_ctx()->io_context()->update_scan_bytes_from_remote_storage(
10961221
deltas.scan_bytes_from_remote_storage);
10971222

1098-
COUNTER_SET(_file_read_bytes_counter, bytes_read);
1099-
COUNTER_SET(_file_read_calls_counter, cast_set<int64_t>(_file_reader_stats->read_calls));
1100-
COUNTER_SET(_file_read_time_counter, cast_set<int64_t>(_file_reader_stats->read_time_ns));
1223+
// Scanner instances share these profile counters. Publish only scanner-private deltas so one
1224+
// scanner cannot erase or double count values already contributed by a sibling.
1225+
const auto profile_deltas = _collect_file_reader_profile_deltas(
1226+
*_file_reader_stats, &_reported_file_read_bytes, &_reported_file_read_calls,
1227+
&_reported_file_read_time);
1228+
COUNTER_UPDATE(_file_read_bytes_counter, profile_deltas.read_bytes);
1229+
COUNTER_UPDATE(_file_read_calls_counter, profile_deltas.read_calls);
1230+
COUNTER_UPDATE(_file_read_time_counter, profile_deltas.read_time_ns);
11011231

11021232
DorisMetrics::instance()->query_scan_bytes->increment(deltas.scan_bytes);
11031233
DorisMetrics::instance()->query_scan_rows->increment(deltas.scan_rows);
@@ -1192,9 +1322,12 @@ void FileScannerV2::_collect_profile_before_close() {
11921322
_reported_file_cache_statistics = *_file_cache_statistics;
11931323
}
11941324
if (_file_reader_stats != nullptr) {
1195-
COUNTER_SET(_file_read_bytes_counter, cast_set<int64_t>(_file_reader_stats->read_bytes));
1196-
COUNTER_SET(_file_read_calls_counter, cast_set<int64_t>(_file_reader_stats->read_calls));
1197-
COUNTER_SET(_file_read_time_counter, cast_set<int64_t>(_file_reader_stats->read_time_ns));
1325+
const auto profile_deltas = _collect_file_reader_profile_deltas(
1326+
*_file_reader_stats, &_reported_file_read_bytes, &_reported_file_read_calls,
1327+
&_reported_file_read_time);
1328+
COUNTER_UPDATE(_file_read_bytes_counter, profile_deltas.read_bytes);
1329+
COUNTER_UPDATE(_file_read_calls_counter, profile_deltas.read_calls);
1330+
COUNTER_UPDATE(_file_read_time_counter, profile_deltas.read_time_ns);
11981331
const auto read_time = cast_set<int64_t>(_file_reader_stats->read_time_ns);
11991332
DORIS_CHECK(read_time >= _reported_io_read_time);
12001333
// Some transports (for example Arrow Flight) record directly into IO, while filesystem

0 commit comments

Comments
 (0)