@@ -281,6 +281,27 @@ Status adapt_runtime_filter_for_table_reader(VExprSPtr* expr) {
281281
282282} // namespace
283283
284+ int64_t FileScannerV2::_cumulative_profile_delta (int64_t current, int64_t * reported) {
285+ DORIS_CHECK (reported != nullptr );
286+ DORIS_CHECK (current >= *reported);
287+ const int64_t delta = current - *reported;
288+ *reported = current;
289+ return delta;
290+ }
291+
292+ FileScannerV2::FileReaderProfileDeltas FileScannerV2::_collect_file_reader_profile_deltas (
293+ const io::FileReaderStats& stats, int64_t * reported_bytes, int64_t * reported_calls,
294+ int64_t * reported_time) {
295+ return {
296+ .read_bytes =
297+ _cumulative_profile_delta (cast_set<int64_t >(stats.read_bytes ), reported_bytes),
298+ .read_calls =
299+ _cumulative_profile_delta (cast_set<int64_t >(stats.read_calls ), reported_calls),
300+ .read_time_ns =
301+ _cumulative_profile_delta (cast_set<int64_t >(stats.read_time_ns ), reported_time),
302+ };
303+ }
304+
284305#ifdef BE_TEST
285306FileScannerV2::FileScannerV2 (RuntimeState* state, RuntimeProfile* profile,
286307 std::unique_ptr<format::TableReader> table_reader)
@@ -380,7 +401,8 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat
380401 const std::unordered_map<std::string, int >* colname_to_slot_id)
381402 : Scanner(state, local_state, limit, profile),
382403 _split_source (std::move(split_source)),
383- _kv_cache(kv_cache) {
404+ _kv_cache(kv_cache),
405+ _file_context_registry(&local_state->_file_context_registry) {
384406 (void )colname_to_slot_id;
385407 if (state->get_query_ctx () != nullptr &&
386408 state->get_query_ctx ()->file_scan_range_params_map .count (local_state->parent_id ()) > 0 ) {
@@ -448,16 +470,50 @@ Status FileScannerV2::_open_impl(RuntimeState* state) {
448470 DORIS_CHECK (_table_reader != nullptr );
449471 RETURN_IF_ERROR (_init_expr_ctxes ());
450472 RETURN_IF_ERROR (_init_table_reader (_current_range));
473+ // Refine the first source split before yielding the scanner worker. Other scanners may be
474+ // waiting for its row-group children, so deferring publication until a later get_block()
475+ // turn could let those waiters occupy the scan thread pool ahead of the producer.
476+ bool eos = false ;
477+ RETURN_IF_ERROR (_prepare_next_split (&eos));
451478 }
452479 return Status::OK ();
453480}
454481
455482Status FileScannerV2::_get_next_scan_range (bool * has_next) {
456483 SCOPED_TIMER (_get_next_range_timer);
457484 DORIS_CHECK (has_next != nullptr );
458- RETURN_IF_ERROR (_split_source->get_next (has_next, &_current_range ));
485+ RETURN_IF_ERROR (_split_source->get_next_split (has_next, &_current_split ));
459486 if (*has_next) {
487+ _current_range = _current_split.materialize_range ();
460488 RETURN_IF_ERROR (_validate_scan_range (*_params, _current_range));
489+ } else {
490+ _current_split = {};
491+ _current_range = {};
492+ }
493+ return Status::OK ();
494+ }
495+
496+ Status FileScannerV2::_retire_current_source_split (std::vector<FileScanSplit> generated_splits) {
497+ if (!_current_split.is_source_split ) {
498+ return Status::OK ();
499+ }
500+ const bool transferred_progress = !generated_splits.empty ();
501+ RETURN_IF_ERROR (
502+ _split_source->finish_source_split (_current_split, std::move (generated_splits)));
503+ _current_split.is_source_split = false ;
504+ if (transferred_progress) {
505+ _current_split.source_progress .reset ();
506+ }
507+ return Status::OK ();
508+ }
509+
510+ Status FileScannerV2::_complete_current_split () {
511+ RETURN_IF_ERROR (_retire_current_source_split ());
512+ if (_current_split.source_progress != nullptr ) {
513+ if (_current_split.source_progress ->complete_one ()) {
514+ _state->update_num_finished_scan_range (1 );
515+ }
516+ _current_split.source_progress .reset ();
461517 }
462518 return Status::OK ();
463519}
@@ -488,7 +544,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e
488544 if (_should_skip_not_found (status, config::ignore_not_found_file_in_external_table)) {
489545 RETURN_IF_ERROR (_table_reader->abort_split ());
490546 COUNTER_UPDATE (_not_found_file_counter, 1 );
491- _state-> update_num_finished_scan_range ( 1 );
547+ RETURN_IF_ERROR ( _complete_current_split () );
492548 _has_prepared_split = false ;
493549 block->clear_column_data (cast_set<int64_t >(_projected_columns.size ()));
494550 *eof = false ;
@@ -502,7 +558,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e
502558 // discard partial reader state, and let the loop fetch the next split.
503559 RETURN_IF_ERROR (_table_reader->abort_split ());
504560 COUNTER_UPDATE (_empty_file_counter, 1 );
505- _state-> update_num_finished_scan_range ( 1 );
561+ RETURN_IF_ERROR ( _complete_current_split () );
506562 _has_prepared_split = false ;
507563 block->clear_column_data (cast_set<int64_t >(_projected_columns.size ()));
508564 *eof = false ;
@@ -511,7 +567,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e
511567 RETURN_IF_ERROR (status);
512568 }
513569 if (*eof) {
514- _state-> update_num_finished_scan_range ( 1 );
570+ RETURN_IF_ERROR ( _complete_current_split () );
515571 _has_prepared_split = false ;
516572 *eof = false ;
517573 continue ;
@@ -552,6 +608,7 @@ Status FileScannerV2::_prepare_next_split(bool* eos) {
552608 }
553609 _first_scan_range = false ;
554610 if (!has_next || _should_stop) {
611+ RETURN_IF_ERROR (_complete_current_split ());
555612 *eos = true ;
556613 return Status::OK ();
557614 }
@@ -575,7 +632,7 @@ Status FileScannerV2::_prepare_next_split(bool* eos) {
575632 if (_should_skip_not_found (status, config::ignore_not_found_file_in_external_table)) {
576633 RETURN_IF_ERROR (_table_reader->abort_split ());
577634 COUNTER_UPDATE (_not_found_file_counter, 1 );
578- _state-> update_num_finished_scan_range ( 1 );
635+ RETURN_IF_ERROR ( _complete_current_split () );
579636 continue ;
580637 }
581638 if (_should_skip_empty (status, _should_stop || _io_ctx->should_stop )) {
@@ -585,14 +642,51 @@ Status FileScannerV2::_prepare_next_split(bool* eos) {
585642 // advance exactly one scan range and preserve later files in the same scan.
586643 RETURN_IF_ERROR (_table_reader->abort_split ());
587644 COUNTER_UPDATE (_empty_file_counter, 1 );
588- _state-> update_num_finished_scan_range ( 1 );
645+ RETURN_IF_ERROR ( _complete_current_split () );
589646 continue ;
590647 }
591648 RETURN_IF_ERROR (status);
592649 if (_table_reader->current_split_pruned ()) {
593- _state-> update_num_finished_scan_range ( 1 );
650+ RETURN_IF_ERROR ( _complete_current_split () );
594651 continue ;
595652 }
653+ if (_current_split.is_source_split && _can_refine_source_split (_current_range)) {
654+ std::vector<FileScanSplit> generated_splits;
655+ bool was_split = false ;
656+ const auto split_status = _table_reader->build_physical_splits (
657+ _current_split, &generated_splits, &was_split);
658+ const auto ignored_split_status = _classify_ignored_split_status (
659+ split_status, config::ignore_not_found_file_in_external_table,
660+ _should_stop || _io_ctx->should_stop );
661+ if (ignored_split_status == IgnoredSplitStatus::NOT_FOUND ) {
662+ RETURN_IF_ERROR (_table_reader->abort_split ());
663+ COUNTER_UPDATE (_not_found_file_counter, 1 );
664+ RETURN_IF_ERROR (_complete_current_split ());
665+ continue ;
666+ }
667+ if (ignored_split_status == IgnoredSplitStatus::EMPTY ) {
668+ RETURN_IF_ERROR (_table_reader->abort_split ());
669+ COUNTER_UPDATE (_empty_file_counter, 1 );
670+ RETURN_IF_ERROR (_complete_current_split ());
671+ continue ;
672+ }
673+ RETURN_IF_ERROR (split_status);
674+ if (was_split) {
675+ RETURN_IF_ERROR (_table_reader->abort_split ());
676+ const bool has_children = !generated_splits.empty ();
677+ RETURN_IF_ERROR (_retire_current_source_split (std::move (generated_splits)));
678+ if (!has_children) {
679+ RETURN_IF_ERROR (_complete_current_split ());
680+ }
681+ continue ;
682+ }
683+ }
684+ if (_current_split.is_source_split ) {
685+ // Non-Parquet and metadata-only source splits are already fully prepared and cannot
686+ // publish children. Release the source reservation before yielding the scanner worker
687+ // so waiters never depend on a later get_block() turn to observe completion.
688+ RETURN_IF_ERROR (_retire_current_source_split ());
689+ }
596690 COUNTER_UPDATE (_file_counter, 1 );
597691 _has_prepared_split = true ;
598692 _table_reader_rf_num = _applied_rf_num;
@@ -636,6 +730,7 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) {
636730 .push_down_agg_type = _local_state->get_push_down_agg_type (),
637731 .push_down_count_columns = std::move (push_down_count_columns),
638732 .condition_cache_digest = _local_state->get_condition_cache_digest (),
733+ .file_context_registry = _file_context_registry,
639734 }));
640735 return Status::OK ();
641736}
@@ -709,6 +804,8 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range,
709804 .cache = _kv_cache,
710805 .current_range = range,
711806 .current_split_format = current_split_format,
807+ .file_context = _current_split.file_context ,
808+ .format_split_id = _current_split.format_split_id ,
712809 .global_rowid_context = _create_global_rowid_context (range),
713810 }));
714811 return Status::OK ();
@@ -726,6 +823,27 @@ bool FileScannerV2::_should_skip_empty(const Status& status, bool stopped) {
726823 return !stopped && status.is <ErrorCode::END_OF_FILE >();
727824}
728825
826+ FileScannerV2::IgnoredSplitStatus FileScannerV2::_classify_ignored_split_status (
827+ const Status& status, bool ignore_not_found, bool stopped) {
828+ if (_should_skip_not_found (status, ignore_not_found)) {
829+ return IgnoredSplitStatus::NOT_FOUND ;
830+ }
831+ if (_should_skip_empty (status, stopped)) {
832+ return IgnoredSplitStatus::EMPTY ;
833+ }
834+ return IgnoredSplitStatus::NONE ;
835+ }
836+
837+ bool FileScannerV2::_can_refine_source_split (const TFileRangeDesc& range) {
838+ if (!range.__isset .table_format_params || !range.table_format_params .__isset .iceberg_params ) {
839+ return true ;
840+ }
841+ const auto & iceberg = range.table_format_params .iceberg_params ;
842+ // Iceberg delete readers build split-local delete state. Refining the data file would rebuild
843+ // that state for every row-group child, so keep the already prepared source reader instead.
844+ return !iceberg.__isset .delete_files || iceberg.delete_files .empty ();
845+ }
846+
729847bool FileScannerV2::_should_enable_file_meta_cache () const {
730848 return ExecEnv::GetInstance ()->file_meta_cache ()->enabled () &&
731849 _split_source->num_scan_ranges () < config::max_external_file_meta_cache_num / 3 ;
@@ -1053,16 +1171,24 @@ Status FileScannerV2::close(RuntimeState* state) {
10531171 if (_table_reader != nullptr ) {
10541172 const auto close_status = _table_reader->close ();
10551173 if (!close_status.ok ()) {
1174+ // A failed reader close remains retryable, but this scanner can no longer publish
1175+ // useful children. Retire its source reservation so peer scanners are not stranded.
1176+ const auto split_status = _complete_current_split ();
1177+ if (!split_status.ok ()) {
1178+ return split_status;
1179+ }
10561180 return close_status;
10571181 }
10581182 _report_condition_cache_profile ();
10591183 _table_reader.reset ();
10601184 }
1185+ RETURN_IF_ERROR (_complete_current_split ());
10611186 return Scanner::close (state);
10621187}
10631188
10641189void FileScannerV2::try_stop () {
10651190 Scanner::try_stop ();
1191+ _split_source->stop ();
10661192 if (_io_ctx) {
10671193 _io_ctx->should_stop = true ;
10681194 }
@@ -1073,7 +1199,6 @@ void FileScannerV2::update_realtime_counters() {
10731199 return ;
10741200 }
10751201 DORIS_CHECK (_file_cache_statistics != nullptr );
1076- const int64_t bytes_read = cast_set<int64_t >(_file_reader_stats->read_bytes );
10771202 auto * local_state = static_cast <FileScanLocalState*>(_local_state);
10781203 const auto file_type =
10791204 _current_range.__isset .file_type
@@ -1095,9 +1220,14 @@ void FileScannerV2::update_realtime_counters() {
10951220 _state->get_query_ctx ()->resource_ctx ()->io_context ()->update_scan_bytes_from_remote_storage (
10961221 deltas.scan_bytes_from_remote_storage );
10971222
1098- COUNTER_SET (_file_read_bytes_counter, bytes_read);
1099- COUNTER_SET (_file_read_calls_counter, cast_set<int64_t >(_file_reader_stats->read_calls ));
1100- COUNTER_SET (_file_read_time_counter, cast_set<int64_t >(_file_reader_stats->read_time_ns ));
1223+ // Scanner instances share these profile counters. Publish only scanner-private deltas so one
1224+ // scanner cannot erase or double count values already contributed by a sibling.
1225+ const auto profile_deltas = _collect_file_reader_profile_deltas (
1226+ *_file_reader_stats, &_reported_file_read_bytes, &_reported_file_read_calls,
1227+ &_reported_file_read_time);
1228+ COUNTER_UPDATE (_file_read_bytes_counter, profile_deltas.read_bytes );
1229+ COUNTER_UPDATE (_file_read_calls_counter, profile_deltas.read_calls );
1230+ COUNTER_UPDATE (_file_read_time_counter, profile_deltas.read_time_ns );
11011231
11021232 DorisMetrics::instance ()->query_scan_bytes ->increment (deltas.scan_bytes );
11031233 DorisMetrics::instance ()->query_scan_rows ->increment (deltas.scan_rows );
@@ -1192,9 +1322,12 @@ void FileScannerV2::_collect_profile_before_close() {
11921322 _reported_file_cache_statistics = *_file_cache_statistics;
11931323 }
11941324 if (_file_reader_stats != nullptr ) {
1195- COUNTER_SET (_file_read_bytes_counter, cast_set<int64_t >(_file_reader_stats->read_bytes ));
1196- COUNTER_SET (_file_read_calls_counter, cast_set<int64_t >(_file_reader_stats->read_calls ));
1197- COUNTER_SET (_file_read_time_counter, cast_set<int64_t >(_file_reader_stats->read_time_ns ));
1325+ const auto profile_deltas = _collect_file_reader_profile_deltas (
1326+ *_file_reader_stats, &_reported_file_read_bytes, &_reported_file_read_calls,
1327+ &_reported_file_read_time);
1328+ COUNTER_UPDATE (_file_read_bytes_counter, profile_deltas.read_bytes );
1329+ COUNTER_UPDATE (_file_read_calls_counter, profile_deltas.read_calls );
1330+ COUNTER_UPDATE (_file_read_time_counter, profile_deltas.read_time_ns );
11981331 const auto read_time = cast_set<int64_t >(_file_reader_stats->read_time_ns );
11991332 DORIS_CHECK (read_time >= _reported_io_read_time);
12001333 // Some transports (for example Arrow Flight) record directly into IO, while filesystem
0 commit comments