2424#include < map>
2525#include < memory>
2626#include < optional>
27+ #include < sstream>
2728#include < string>
2829#include < utility>
2930
@@ -226,7 +227,9 @@ Status rewrite_slot_refs_to_global_index(
226227#ifdef BE_TEST
227228FileScannerV2::FileScannerV2 (RuntimeState* state, RuntimeProfile* profile,
228229 std::unique_ptr<format::TableReader> table_reader)
229- : Scanner(state, profile), _table_reader(std::move(table_reader)) {}
230+ : Scanner(state, profile),
231+ _table_reader(std::move(table_reader)),
232+ _scanner_profile(profile) {}
230233
231234Status FileScannerV2::TEST_validate_scan_range (const TFileScanRangeParams& params,
232235 const TFileRangeDesc& range) {
@@ -326,7 +329,9 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat
326329
327330Status FileScannerV2::init (RuntimeState* state, const VExprContextSPtrs& conjuncts) {
328331 RETURN_IF_ERROR (Scanner::init (state, conjuncts));
332+ _initialize_scanner_residual_conjuncts ();
329333 auto * profile = _local_state->scanner_profile ();
334+ _scanner_profile = profile;
330335 const auto hierarchy = file_scan_profile::ensure_hierarchy (profile);
331336 _scanner_total_timer = hierarchy.scanner ;
332337 _io_timer = hierarchy.io ;
@@ -360,6 +365,11 @@ Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjunc
360365 profile, " AdaptiveBatchActualBytes" , TUnit::BYTES , file_scan_profile::SCANNER , 1 );
361366 _adaptive_batch_probe_count_counter = ADD_CHILD_COUNTER_WITH_LEVEL (
362367 profile, " AdaptiveBatchProbeCount" , TUnit::UNIT , file_scan_profile::SCANNER , 1 );
368+ _scanner_residual_filter_timer = ADD_CHILD_TIMER_WITH_LEVEL (
369+ profile, " ScannerResidualFilterTime" , file_scan_profile::SCANNER , 1 );
370+ _scanner_residual_rows_filtered_counter = ADD_CHILD_COUNTER_WITH_LEVEL (
371+ profile, " ScannerResidualRowsFiltered" , TUnit::UNIT , file_scan_profile::SCANNER , 1 );
372+ _refresh_scanner_residual_profile ();
363373 SCOPED_TIMER (_scanner_total_timer);
364374 SCOPED_TIMER (_init_timer);
365375 _file_cache_statistics = std::make_unique<io::FileCacheStatistics>();
@@ -401,6 +411,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e
401411 SCOPED_TIMER (_get_block_timer);
402412 while (true ) {
403413 RETURN_IF_CANCELLED (state);
414+ RETURN_IF_ERROR (_sync_table_reader_conjuncts ());
404415 if (!_has_prepared_split) {
405416 RETURN_IF_ERROR (_prepare_next_split (eof));
406417 if (*eof) {
@@ -450,18 +461,33 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e
450461}
451462
452463Status FileScannerV2::_filter_output_block (Block* block) {
453- return _contextualize_output_filter_status (Scanner::_filter_output_block (block),
454- _get_current_format_type ());
455- }
456-
457- Status FileScannerV2::_contextualize_output_filter_status (Status status,
458- TFileFormatType::type format_type) {
459- if (!status.ok () && format_type == TFileFormatType::FORMAT_ORC ) {
460- // Error-preserving expressions cannot be reordered into the ORC reader and therefore run
461- // at the scanner boundary; keep their error context identical to ORC callback failures.
464+ if (_scanner_residual_conjuncts.empty () || block->rows () == 0 ) {
465+ return Status::OK ();
466+ }
467+ SCOPED_TIMER (_scanner_residual_filter_timer);
468+ const size_t rows_before_filter = block->rows ();
469+ auto status = VExprContext::filter_block (_scanner_residual_conjuncts, block, block->columns ());
470+ if (!status.ok () && _params != nullptr &&
471+ _get_current_format_type () == TFileFormatType::FORMAT_ORC ) {
462472 status.prepend (" Orc row reader nextBatch failed. reason = " );
463473 }
464- return status;
474+ RETURN_IF_ERROR (status);
475+ const int64_t filtered_rows = cast_set<int64_t >(rows_before_filter - block->rows ());
476+ _counter.num_rows_unselected += filtered_rows;
477+ if (_scanner_residual_rows_filtered_counter != nullptr ) {
478+ COUNTER_UPDATE (_scanner_residual_rows_filtered_counter, filtered_rows);
479+ }
480+ return Status::OK ();
481+ }
482+
483+ size_t FileScannerV2::_last_block_rows_read (const Block& block) const {
484+ const auto & stats = _table_reader->last_materialized_block_stats ();
485+ return stats.has_materialized_input ? stats.rows : block.rows ();
486+ }
487+
488+ size_t FileScannerV2::_last_block_bytes_read (const Block& block) const {
489+ const auto & stats = _table_reader->last_materialized_block_stats ();
490+ return stats.has_materialized_input ? stats.allocated_bytes : block.allocated_bytes ();
465491}
466492
467493Status FileScannerV2::_prepare_next_split (bool * eos) {
@@ -547,6 +573,7 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) {
547573 RETURN_IF_ERROR (_table_reader->init ({
548574 .projected_columns = _projected_columns,
549575 .conjuncts = std::move (table_conjuncts),
576+ .table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count,
550577 .format = file_format,
551578 .scan_params = const_cast <TFileScanRangeParams*>(_params),
552579 .io_ctx = _io_ctx,
@@ -557,6 +584,9 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) {
557584 .push_down_count_columns = std::move (push_down_count_columns),
558585 .condition_cache_digest = _local_state->get_condition_cache_digest (),
559586 }));
587+ _table_reader_applied_rf_num = _applied_rf_num;
588+ // RFs collected before TableReader initialization are already present in the full snapshot.
589+ _late_arrival_rf_conjuncts.clear ();
560590 return Status::OK ();
561591}
562592
@@ -601,15 +631,12 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range,
601631 std::map<std::string, Field> partition_values) {
602632 format::FileFormat current_split_format;
603633 RETURN_IF_ERROR (_to_file_format (get_range_format_type (*_params, range), ¤t_split_format));
604- VExprContextSPtrs conjuncts;
605- RETURN_IF_ERROR (_build_table_conjuncts (&conjuncts));
606634 VExprContextSPtrs partition_prune_conjuncts;
607635 if (_state->query_options ().enable_runtime_filter_partition_prune ) {
608636 RETURN_IF_ERROR (_build_table_conjuncts (&partition_prune_conjuncts));
609637 }
610638 RETURN_IF_ERROR (_table_reader->prepare_split ({
611639 .partition_values = std::move (partition_values),
612- .conjuncts = std::move (conjuncts),
613640 .partition_prune_conjuncts = std::move (partition_prune_conjuncts),
614641 // A metadata COUNT split may span scheduler turns. Do not enter that irreversible
615642 // synthetic-row path while a runtime filter can still arrive between batches.
@@ -799,10 +826,15 @@ format::ColumnDefinition FileScannerV2::_build_table_column(const SlotDescriptor
799826}
800827
801828Status FileScannerV2::_build_table_conjuncts (VExprContextSPtrs* conjuncts) const {
829+ return _build_table_conjuncts (_conjuncts, conjuncts);
830+ }
831+
832+ Status FileScannerV2::_build_table_conjuncts (const VExprContextSPtrs& source,
833+ VExprContextSPtrs* conjuncts) const {
802834 DORIS_CHECK (conjuncts != nullptr );
803835 conjuncts->clear ();
804- conjuncts->reserve (_conjuncts .size ());
805- for (const auto & conjunct : _conjuncts ) {
836+ conjuncts->reserve (source .size ());
837+ for (const auto & conjunct : source ) {
806838 VExprSPtr root;
807839 RETURN_IF_ERROR (format::clone_table_expr_tree (conjunct->root (), &root));
808840 RETURN_IF_ERROR (rewrite_slot_refs_to_global_index (&root, _slot_id_to_global_index));
@@ -811,6 +843,68 @@ Status FileScannerV2::_build_table_conjuncts(VExprContextSPtrs* conjuncts) const
811843 return Status::OK ();
812844}
813845
846+ size_t FileScannerV2::_safe_conjunct_prefix_size (const VExprContextSPtrs& conjuncts) {
847+ for (size_t conjunct_index = 0 ; conjunct_index < conjuncts.size (); ++conjunct_index) {
848+ if (!format::TableReader::is_safe_to_pre_execute (conjuncts[conjunct_index])) {
849+ return conjunct_index;
850+ }
851+ }
852+ return conjuncts.size ();
853+ }
854+
855+ void FileScannerV2::_initialize_scanner_residual_conjuncts () {
856+ _table_reader_owned_conjunct_count = _safe_conjunct_prefix_size (_conjuncts);
857+ // Preserve the entire suffix, not only the unsafe expression. Otherwise a later safe
858+ // predicate could run below Scanner before a stateful/error-preserving ordering barrier.
859+ _scanner_residual_conjuncts.assign (
860+ _conjuncts.begin () + cast_set<ptrdiff_t >(_table_reader_owned_conjunct_count),
861+ _conjuncts.end ());
862+ _refresh_scanner_residual_profile ();
863+ }
864+
865+ void FileScannerV2::_refresh_scanner_residual_profile () {
866+ if (_scanner_profile == nullptr || _scanner_residual_conjuncts.empty ()) {
867+ return ;
868+ }
869+ std::ostringstream predicates;
870+ predicates << " [" ;
871+ for (size_t conjunct_index = 0 ; conjunct_index < _scanner_residual_conjuncts.size ();
872+ ++conjunct_index) {
873+ if (conjunct_index > 0 ) {
874+ predicates << " , " ;
875+ }
876+ predicates << _scanner_residual_conjuncts[conjunct_index]->root ()->debug_string ();
877+ }
878+ predicates << " ]" ;
879+ _scanner_profile->add_info_string (" ScannerResidualPredicates" , predicates.str ());
880+ }
881+
882+ Status FileScannerV2::_sync_table_reader_conjuncts () {
883+ if (_table_reader == nullptr ) {
884+ return Status::OK ();
885+ }
886+ if (_table_reader_applied_rf_num == _applied_rf_num) {
887+ return Status::OK ();
888+ }
889+ VExprContextSPtrs appended;
890+ RETURN_IF_ERROR (_build_table_conjuncts (_late_arrival_rf_conjuncts, &appended));
891+ const size_t owned_count = _scanner_residual_conjuncts.empty ()
892+ ? _safe_conjunct_prefix_size (_late_arrival_rf_conjuncts)
893+ : 0 ;
894+ // Preserve existing expression state and append the identity-tracked RF delta. Cost sorting
895+ // may move a late RF ahead of an old stateful predicate in the full scanner snapshot.
896+ RETURN_IF_ERROR (_table_reader->append_conjuncts_with_ownership (appended, owned_count));
897+ _table_reader_owned_conjunct_count += owned_count;
898+ _scanner_residual_conjuncts.insert (
899+ _scanner_residual_conjuncts.end (),
900+ _late_arrival_rf_conjuncts.begin () + cast_set<ptrdiff_t >(owned_count),
901+ _late_arrival_rf_conjuncts.end ());
902+ _refresh_scanner_residual_profile ();
903+ _late_arrival_rf_conjuncts.clear ();
904+ _table_reader_applied_rf_num = _applied_rf_num;
905+ return Status::OK ();
906+ }
907+
814908TFileFormatType::type FileScannerV2::_get_current_format_type () const {
815909 return get_range_format_type (*_params, _current_range);
816910}
@@ -934,17 +1028,19 @@ void FileScannerV2::_update_adaptive_batch_size(const Block& block) {
9341028 if (!_should_run_adaptive_batch_size ()) {
9351029 return ;
9361030 }
937- COUNTER_SET (_adaptive_batch_actual_bytes_counter, static_cast <int64_t >(block.bytes ()));
938- if (block.rows () == 0 ) {
1031+ const auto & stats = _table_reader->last_materialized_block_stats ();
1032+ const size_t rows = stats.has_materialized_input ? stats.rows : block.rows ();
1033+ const size_t bytes = stats.has_materialized_input ? stats.bytes : block.bytes ();
1034+ COUNTER_SET (_adaptive_batch_actual_bytes_counter, static_cast <int64_t >(bytes));
1035+ if (rows == 0 ) {
9391036 return ;
9401037 }
941- // The sample is taken after TableReader has finalized file-local columns to table columns.
942- // This matches the memory shape seen by upstream operators and catches very wide nested
943- // columns, such as map/string payloads, after the first probe batch.
1038+ // Residual predicates run after wide table columns are materialized. Learn from that pre-filter
1039+ // shape so selective predicates cannot make the next reader batch dangerously large.
9441040 if (!_block_size_predictor->has_history ()) {
9451041 COUNTER_UPDATE (_adaptive_batch_probe_count_counter, 1 );
9461042 }
947- _block_size_predictor->update (block );
1043+ _block_size_predictor->update (rows, bytes );
9481044}
9491045
9501046Status FileScannerV2::close (RuntimeState* state) {
@@ -1145,9 +1241,8 @@ void FileScannerV2::_report_file_reader_predicate_filtered_rows() {
11451241 const int64_t filtered_rows = _io_ctx != nullptr ? _io_ctx->predicate_filtered_rows : 0 ;
11461242 const int64_t filtered_delta = filtered_rows - _reported_predicate_filtered_rows;
11471243 if (filtered_delta > 0 ) {
1148- // File readers can evaluate localized conjuncts before a block reaches Scanner. Count
1149- // those rows as scanner-level unselected rows so load statistics stay identical no matter
1150- // whether a predicate is pushed down or evaluated by Scanner::_filter_output_block().
1244+ // FileReader and TableReader both report their owned predicate rows through the shared IO
1245+ // context. Preserve scanner-level load statistics without re-evaluating either predicate.
11511246 _counter.num_rows_unselected += filtered_delta;
11521247 _reported_predicate_filtered_rows = filtered_rows;
11531248 }
0 commit comments