2424#include < map>
2525#include < memory>
2626#include < optional>
27- #include < sstream>
2827#include < string>
2928#include < utility>
3029
@@ -227,9 +226,7 @@ Status rewrite_slot_refs_to_global_index(
227226#ifdef BE_TEST
228227FileScannerV2::FileScannerV2 (RuntimeState* state, RuntimeProfile* profile,
229228 std::unique_ptr<format::TableReader> table_reader)
230- : Scanner(state, profile),
231- _table_reader(std::move(table_reader)),
232- _scanner_profile(profile) {}
229+ : Scanner(state, profile), _table_reader(std::move(table_reader)) {}
233230
234231Status FileScannerV2::TEST_validate_scan_range (const TFileScanRangeParams& params,
235232 const TFileRangeDesc& range) {
@@ -329,9 +326,7 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat
329326
330327Status FileScannerV2::init (RuntimeState* state, const VExprContextSPtrs& conjuncts) {
331328 RETURN_IF_ERROR (Scanner::init (state, conjuncts));
332- _initialize_scanner_residual_conjuncts ();
333329 auto * profile = _local_state->scanner_profile ();
334- _scanner_profile = profile;
335330 const auto hierarchy = file_scan_profile::ensure_hierarchy (profile);
336331 _scanner_total_timer = hierarchy.scanner ;
337332 _io_timer = hierarchy.io ;
@@ -365,11 +360,6 @@ Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjunc
365360 profile, " AdaptiveBatchActualBytes" , TUnit::BYTES , file_scan_profile::SCANNER , 1 );
366361 _adaptive_batch_probe_count_counter = ADD_CHILD_COUNTER_WITH_LEVEL (
367362 profile, " AdaptiveBatchProbeCount" , TUnit::UNIT , file_scan_profile::SCANNER , 1 );
368- _scanner_residual_filter_timer = ADD_CHILD_TIMER_WITH_LEVEL (
369- profile, " ScannerResidualFilterTime" , file_scan_profile::SCANNER , 1 );
370- _scanner_residual_rows_filtered_counter = ADD_CHILD_COUNTER_WITH_LEVEL (
371- profile, " ScannerResidualRowsFiltered" , TUnit::UNIT , file_scan_profile::SCANNER , 1 );
372- _refresh_scanner_residual_profile ();
373363 SCOPED_TIMER (_scanner_total_timer);
374364 SCOPED_TIMER (_init_timer);
375365 _file_cache_statistics = std::make_unique<io::FileCacheStatistics>();
@@ -411,7 +401,6 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e
411401 SCOPED_TIMER (_get_block_timer);
412402 while (true ) {
413403 RETURN_IF_CANCELLED (state);
414- RETURN_IF_ERROR (_sync_table_reader_conjuncts ());
415404 if (!_has_prepared_split) {
416405 RETURN_IF_ERROR (_prepare_next_split (eof));
417406 if (*eof) {
@@ -461,33 +450,18 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e
461450}
462451
463452Status FileScannerV2::_filter_output_block (Block* block) {
464- if (_scanner_residual_conjuncts.empty () || block->rows () == 0 ) {
465- return Status::OK ();
466- }
467- SCOPED_TIMER (_scanner_residual_filter_timer);
468- const size_t rows_before_filter = block->rows ();
469- auto status = VExprContext::filter_block (_scanner_residual_conjuncts, block, block->columns ());
470- if (!status.ok () && _params != nullptr &&
471- _get_current_format_type () == TFileFormatType::FORMAT_ORC ) {
472- status.prepend (" Orc row reader nextBatch failed. reason = " );
473- }
474- RETURN_IF_ERROR (status);
475- const int64_t filtered_rows = cast_set<int64_t >(rows_before_filter - block->rows ());
476- _counter.num_rows_unselected += filtered_rows;
477- if (_scanner_residual_rows_filtered_counter != nullptr ) {
478- COUNTER_UPDATE (_scanner_residual_rows_filtered_counter, filtered_rows);
479- }
480- return Status::OK ();
481- }
482-
483- size_t FileScannerV2::_last_block_rows_read (const Block& block) const {
484- const auto & stats = _table_reader->last_materialized_block_stats ();
485- return stats.has_materialized_input ? stats.rows : block.rows ();
453+ return _contextualize_output_filter_status (Scanner::_filter_output_block (block),
454+ _get_current_format_type ());
486455}
487456
488- size_t FileScannerV2::_last_block_bytes_read (const Block& block) const {
489- const auto & stats = _table_reader->last_materialized_block_stats ();
490- return stats.has_materialized_input ? stats.allocated_bytes : block.allocated_bytes ();
457+ Status FileScannerV2::_contextualize_output_filter_status (Status status,
458+ TFileFormatType::type format_type) {
459+ if (!status.ok () && format_type == TFileFormatType::FORMAT_ORC ) {
460+ // Error-preserving expressions cannot be reordered into the ORC reader and therefore run
461+ // at the scanner boundary; keep their error context identical to ORC callback failures.
462+ status.prepend (" Orc row reader nextBatch failed. reason = " );
463+ }
464+ return status;
491465}
492466
493467Status FileScannerV2::_prepare_next_split (bool * eos) {
@@ -573,7 +547,6 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) {
573547 RETURN_IF_ERROR (_table_reader->init ({
574548 .projected_columns = _projected_columns,
575549 .conjuncts = std::move (table_conjuncts),
576- .table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count,
577550 .format = file_format,
578551 .scan_params = const_cast <TFileScanRangeParams*>(_params),
579552 .io_ctx = _io_ctx,
@@ -584,9 +557,6 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) {
584557 .push_down_count_columns = std::move (push_down_count_columns),
585558 .condition_cache_digest = _local_state->get_condition_cache_digest (),
586559 }));
587- _table_reader_applied_rf_num = _applied_rf_num;
588- // RFs collected before TableReader initialization are already present in the full snapshot.
589- _late_arrival_rf_conjuncts.clear ();
590560 return Status::OK ();
591561}
592562
@@ -631,12 +601,15 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range,
631601 std::map<std::string, Field> partition_values) {
632602 format::FileFormat current_split_format;
633603 RETURN_IF_ERROR (_to_file_format (get_range_format_type (*_params, range), ¤t_split_format));
604+ VExprContextSPtrs conjuncts;
605+ RETURN_IF_ERROR (_build_table_conjuncts (&conjuncts));
634606 VExprContextSPtrs partition_prune_conjuncts;
635607 if (_state->query_options ().enable_runtime_filter_partition_prune ) {
636608 RETURN_IF_ERROR (_build_table_conjuncts (&partition_prune_conjuncts));
637609 }
638610 RETURN_IF_ERROR (_table_reader->prepare_split ({
639611 .partition_values = std::move (partition_values),
612+ .conjuncts = std::move (conjuncts),
640613 .partition_prune_conjuncts = std::move (partition_prune_conjuncts),
641614 // A metadata COUNT split may span scheduler turns. Do not enter that irreversible
642615 // synthetic-row path while a runtime filter can still arrive between batches.
@@ -826,15 +799,10 @@ format::ColumnDefinition FileScannerV2::_build_table_column(const SlotDescriptor
826799}
827800
828801Status FileScannerV2::_build_table_conjuncts (VExprContextSPtrs* conjuncts) const {
829- return _build_table_conjuncts (_conjuncts, conjuncts);
830- }
831-
832- Status FileScannerV2::_build_table_conjuncts (const VExprContextSPtrs& source,
833- VExprContextSPtrs* conjuncts) const {
834802 DORIS_CHECK (conjuncts != nullptr );
835803 conjuncts->clear ();
836- conjuncts->reserve (source .size ());
837- for (const auto & conjunct : source ) {
804+ conjuncts->reserve (_conjuncts .size ());
805+ for (const auto & conjunct : _conjuncts ) {
838806 VExprSPtr root;
839807 RETURN_IF_ERROR (format::clone_table_expr_tree (conjunct->root (), &root));
840808 RETURN_IF_ERROR (rewrite_slot_refs_to_global_index (&root, _slot_id_to_global_index));
@@ -843,68 +811,6 @@ Status FileScannerV2::_build_table_conjuncts(const VExprContextSPtrs& source,
843811 return Status::OK ();
844812}
845813
846- size_t FileScannerV2::_safe_conjunct_prefix_size (const VExprContextSPtrs& conjuncts) {
847- for (size_t conjunct_index = 0 ; conjunct_index < conjuncts.size (); ++conjunct_index) {
848- if (!format::TableReader::is_safe_to_pre_execute (conjuncts[conjunct_index])) {
849- return conjunct_index;
850- }
851- }
852- return conjuncts.size ();
853- }
854-
855- void FileScannerV2::_initialize_scanner_residual_conjuncts () {
856- _table_reader_owned_conjunct_count = _safe_conjunct_prefix_size (_conjuncts);
857- // Preserve the entire suffix, not only the unsafe expression. Otherwise a later safe
858- // predicate could run below Scanner before a stateful/error-preserving ordering barrier.
859- _scanner_residual_conjuncts.assign (
860- _conjuncts.begin () + cast_set<ptrdiff_t >(_table_reader_owned_conjunct_count),
861- _conjuncts.end ());
862- _refresh_scanner_residual_profile ();
863- }
864-
865- void FileScannerV2::_refresh_scanner_residual_profile () {
866- if (_scanner_profile == nullptr || _scanner_residual_conjuncts.empty ()) {
867- return ;
868- }
869- std::ostringstream predicates;
870- predicates << " [" ;
871- for (size_t conjunct_index = 0 ; conjunct_index < _scanner_residual_conjuncts.size ();
872- ++conjunct_index) {
873- if (conjunct_index > 0 ) {
874- predicates << " , " ;
875- }
876- predicates << _scanner_residual_conjuncts[conjunct_index]->root ()->debug_string ();
877- }
878- predicates << " ]" ;
879- _scanner_profile->add_info_string (" ScannerResidualPredicates" , predicates.str ());
880- }
881-
882- Status FileScannerV2::_sync_table_reader_conjuncts () {
883- if (_table_reader == nullptr ) {
884- return Status::OK ();
885- }
886- if (_table_reader_applied_rf_num == _applied_rf_num) {
887- return Status::OK ();
888- }
889- VExprContextSPtrs appended;
890- RETURN_IF_ERROR (_build_table_conjuncts (_late_arrival_rf_conjuncts, &appended));
891- const size_t owned_count = _scanner_residual_conjuncts.empty ()
892- ? _safe_conjunct_prefix_size (_late_arrival_rf_conjuncts)
893- : 0 ;
894- // Preserve existing expression state and append the identity-tracked RF delta. Cost sorting
895- // may move a late RF ahead of an old stateful predicate in the full scanner snapshot.
896- RETURN_IF_ERROR (_table_reader->append_conjuncts_with_ownership (appended, owned_count));
897- _table_reader_owned_conjunct_count += owned_count;
898- _scanner_residual_conjuncts.insert (
899- _scanner_residual_conjuncts.end (),
900- _late_arrival_rf_conjuncts.begin () + cast_set<ptrdiff_t >(owned_count),
901- _late_arrival_rf_conjuncts.end ());
902- _refresh_scanner_residual_profile ();
903- _late_arrival_rf_conjuncts.clear ();
904- _table_reader_applied_rf_num = _applied_rf_num;
905- return Status::OK ();
906- }
907-
908814TFileFormatType::type FileScannerV2::_get_current_format_type () const {
909815 return get_range_format_type (*_params, _current_range);
910816}
@@ -1028,19 +934,17 @@ void FileScannerV2::_update_adaptive_batch_size(const Block& block) {
1028934 if (!_should_run_adaptive_batch_size ()) {
1029935 return ;
1030936 }
1031- const auto & stats = _table_reader->last_materialized_block_stats ();
1032- const size_t rows = stats.has_materialized_input ? stats.rows : block.rows ();
1033- const size_t bytes = stats.has_materialized_input ? stats.bytes : block.bytes ();
1034- COUNTER_SET (_adaptive_batch_actual_bytes_counter, static_cast <int64_t >(bytes));
1035- if (rows == 0 ) {
937+ COUNTER_SET (_adaptive_batch_actual_bytes_counter, static_cast <int64_t >(block.bytes ()));
938+ if (block.rows () == 0 ) {
1036939 return ;
1037940 }
1038- // Residual predicates run after wide table columns are materialized. Learn from that pre-filter
1039- // shape so selective predicates cannot make the next reader batch dangerously large.
941+ // The sample is taken after TableReader has finalized file-local columns to table columns.
942+ // This matches the memory shape seen by upstream operators and catches very wide nested
943+ // columns, such as map/string payloads, after the first probe batch.
1040944 if (!_block_size_predictor->has_history ()) {
1041945 COUNTER_UPDATE (_adaptive_batch_probe_count_counter, 1 );
1042946 }
1043- _block_size_predictor->update (rows, bytes );
947+ _block_size_predictor->update (block );
1044948}
1045949
1046950Status FileScannerV2::close (RuntimeState* state) {
@@ -1241,8 +1145,9 @@ void FileScannerV2::_report_file_reader_predicate_filtered_rows() {
12411145 const int64_t filtered_rows = _io_ctx != nullptr ? _io_ctx->predicate_filtered_rows : 0 ;
12421146 const int64_t filtered_delta = filtered_rows - _reported_predicate_filtered_rows;
12431147 if (filtered_delta > 0 ) {
1244- // FileReader and TableReader both report their owned predicate rows through the shared IO
1245- // context. Preserve scanner-level load statistics without re-evaluating either predicate.
1148+ // File readers can evaluate localized conjuncts before a block reaches Scanner. Count
1149+ // those rows as scanner-level unselected rows so load statistics stay identical no matter
1150+ // whether a predicate is pushed down or evaluated by Scanner::_filter_output_block().
12461151 _counter.num_rows_unselected += filtered_delta;
12471152 _reported_predicate_filtered_rows = filtered_rows;
12481153 }
0 commit comments