Skip to content

Commit 27cb451

Browse files
authored
[improvement](file) Execute split residual predicates in TableReader (apache#65925)
## What changed - Add a split-local localization result so `TableReader` knows which table predicates are enforced exactly by the current `FileReader`. - Evaluate only unlocalized predicates at the end of `TableReader::finalize_chunk`, after schema evolution, defaults, partition values, and virtual columns are materialized. - Make `FileScannerV2` skip row-level conjunct evaluation; it keeps scheduling and accounting responsibilities only. - Preserve predicate order by keeping unsafe/stateful predicates and every later predicate on the table-materialization path. - Apply the same table-level residual filtering to the Iceberg position-delete system-table reader. ## Why `FileScannerV2` previously reevaluated every original conjunct after `FileReader` had already enforced localized predicates. This duplicated expression work and made predicate execution ownership unclear. Localization also depends on each split's physical schema, so ownership cannot be cached at scanner or table-reader lifetime scope. The new contract recomputes ownership for every split: localized predicates belong to `FileReader`, while all remaining predicates belong to `TableReader` after final materialization. ## Validation - ASAN BE UT build: `cmake --build be/ut_build_ASAN --target doris_be_test -j128` - 287 focused TableReader/ColumnMapper/Iceberg/FileScannerV2 tests passed. - 41 JNI/Hudi/Paimon reader tests passed. - `build-support/check-format.sh` passed with clang-format 16.0.6. - clang-tidy was attempted but could not analyze the tree because of existing toolchain/baseline errors (`stddef.h` not found and an unmatched `NOLINTEND` in `core/types.h`).
1 parent bd16e5d commit 27cb451

27 files changed

Lines changed: 1424 additions & 142 deletions

‎be/src/exec/operator/scan_operator.cpp‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,13 +71,19 @@ bool ScanLocalState<Derived>::should_run_serial() const {
7171
return _parent->cast<typename Derived::Parent>()._should_run_serial;
7272
}
7373

74-
Status ScanLocalStateBase::update_late_arrival_runtime_filter(RuntimeState* state,
75-
int& arrived_rf_num) {
74+
Status ScanLocalStateBase::update_late_arrival_runtime_filter(
75+
RuntimeState* state, int applied_rf_num, int& arrived_rf_num,
76+
VExprContextSPtrs& arrived_conjuncts) {
7677
// Lock needed because _conjuncts can be accessed concurrently by multiple scanner threads
7778
LockGuard lock(_conjuncts_lock);
79+
arrived_conjuncts.clear();
7880
size_t conjuncts_before = _conjuncts.size();
7981
RETURN_IF_ERROR(_helper.try_append_late_arrival_runtime_filter(state, _parent->row_descriptor(),
8082
arrived_rf_num, _conjuncts));
83+
if (_conjuncts.size() > conjuncts_before) {
84+
VExprContextSPtrs appended(_conjuncts.begin() + conjuncts_before, _conjuncts.end());
85+
_late_arrival_conjunct_batches.emplace_back(arrived_rf_num, std::move(appended));
86+
}
8187
if (state->enable_adjust_conjunct_order_by_cost()) {
8288
std::ranges::stable_sort(_conjuncts, [](const auto& a, const auto& b) {
8389
return a->execute_cost() < b->execute_cost();
@@ -91,6 +97,16 @@ Status ScanLocalStateBase::update_late_arrival_runtime_filter(RuntimeState* stat
9197
if (_conjuncts.size() > conjuncts_before) {
9298
RETURN_IF_ERROR(_on_runtime_filter_update());
9399
}
100+
for (const auto& [batch_arrived_rf_num, batch] : _late_arrival_conjunct_batches) {
101+
if (batch_arrived_rf_num <= applied_rf_num) {
102+
continue;
103+
}
104+
for (const auto& conjunct : batch) {
105+
VExprContextSPtr cloned;
106+
RETURN_IF_ERROR(conjunct->clone(state, cloned));
107+
arrived_conjuncts.push_back(std::move(cloned));
108+
}
109+
}
94110
return Status::OK();
95111
}
96112

‎be/src/exec/operator/scan_operator.h‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
#include <optional>
2222
#include <set>
2323
#include <string>
24+
#include <utility>
25+
#include <vector>
2426

2527
#include "common/status.h"
2628
#include "common/thread_safety_annotations.h"
@@ -96,7 +98,9 @@ class ScanLocalStateBase : public PipelineXLocalState<> {
9698

9799
uint64_t get_condition_cache_digest() const { return _condition_cache_digest; }
98100

99-
Status update_late_arrival_runtime_filter(RuntimeState* state, int& arrived_rf_num);
101+
Status update_late_arrival_runtime_filter(RuntimeState* state, int applied_rf_num,
102+
int& arrived_rf_num,
103+
VExprContextSPtrs& arrived_conjuncts);
100104

101105
Status clone_conjunct_ctxs(VExprContextSPtrs& scanner_conjuncts);
102106

@@ -142,6 +146,9 @@ class ScanLocalStateBase : public PipelineXLocalState<> {
142146

143147
AnnotatedMutex _conjuncts_lock;
144148
RuntimeFilterConsumerHelper _helper;
149+
// Preserve append identity independently of the cost-sorted operator snapshot. Every scanner
150+
// needs the exact RF contexts added since its own applied count.
151+
std::vector<std::pair<int, VExprContextSPtrs>> _late_arrival_conjunct_batches;
145152
// magic number as seed to generate hash value for condition cache
146153
uint64_t _condition_cache_digest = 0;
147154
// condition cache filter stats

‎be/src/exec/scan/file_scanner_v2.cpp‎

Lines changed: 120 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include <map>
2525
#include <memory>
2626
#include <optional>
27+
#include <sstream>
2728
#include <string>
2829
#include <utility>
2930

@@ -226,7 +227,9 @@ Status rewrite_slot_refs_to_global_index(
226227
#ifdef BE_TEST
227228
FileScannerV2::FileScannerV2(RuntimeState* state, RuntimeProfile* profile,
228229
std::unique_ptr<format::TableReader> table_reader)
229-
: Scanner(state, profile), _table_reader(std::move(table_reader)) {}
230+
: Scanner(state, profile),
231+
_table_reader(std::move(table_reader)),
232+
_scanner_profile(profile) {}
230233

231234
Status FileScannerV2::TEST_validate_scan_range(const TFileScanRangeParams& params,
232235
const TFileRangeDesc& range) {
@@ -326,7 +329,9 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat
326329

327330
Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) {
328331
RETURN_IF_ERROR(Scanner::init(state, conjuncts));
332+
_initialize_scanner_residual_conjuncts();
329333
auto* profile = _local_state->scanner_profile();
334+
_scanner_profile = profile;
330335
const auto hierarchy = file_scan_profile::ensure_hierarchy(profile);
331336
_scanner_total_timer = hierarchy.scanner;
332337
_io_timer = hierarchy.io;
@@ -360,6 +365,11 @@ Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjunc
360365
profile, "AdaptiveBatchActualBytes", TUnit::BYTES, file_scan_profile::SCANNER, 1);
361366
_adaptive_batch_probe_count_counter = ADD_CHILD_COUNTER_WITH_LEVEL(
362367
profile, "AdaptiveBatchProbeCount", TUnit::UNIT, file_scan_profile::SCANNER, 1);
368+
_scanner_residual_filter_timer = ADD_CHILD_TIMER_WITH_LEVEL(
369+
profile, "ScannerResidualFilterTime", file_scan_profile::SCANNER, 1);
370+
_scanner_residual_rows_filtered_counter = ADD_CHILD_COUNTER_WITH_LEVEL(
371+
profile, "ScannerResidualRowsFiltered", TUnit::UNIT, file_scan_profile::SCANNER, 1);
372+
_refresh_scanner_residual_profile();
363373
SCOPED_TIMER(_scanner_total_timer);
364374
SCOPED_TIMER(_init_timer);
365375
_file_cache_statistics = std::make_unique<io::FileCacheStatistics>();
@@ -401,6 +411,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e
401411
SCOPED_TIMER(_get_block_timer);
402412
while (true) {
403413
RETURN_IF_CANCELLED(state);
414+
RETURN_IF_ERROR(_sync_table_reader_conjuncts());
404415
if (!_has_prepared_split) {
405416
RETURN_IF_ERROR(_prepare_next_split(eof));
406417
if (*eof) {
@@ -450,18 +461,33 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e
450461
}
451462

452463
Status FileScannerV2::_filter_output_block(Block* block) {
453-
return _contextualize_output_filter_status(Scanner::_filter_output_block(block),
454-
_get_current_format_type());
455-
}
456-
457-
Status FileScannerV2::_contextualize_output_filter_status(Status status,
458-
TFileFormatType::type format_type) {
459-
if (!status.ok() && format_type == TFileFormatType::FORMAT_ORC) {
460-
// Error-preserving expressions cannot be reordered into the ORC reader and therefore run
461-
// at the scanner boundary; keep their error context identical to ORC callback failures.
464+
if (_scanner_residual_conjuncts.empty() || block->rows() == 0) {
465+
return Status::OK();
466+
}
467+
SCOPED_TIMER(_scanner_residual_filter_timer);
468+
const size_t rows_before_filter = block->rows();
469+
auto status = VExprContext::filter_block(_scanner_residual_conjuncts, block, block->columns());
470+
if (!status.ok() && _params != nullptr &&
471+
_get_current_format_type() == TFileFormatType::FORMAT_ORC) {
462472
status.prepend("Orc row reader nextBatch failed. reason = ");
463473
}
464-
return status;
474+
RETURN_IF_ERROR(status);
475+
const int64_t filtered_rows = cast_set<int64_t>(rows_before_filter - block->rows());
476+
_counter.num_rows_unselected += filtered_rows;
477+
if (_scanner_residual_rows_filtered_counter != nullptr) {
478+
COUNTER_UPDATE(_scanner_residual_rows_filtered_counter, filtered_rows);
479+
}
480+
return Status::OK();
481+
}
482+
483+
size_t FileScannerV2::_last_block_rows_read(const Block& block) const {
484+
const auto& stats = _table_reader->last_materialized_block_stats();
485+
return stats.has_materialized_input ? stats.rows : block.rows();
486+
}
487+
488+
size_t FileScannerV2::_last_block_bytes_read(const Block& block) const {
489+
const auto& stats = _table_reader->last_materialized_block_stats();
490+
return stats.has_materialized_input ? stats.allocated_bytes : block.allocated_bytes();
465491
}
466492

467493
Status FileScannerV2::_prepare_next_split(bool* eos) {
@@ -547,6 +573,7 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) {
547573
RETURN_IF_ERROR(_table_reader->init({
548574
.projected_columns = _projected_columns,
549575
.conjuncts = std::move(table_conjuncts),
576+
.table_reader_owned_conjunct_count = _table_reader_owned_conjunct_count,
550577
.format = file_format,
551578
.scan_params = const_cast<TFileScanRangeParams*>(_params),
552579
.io_ctx = _io_ctx,
@@ -557,6 +584,9 @@ Status FileScannerV2::_init_table_reader(const TFileRangeDesc& range) {
557584
.push_down_count_columns = std::move(push_down_count_columns),
558585
.condition_cache_digest = _local_state->get_condition_cache_digest(),
559586
}));
587+
_table_reader_applied_rf_num = _applied_rf_num;
588+
// RFs collected before TableReader initialization are already present in the full snapshot.
589+
_late_arrival_rf_conjuncts.clear();
560590
return Status::OK();
561591
}
562592

@@ -601,15 +631,12 @@ Status FileScannerV2::_prepare_table_reader_split(const TFileRangeDesc& range,
601631
std::map<std::string, Field> partition_values) {
602632
format::FileFormat current_split_format;
603633
RETURN_IF_ERROR(_to_file_format(get_range_format_type(*_params, range), &current_split_format));
604-
VExprContextSPtrs conjuncts;
605-
RETURN_IF_ERROR(_build_table_conjuncts(&conjuncts));
606634
VExprContextSPtrs partition_prune_conjuncts;
607635
if (_state->query_options().enable_runtime_filter_partition_prune) {
608636
RETURN_IF_ERROR(_build_table_conjuncts(&partition_prune_conjuncts));
609637
}
610638
RETURN_IF_ERROR(_table_reader->prepare_split({
611639
.partition_values = std::move(partition_values),
612-
.conjuncts = std::move(conjuncts),
613640
.partition_prune_conjuncts = std::move(partition_prune_conjuncts),
614641
// A metadata COUNT split may span scheduler turns. Do not enter that irreversible
615642
// synthetic-row path while a runtime filter can still arrive between batches.
@@ -799,10 +826,15 @@ format::ColumnDefinition FileScannerV2::_build_table_column(const SlotDescriptor
799826
}
800827

801828
Status FileScannerV2::_build_table_conjuncts(VExprContextSPtrs* conjuncts) const {
829+
return _build_table_conjuncts(_conjuncts, conjuncts);
830+
}
831+
832+
Status FileScannerV2::_build_table_conjuncts(const VExprContextSPtrs& source,
833+
VExprContextSPtrs* conjuncts) const {
802834
DORIS_CHECK(conjuncts != nullptr);
803835
conjuncts->clear();
804-
conjuncts->reserve(_conjuncts.size());
805-
for (const auto& conjunct : _conjuncts) {
836+
conjuncts->reserve(source.size());
837+
for (const auto& conjunct : source) {
806838
VExprSPtr root;
807839
RETURN_IF_ERROR(format::clone_table_expr_tree(conjunct->root(), &root));
808840
RETURN_IF_ERROR(rewrite_slot_refs_to_global_index(&root, _slot_id_to_global_index));
@@ -811,6 +843,68 @@ Status FileScannerV2::_build_table_conjuncts(VExprContextSPtrs* conjuncts) const
811843
return Status::OK();
812844
}
813845

846+
size_t FileScannerV2::_safe_conjunct_prefix_size(const VExprContextSPtrs& conjuncts) {
847+
for (size_t conjunct_index = 0; conjunct_index < conjuncts.size(); ++conjunct_index) {
848+
if (!format::TableReader::is_safe_to_pre_execute(conjuncts[conjunct_index])) {
849+
return conjunct_index;
850+
}
851+
}
852+
return conjuncts.size();
853+
}
854+
855+
void FileScannerV2::_initialize_scanner_residual_conjuncts() {
856+
_table_reader_owned_conjunct_count = _safe_conjunct_prefix_size(_conjuncts);
857+
// Preserve the entire suffix, not only the unsafe expression. Otherwise a later safe
858+
// predicate could run below Scanner before a stateful/error-preserving ordering barrier.
859+
_scanner_residual_conjuncts.assign(
860+
_conjuncts.begin() + cast_set<ptrdiff_t>(_table_reader_owned_conjunct_count),
861+
_conjuncts.end());
862+
_refresh_scanner_residual_profile();
863+
}
864+
865+
void FileScannerV2::_refresh_scanner_residual_profile() {
866+
if (_scanner_profile == nullptr || _scanner_residual_conjuncts.empty()) {
867+
return;
868+
}
869+
std::ostringstream predicates;
870+
predicates << "[";
871+
for (size_t conjunct_index = 0; conjunct_index < _scanner_residual_conjuncts.size();
872+
++conjunct_index) {
873+
if (conjunct_index > 0) {
874+
predicates << ", ";
875+
}
876+
predicates << _scanner_residual_conjuncts[conjunct_index]->root()->debug_string();
877+
}
878+
predicates << "]";
879+
_scanner_profile->add_info_string("ScannerResidualPredicates", predicates.str());
880+
}
881+
882+
Status FileScannerV2::_sync_table_reader_conjuncts() {
883+
if (_table_reader == nullptr) {
884+
return Status::OK();
885+
}
886+
if (_table_reader_applied_rf_num == _applied_rf_num) {
887+
return Status::OK();
888+
}
889+
VExprContextSPtrs appended;
890+
RETURN_IF_ERROR(_build_table_conjuncts(_late_arrival_rf_conjuncts, &appended));
891+
const size_t owned_count = _scanner_residual_conjuncts.empty()
892+
? _safe_conjunct_prefix_size(_late_arrival_rf_conjuncts)
893+
: 0;
894+
// Preserve existing expression state and append the identity-tracked RF delta. Cost sorting
895+
// may move a late RF ahead of an old stateful predicate in the full scanner snapshot.
896+
RETURN_IF_ERROR(_table_reader->append_conjuncts_with_ownership(appended, owned_count));
897+
_table_reader_owned_conjunct_count += owned_count;
898+
_scanner_residual_conjuncts.insert(
899+
_scanner_residual_conjuncts.end(),
900+
_late_arrival_rf_conjuncts.begin() + cast_set<ptrdiff_t>(owned_count),
901+
_late_arrival_rf_conjuncts.end());
902+
_refresh_scanner_residual_profile();
903+
_late_arrival_rf_conjuncts.clear();
904+
_table_reader_applied_rf_num = _applied_rf_num;
905+
return Status::OK();
906+
}
907+
814908
TFileFormatType::type FileScannerV2::_get_current_format_type() const {
815909
return get_range_format_type(*_params, _current_range);
816910
}
@@ -934,17 +1028,19 @@ void FileScannerV2::_update_adaptive_batch_size(const Block& block) {
9341028
if (!_should_run_adaptive_batch_size()) {
9351029
return;
9361030
}
937-
COUNTER_SET(_adaptive_batch_actual_bytes_counter, static_cast<int64_t>(block.bytes()));
938-
if (block.rows() == 0) {
1031+
const auto& stats = _table_reader->last_materialized_block_stats();
1032+
const size_t rows = stats.has_materialized_input ? stats.rows : block.rows();
1033+
const size_t bytes = stats.has_materialized_input ? stats.bytes : block.bytes();
1034+
COUNTER_SET(_adaptive_batch_actual_bytes_counter, static_cast<int64_t>(bytes));
1035+
if (rows == 0) {
9391036
return;
9401037
}
941-
// The sample is taken after TableReader has finalized file-local columns to table columns.
942-
// This matches the memory shape seen by upstream operators and catches very wide nested
943-
// columns, such as map/string payloads, after the first probe batch.
1038+
// Residual predicates run after wide table columns are materialized. Learn from that pre-filter
1039+
// shape so selective predicates cannot make the next reader batch dangerously large.
9441040
if (!_block_size_predictor->has_history()) {
9451041
COUNTER_UPDATE(_adaptive_batch_probe_count_counter, 1);
9461042
}
947-
_block_size_predictor->update(block);
1043+
_block_size_predictor->update(rows, bytes);
9481044
}
9491045

9501046
Status FileScannerV2::close(RuntimeState* state) {
@@ -1145,9 +1241,8 @@ void FileScannerV2::_report_file_reader_predicate_filtered_rows() {
11451241
const int64_t filtered_rows = _io_ctx != nullptr ? _io_ctx->predicate_filtered_rows : 0;
11461242
const int64_t filtered_delta = filtered_rows - _reported_predicate_filtered_rows;
11471243
if (filtered_delta > 0) {
1148-
// File readers can evaluate localized conjuncts before a block reaches Scanner. Count
1149-
// those rows as scanner-level unselected rows so load statistics stay identical no matter
1150-
// whether a predicate is pushed down or evaluated by Scanner::_filter_output_block().
1244+
// FileReader and TableReader both report their owned predicate rows through the shared IO
1245+
// context. Preserve scanner-level load statistics without re-evaluating either predicate.
11511246
_counter.num_rows_unselected += filtered_delta;
11521247
_reported_predicate_filtered_rows = filtered_rows;
11531248
}

0 commit comments

Comments
 (0)