|
19 | 19 |
|
20 | 20 | #include <fmt/format.h> |
21 | 21 |
|
| 22 | +#include <algorithm> |
22 | 23 | #include <memory> |
23 | 24 | #include <numeric> |
24 | 25 | #include <optional> |
@@ -140,6 +141,13 @@ Status OlapScanLocalState::_init_profile() { |
140 | 141 | // 2. init timer and counters |
141 | 142 | _reader_init_timer = ADD_TIMER(_scanner_profile, "ReaderInitTime"); |
142 | 143 | _scanner_init_timer = ADD_TIMER(_scanner_profile, "ScannerInitTime"); |
| 144 | + _rowset_tso_prune_timer = ADD_TIMER(custom_profile(), "RowsetTsoPruneTime"); |
| 145 | + _rowsets_pruned_by_tso_counter = |
| 146 | + ADD_COUNTER(custom_profile(), "RowsetsPrunedByTso", TUnit::UNIT); |
| 147 | + _segments_pruned_by_tso_counter = |
| 148 | + ADD_COUNTER(custom_profile(), "SegmentsPrunedByTso", TUnit::UNIT); |
| 149 | + _tablets_pruned_by_tso_counter = |
| 150 | + ADD_COUNTER(custom_profile(), "TabletsPrunedByTso", TUnit::UNIT); |
143 | 151 | _process_conjunct_timer = ADD_TIMER(custom_profile(), "ProcessConjunctTime"); |
144 | 152 | _read_compressed_counter = ADD_COUNTER(_segment_profile, "CompressedBytesRead", TUnit::BYTES); |
145 | 153 | _read_uncompressed_counter = |
@@ -541,7 +549,6 @@ Status OlapScanLocalState::_should_push_down_function_filter(VectorizedFnCall* f |
541 | 549 | const auto& children = fn_call->children(); |
542 | 550 | doris::FunctionContext* func_cxt = expr_ctx->fn_context(fn_call->fn_context_index()); |
543 | 551 | DCHECK(func_cxt != nullptr); |
544 | | - DCHECK(children.size() == 2); |
545 | 552 | for (size_t i = 0; i < children.size(); i++) { |
546 | 553 | if (VExpr::expr_without_cast(children[i])->node_type() != TExprNodeType::SLOT_REF) { |
547 | 554 | // not a slot ref(column) |
@@ -635,6 +642,32 @@ bool OlapScanLocalState::_is_binlog_merge_scan() const { |
635 | 642 | return scan_type == TBinlogScanType::MIN_DELTA || scan_type == TBinlogScanType::DETAIL; |
636 | 643 | } |
637 | 644 |
|
| 645 | +void OlapScanLocalState::_prune_rowsets_by_tso(const TPaloScanRange& scan_range, |
| 646 | + TabletReadSource& read_source) { |
| 647 | + SCOPED_TIMER(_rowset_tso_prune_timer); |
| 648 | + int64_t pruned_segments = 0; |
| 649 | + const auto pruned_rowsets = std::erase_if(read_source.rs_splits, [&](const auto& split) { |
| 650 | + const auto& rowset = split.rs_reader->rowset(); |
| 651 | + const auto tso = rowset->commit_tso(); |
| 652 | + // Old rowsets can lack commit TSO metadata, including compaction inputs with an |
| 653 | + // unknown endpoint. Keep them for the existing segment/row-level predicates. |
| 654 | + if (tso.start_tso() < 0 || tso.end_tso() < 0) { |
| 655 | + return false; |
| 656 | + } |
| 657 | + DCHECK_LE(tso.start_tso(), tso.end_tso()); |
| 658 | + // Rowset metadata is inclusive [min, max]; the query is half-open [start, end). |
| 659 | + const bool outside_window = |
| 660 | + (scan_range.__isset.start_tso && tso.end_tso() < scan_range.start_tso) || |
| 661 | + (scan_range.__isset.end_tso && tso.start_tso() >= scan_range.end_tso); |
| 662 | + if (outside_window) { |
| 663 | + pruned_segments += rowset->num_segments(); |
| 664 | + } |
| 665 | + return outside_window; |
| 666 | + }); |
| 667 | + COUNTER_UPDATE(_rowsets_pruned_by_tso_counter, pruned_rowsets); |
| 668 | + COUNTER_UPDATE(_segments_pruned_by_tso_counter, pruned_segments); |
| 669 | +} |
| 670 | + |
638 | 671 | Status OlapScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) { |
639 | 672 | if (_scan_ranges.empty()) { |
640 | 673 | _eos = true; |
@@ -795,6 +828,22 @@ Status OlapScanLocalState::_init_scanners(std::list<ScannerSPtr>* scanners) { |
795 | 828 | int scanners_per_tablet = std::max(1, 64 / (int)_scan_ranges.size()); |
796 | 829 | for (size_t scan_range_idx = 0; scan_range_idx < _scan_ranges.size(); scan_range_idx++) { |
797 | 830 | const auto& palo_scan_range = *_scan_ranges[scan_range_idx]; |
| 831 | + if (read_row_binlog && |
| 832 | + (palo_scan_range.__isset.start_tso || palo_scan_range.__isset.end_tso)) { |
| 833 | + auto& read_source = _read_sources[scan_range_idx]; |
| 834 | + // The version-consistent read source and delete predicates have already been |
| 835 | + // captured. Prune before cloning readers or opening any segment footers. |
| 836 | + _prune_rowsets_by_tso(palo_scan_range, read_source); |
| 837 | + if (std::all_of(read_source.rs_splits.begin(), read_source.rs_splits.end(), |
| 838 | + [](const auto& split) { |
| 839 | + return split.rs_reader->rowset()->num_rows() == 0; |
| 840 | + })) { |
| 841 | + // Empty bootstrap rowsets may have no TSO. Skip the tablet even if those |
| 842 | + // remain, and do not let OlapScanner recapture an empty read source. |
| 843 | + COUNTER_UPDATE(_tablets_pruned_by_tso_counter, 1); |
| 844 | + continue; |
| 845 | + } |
| 846 | + } |
798 | 847 | int64_t version = 0; |
799 | 848 | std::from_chars(palo_scan_range.version.data(), |
800 | 849 | palo_scan_range.version.data() + palo_scan_range.version.size(), version); |
|
0 commit comments