Skip to content

Commit 28cc2bf

Browse files
[Improvement](scan) support push down limit to segment iterator (apache#62222)
This pull request introduces significant improvements to scan operator logic, particularly enhancing the correctness and efficiency of LIMIT and predicate pushdown in OLAP scans. The main changes include a robust mechanism for sharing LIMIT counters among scanners, more precise control over predicate pushdown, and stricter validation for residual predicates. These changes help ensure that queries with LIMIT and TopN semantics return accurate results and avoid incorrect or inefficient execution paths. **Improvements to LIMIT handling and scanner coordination:** - Introduced a shared atomic counter (`_shared_scan_limit`) for LIMIT queries, allowing all scanners to collectively respect the global row limit. Scanners now check and update this counter to stop early when the LIMIT is reached, preventing over-scanning and improving efficiency. TopN scans bypass this mechanism as required. [[1]](diffhunk://#diff-7fe2376680f9ab835053a0a93c7f9458e6d526acfc180318be0ad936e0c2dedeR50-R55) [[2]](diffhunk://#diff-7fe2376680f9ab835053a0a93c7f9458e6d526acfc180318be0ad936e0c2dedeR135-R143) [[3]](diffhunk://#diff-7fe2376680f9ab835053a0a93c7f9458e6d526acfc180318be0ad936e0c2dedeR177-R183) [[4]](diffhunk://#diff-7fe2376680f9ab835053a0a93c7f9458e6d526acfc180318be0ad936e0c2dedeR196) [[5]](diffhunk://#diff-9c777f60e1d076777f042a0b702967bd5864bcfe9ee1eee9b456bb67c6fdf817R266-R269) [[6]](diffhunk://#diff-a45108106b12759815ac5991d56a9d6ead7ffb9c915fb0cf7b6f0b8123b30262R71) [[7]](diffhunk://#diff-a45108106b12759815ac5991d56a9d6ead7ffb9c915fb0cf7b6f0b8123b30262R239) [[8]](diffhunk://#diff-a45108106b12759815ac5991d56a9d6ead7ffb9c915fb0cf7b6f0b8123b30262L283-R285) [[9]](diffhunk://#diff-3c4794a864169735a628d8fcb1de986523a9516aad318aa0a893913f4ce031d4R1043-R1050) - Adjusted scanner and scan local state interfaces to support the new shared LIMIT mechanism, including new virtual methods and atomic counter management. [[1]](diffhunk://#diff-a45108106b12759815ac5991d56a9d6ead7ffb9c915fb0cf7b6f0b8123b30262R71) [[2]](diffhunk://#diff-a45108106b12759815ac5991d56a9d6ead7ffb9c915fb0cf7b6f0b8123b30262R239) [[3]](diffhunk://#diff-a45108106b12759815ac5991d56a9d6ead7ffb9c915fb0cf7b6f0b8123b30262L283-R285) **Predicate pushdown and validation enhancements:** - Refactored `_should_push_down_common_expr` to require the expression as an argument and added logic to ensure only eligible expressions are pushed down, considering storage merge requirements and key columns. [[1]](diffhunk://#diff-780cdfbcbb1d30adb63c61521e6b681cad347bebeaa5332a781d05bb51543527L46-R46) [[2]](diffhunk://#diff-235cc51f4698ebf0ebe796e7253912907bb7c1caf4efcac92538ac1ff2eac171L104-R107) [[3]](diffhunk://#diff-3ddc75656071d9c0e6b0be450e152a1c94559f7e70ea820e7f0c80a7078e3292L465-R547) [[4]](diffhunk://#diff-3c4794a864169735a628d8fcb1de986523a9516aad318aa0a893913f4ce031d4L313-R313) - Added a validation step (`validate_residual_scan_conjuncts`) to prevent unsupported residual predicates (like SEARCH or disabled MATCH) and to ensure correctness when using COUNT_ON_INDEX pushdown. **OLAP scan and segment limit logic improvements:** - Improved the logic for pushing down segment limits and TopN optimizations, ensuring that such optimizations are only enabled when all necessary conditions are met (e.g., no residual predicates, no runtime filters, storage does not require merging). The code now asserts correct states and disables shared LIMITs for TopN scans as appropriate. [[1]](diffhunk://#diff-4d3f2b2a64e1dac3bc076994d3cd9a6c569f7a8c6ad5bc69425188ffccb4266bL446-R502) [[2]](diffhunk://#diff-3ddc75656071d9c0e6b0be450e152a1c94559f7e70ea820e7f0c80a7078e3292R1155-R1156) **Code cleanup and maintenance:** - Removed unused or obsolete logic related to filter block conjuncts and the old limit quota acquisition method. [[1]](diffhunk://#diff-4d3f2b2a64e1dac3bc076994d3cd9a6c569f7a8c6ad5bc69425188ffccb4266bL90) [[2]](diffhunk://#diff-0c9a817d45d8130ea3211189e1321d1275e22fd4a9a3fac2bd707b1cfeefa5e5L115-L135) **Dependency and include updates:** - Added missing includes for expression types to support new logic. ```sql CREATE DATABASE IF NOT EXISTS lm_bench; USE lm_bench; DROP TABLE IF EXISTS lm_wide_fact_10m; CREATE TABLE lm_wide_fact_10m ( id BIGINT NOT NULL, filter_key INT NOT NULL, sort_key BIGINT NOT NULL, metric BIGINT NOT NULL, payload_a VARCHAR(1024) NOT NULL, payload_b VARCHAR(1024) NOT NULL, long_payload_a VARCHAR(4096) NOT NULL, long_payload_b VARCHAR(4096) NOT NULL ) ENGINE=OLAP DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 32 PROPERTIES ( "replication_num" = "1" ); INSERT INTO lm_wide_fact_10m SELECT number AS id, CAST(number % 1000000 AS INT) AS filter_key, (number * 2654435761) % 1000000007 AS sort_key, (number * 1103515245 + 12345) % 2147483647 AS metric, REPEAT(CONCAT('payload_a_', CAST(number AS STRING), '_', CAST((number * 1315423911) % 1000000007 AS STRING), '_'), 16) AS payload_a, REPEAT(CONCAT('payload_b_', CAST(number AS STRING), '_', CAST((number * 2654435761) % 1000000009 AS STRING), '_'), 16) AS payload_b, REPEAT(CONCAT('long_payload_a_', CAST(number AS STRING), '_', CAST((number * 48271) % 2147483647 AS STRING), '_'), 64) AS long_payload_a, REPEAT(CONCAT('long_payload_b_', CAST(number AS STRING), '_', CAST((number * 69621) % 2147483629 AS STRING), '_'), 64) AS long_payload_b FROM numbers("number" = "10000000"); ANALYZE TABLE lm_wide_fact_10m WITH SYNC; -- ============================================================ -- Case 1: TopN next + LIMIT + high-filter-rate pushed-down predicate -- Expected observation: -- filter_key is a narrow pushed-down predicate and keeps about 1% rows. -- TopN works on narrow metric/sort_key, while long strings are final output. -- ============================================================ SELECT id, filter_key, metric, sort_key, payload_a, payload_b, long_payload_a, long_payload_b FROM lm_wide_fact_10m WHERE filter_key < 10000 ORDER BY metric DESC, sort_key ASC LIMIT 10; -- Result: 0.12 sec -> 0.10 sec -- ============================================================ -- Case 2: Normal scan + LIMIT -- Expected observation: -- No ORDER BY and no predicate. This isolates ordinary scan limit behavior -- while still returning long string columns. -- ============================================================ SELECT id, filter_key, metric, sort_key, payload_a, payload_b, long_payload_a, long_payload_b FROM lm_wide_fact_10m LIMIT 10; -- Result: 0.06 sec -> 0.01 sec -- ============================================================ -- Case 3: Normal scan + LIMIT + high-filter-rate pushed-down predicate -- Expected observation: -- No ORDER BY. This isolates scan predicate + limit behavior with long -- string result columns and a predicate that filters about 99% of rows. -- ============================================================ SELECT id, filter_key, metric, sort_key, payload_a, payload_b, long_payload_a, long_payload_b FROM lm_wide_fact_10m WHERE filter_key < 10000 LIMIT 10; -- Result: 0.07 sec -> 0.02 sec ``` --------- Co-authored-by: Copilot <[email protected]>
1 parent 464d89c commit 28cc2bf

34 files changed

Lines changed: 849 additions & 223 deletions

‎be/src/exec/operator/mock_scan_operator.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ class MockScanLocalState final : public ScanLocalState<MockScanLocalState> {
4343
return PushDownType::ACCEPTABLE;
4444
}
4545

46-
bool _should_push_down_common_expr() override { return true; }
46+
bool _should_push_down_common_expr(const VExprSPtr&) override { return true; }
4747
PushDownType _should_push_down_topn_filter() const override { return PushDownType::ACCEPTABLE; }
4848

4949
PushDownType _should_push_down_is_null_predicate(VectorizedFnCall* fn_call) const override {

‎be/src/exec/operator/olap_scan_operator.cpp‎

Lines changed: 85 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include "exprs/vectorized_fn_call.h"
3838
#include "exprs/vexpr.h"
3939
#include "exprs/vexpr_context.h"
40+
#include "exprs/virtual_slot_ref.h"
4041
#include "exprs/vslot_ref.h"
4142
#include "io/cache/block_file_cache_profile.h"
4243
#include "runtime/query_cache/query_cache.h"
@@ -407,12 +408,62 @@ Status OlapScanLocalState::_init_profile() {
407408
return Status::OK();
408409
}
409410

411+
static bool contains_expr_node_type(const VExprSPtr& expr, TExprNodeType::type node_type) {
412+
DORIS_CHECK(expr != nullptr);
413+
if (expr->node_type() == node_type) {
414+
return true;
415+
}
416+
if (expr->is_rf_wrapper() && contains_expr_node_type(expr->get_impl(), node_type)) {
417+
return true;
418+
}
419+
return std::ranges::any_of(expr->children(), [node_type](const auto& child) {
420+
return contains_expr_node_type(child, node_type);
421+
});
422+
}
423+
424+
static Status validate_residual_scan_conjuncts(RuntimeState* state,
425+
TPushAggOp::type push_down_agg_type,
426+
const VExprContextSPtrs& conjuncts) {
427+
for (const auto& conjunct : conjuncts) {
428+
const auto& root = conjunct->root();
429+
if (contains_expr_node_type(root, TExprNodeType::SEARCH_EXPR)) {
430+
return Status::InvalidArgument(
431+
"SEARCH expression remains as a residual scan predicate. A valid search() "
432+
"must bind at least one indexed field and be evaluated in SegmentIterator. "
433+
"enable_segment_limit_pushdown only controls SegmentIterator LIMIT pushdown "
434+
"and cannot make residual SEARCH executable.");
435+
}
436+
if (!state->query_options().enable_match_without_inverted_index &&
437+
contains_expr_node_type(root, TExprNodeType::MATCH_PRED)) {
438+
return Status::InvalidArgument(
439+
"MATCH expression remains as a residual scan predicate and would fall back to "
440+
"a disabled slow path because enable_match_without_inverted_index is false. "
441+
"enable_segment_limit_pushdown only controls SegmentIterator LIMIT pushdown "
442+
"and cannot make residual MATCH executable. Set "
443+
"enable_match_without_inverted_index=true to allow slow MATCH execution.");
444+
}
445+
}
446+
447+
if (push_down_agg_type == TPushAggOp::COUNT_ON_INDEX && !conjuncts.empty()) {
448+
return Status::InvalidArgument(
449+
"COUNT_ON_INDEX pushdown cannot be used with residual scan predicates. "
450+
"Residual predicates must be evaluated before COUNT_ON_INDEX counts rows; "
451+
"otherwise the query may return incorrect results. "
452+
"enable_segment_limit_pushdown only controls SegmentIterator LIMIT pushdown and "
453+
"does not make COUNT_ON_INDEX safe with residual predicates. Set "
454+
"enable_count_on_index_pushdown=false to disable COUNT_ON_INDEX pushdown.");
455+
}
456+
return Status::OK();
457+
}
458+
410459
Status OlapScanLocalState::_process_conjuncts(RuntimeState* state) {
411460
SCOPED_TIMER(_process_conjunct_timer);
412461
RETURN_IF_ERROR(ScanLocalState::_process_conjuncts(state));
413462
if (ScanLocalState::_eos) {
414463
return Status::OK();
415464
}
465+
auto& p = _parent->cast<OlapScanOperatorX>();
466+
RETURN_IF_ERROR(validate_residual_scan_conjuncts(state, p._push_down_agg_type, _conjuncts));
416467
RETURN_IF_ERROR(_build_key_ranges_and_filters());
417468
return Status::OK();
418469
}
@@ -472,8 +523,38 @@ Status OlapScanLocalState::_should_push_down_function_filter(VectorizedFnCall* f
472523
return Status::OK();
473524
}
474525

475-
bool OlapScanLocalState::_should_push_down_common_expr() {
476-
return state()->enable_common_expr_pushdown() && _storage_no_merge();
526+
bool OlapScanLocalState::_should_push_down_common_expr(const VExprSPtr& expr) {
527+
// SegmentIterator common exprs must eventually act on at least one scan slot.
528+
if (!_check_expr_storage_filter(expr, ExprStorageFilterCheckMode::HAS_SEGMENT_EVALUABLE_EXPR)) {
529+
return false;
530+
}
531+
532+
// DUP and UNIQUE-MOW/MOR-as-DUP do not need storage aggregation/merge, so any slot-based common
533+
// expression can be evaluated together with SegmentIterator lazy materialization.
534+
if (_storage_no_merge()) {
535+
return true;
536+
}
537+
538+
// AGG and UNIQUE-MOR may still merge value columns above SegmentIterator. Push only key-column
539+
// expressions so filtering does not observe pre-merge values.
540+
return !_check_expr_storage_filter(expr, ExprStorageFilterCheckMode::HAS_NON_KEY_SLOT);
541+
}
542+
543+
bool OlapScanLocalState::_check_expr_storage_filter(const VExprSPtr& expr,
544+
ExprStorageFilterCheckMode mode) {
545+
if (expr->is_slot_ref()) {
546+
const auto* slot_ref = assert_cast<const VSlotRef*>(expr.get());
547+
return mode == ExprStorageFilterCheckMode::HAS_SEGMENT_EVALUABLE_EXPR ||
548+
!_is_key_column(slot_ref->expr_name());
549+
}
550+
if (expr->is_virtual_slot_ref()) {
551+
// Treat virtual slot ref as non-key because it may depend on non-key source columns.
552+
return true;
553+
}
554+
555+
return std::ranges::any_of(expr->children(), [this, mode](const auto& child) {
556+
return _check_expr_storage_filter(child, mode);
557+
});
477558
}
478559

479560
bool OlapScanLocalState::_storage_no_merge() {
@@ -1081,6 +1162,8 @@ OlapScanOperatorX::OlapScanOperatorX(ObjectPool* pool, const TPlanNode& tnode, i
10811162
_cache_param(param) {
10821163
_output_tuple_id = tnode.olap_scan_node.tuple_id;
10831164
if (_olap_scan_node.__isset.sort_info && _olap_scan_node.__isset.sort_limit) {
1165+
DORIS_CHECK(_limit < 0);
1166+
DORIS_CHECK(_olap_scan_node.sort_limit > 0);
10841167
_limit_per_scanner = _olap_scan_node.sort_limit;
10851168
}
10861169
DBUG_EXECUTE_IF("segment_iterator.topn_opt_1", {

‎be/src/exec/operator/olap_scan_operator.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,10 @@ class OlapScanLocalState final : public ScanLocalState<OlapScanLocalState> {
101101
VectorizedFnCall* fn_call, VExprContext* expr_ctx, Field& constant_val,
102102
const std::set<std::string> fn_name) const override;
103103

104-
bool _should_push_down_common_expr() override;
104+
bool _should_push_down_common_expr(const VExprSPtr& expr) override;
105+
106+
enum class ExprStorageFilterCheckMode { HAS_SEGMENT_EVALUABLE_EXPR, HAS_NON_KEY_SLOT };
107+
bool _check_expr_storage_filter(const VExprSPtr& expr, ExprStorageFilterCheckMode mode);
105108

106109
bool _storage_no_merge() override;
107110

‎be/src/exec/operator/scan_operator.cpp‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -310,8 +310,7 @@ Status ScanLocalState<Derived>::_normalize_conjuncts(RuntimeState* state) {
310310
RETURN_IF_ERROR(_normalize_predicate(conjunct.get(), conjunct->root(), new_root));
311311
if (new_root) {
312312
conjunct->set_root(new_root);
313-
if (_should_push_down_common_expr() &&
314-
VExpr::is_acting_on_a_slot(*(conjunct->root()))) {
313+
if (_should_push_down_common_expr(conjunct->root())) {
315314
_common_expr_ctxs_push_down.emplace_back(conjunct);
316315
it = _conjuncts.erase(it);
317316
continue;
@@ -1041,6 +1040,14 @@ int64_t ScanLocalState<Derived>::limit_per_scanner() {
10411040
return _parent->cast<typename Derived::Parent>()._limit_per_scanner;
10421041
}
10431042

1043+
template <typename Derived>
1044+
std::atomic<int64_t>* ScanLocalState<Derived>::shared_scan_limit_ptr() {
1045+
auto* p = &_parent->cast<typename Derived::Parent>()._shared_scan_limit;
1046+
// -1 means "no SQL LIMIT" — return nullptr so callers naturally skip
1047+
// all limit logic.
1048+
return p->load(std::memory_order_relaxed) < 0 ? nullptr : p;
1049+
}
1050+
10441051
template <typename Derived>
10451052
Status ScanLocalState<Derived>::_init_profile() {
10461053
// 1. counters for scan node

‎be/src/exec/operator/scan_operator.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ class ScanLocalStateBase : public PipelineXLocalState<> {
6868
[[nodiscard]] virtual const TupleDescriptor* output_tuple_desc() const = 0;
6969

7070
virtual int64_t limit_per_scanner() = 0;
71+
virtual std::atomic<int64_t>* shared_scan_limit_ptr() = 0;
7172

7273
virtual void set_scan_ranges(RuntimeState* state,
7374
const std::vector<TScanRangeParams>& scan_ranges) = 0;
@@ -235,6 +236,7 @@ class ScanLocalState : public ScanLocalStateBase {
235236
[[nodiscard]] const TupleDescriptor* output_tuple_desc() const override;
236237

237238
int64_t limit_per_scanner() override;
239+
std::atomic<int64_t>* shared_scan_limit_ptr() override;
238240

239241
void set_scan_ranges(RuntimeState* state,
240242
const std::vector<TScanRangeParams>& scan_ranges) override {}
@@ -280,7 +282,7 @@ class ScanLocalState : public ScanLocalStateBase {
280282

281283
Status _init_profile() override;
282284
virtual Status _process_conjuncts(RuntimeState* state) { return _normalize_conjuncts(state); }
283-
virtual bool _should_push_down_common_expr() { return false; }
285+
virtual bool _should_push_down_common_expr(const VExprSPtr&) { return false; }
284286

285287
virtual bool _storage_no_merge() { return false; }
286288
virtual bool _is_key_column(const std::string& col_name) { return false; }

‎be/src/exec/scan/olap_scanner.cpp‎

Lines changed: 47 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,6 @@ OlapScanner::OlapScanner(ScanLocalStateBase* parent, OlapScanner::Params&& param
8787
.remaining_conjunct_roots {},
8888
.common_expr_ctxs_push_down {},
8989
.topn_filter_source_node_ids {},
90-
.filter_block_conjuncts {},
9190
.key_group_cluster_key_idxes {},
9291
.virtual_column_exprs {},
9392
.vir_cid_to_idx_in_block {},
@@ -443,46 +442,64 @@ Status OlapScanner::_init_tablet_reader_params(
443442
_tablet_reader_params.enable_mor_value_predicate_pushdown = true;
444443
}
445444

446-
// Skip topn / general-limit storage-layer optimizations when runtime
447-
// filters exist. Late-arriving filters would re-populate _conjuncts
448-
// at the scanner level while the storage layer has already committed
449-
// to a row budget counted before those filters, causing the scan to
450-
// return fewer rows than the limit requires.
451-
if (_total_rf_num == 0) {
452-
// order by table keys optimization for topn
453-
// will only read head/tail of data file since it's already sorted by keys
454-
if (olap_scan_node.__isset.sort_info &&
455-
!olap_scan_node.sort_info.is_asc_order.empty()) {
456-
_limit = _local_state->limit_per_scanner();
445+
const bool has_key_topn =
446+
olap_scan_node.__isset.sort_info && !olap_scan_node.sort_info.is_asc_order.empty();
447+
if (has_key_topn) {
448+
_limit = _local_state->limit_per_scanner();
449+
}
450+
451+
const bool no_runtime_filters = _total_rf_num == 0;
452+
const bool segment_limit_enabled = _state->enable_segment_limit_pushdown();
453+
const bool storage_no_merge = olap_scan_local_state->_storage_no_merge();
454+
455+
if (_limit > 0 && no_runtime_filters && segment_limit_enabled && storage_no_merge) {
456+
for (const auto& conjunct : _conjuncts) {
457+
DORIS_CHECK(!olap_scan_local_state->_check_expr_storage_filter(
458+
conjunct->root(), OlapScanLocalState::ExprStorageFilterCheckMode::
459+
HAS_SEGMENT_EVALUABLE_EXPR));
460+
}
461+
}
462+
463+
// Segment LIMIT has only two legal states: completely disabled, or enabled after every
464+
// row-filtering conjunct has become a storage predicate or SegmentIterator common expr.
465+
const bool can_push_down_segment_limit = _limit > 0 && no_runtime_filters &&
466+
_conjuncts.empty() && segment_limit_enabled &&
467+
storage_no_merge;
468+
if (can_push_down_segment_limit) {
469+
if (has_key_topn) {
457470
_tablet_reader_params.read_orderby_key = true;
458471
if (!olap_scan_node.sort_info.is_asc_order[0]) {
459472
_tablet_reader_params.read_orderby_key_reverse = true;
460473
}
461474
_tablet_reader_params.read_orderby_key_num_prefix_columns =
462475
olap_scan_node.sort_info.is_asc_order.size();
463476
_tablet_reader_params.read_orderby_key_limit = _limit;
464-
465-
if (_tablet_reader_params.read_orderby_key_limit > 0 &&
466-
olap_scan_local_state->_storage_no_merge()) {
467-
_tablet_reader_params.filter_block_conjuncts = _conjuncts;
468-
_conjuncts.clear();
469-
}
470-
} else if (_limit > 0 && olap_scan_local_state->_storage_no_merge()) {
471-
// General limit pushdown for DUP_KEYS and UNIQUE_KEYS with MOW
472-
// (non-merge path). Only when topn optimization is NOT active.
473-
// NOTE: _limit is the global query limit (TPlanNode.limit), not a
474-
// per-scanner budget. With N scanners each scanner may read up to
475-
// _limit rows, so up to N * _limit rows are read in total before
476-
// the _shared_scan_limit coordinator stops them. This is
477-
// acceptable because _shared_scan_limit guarantees correctness,
478-
// and the over-read is bounded by (N-1) * _limit which is small
479-
// for typical LIMIT values.
477+
} else {
480478
_tablet_reader_params.general_read_limit = _limit;
481-
_tablet_reader_params.filter_block_conjuncts = _conjuncts;
482-
_conjuncts.clear();
483479
}
484480
}
485481

482+
if (_tablet_reader_params.read_orderby_key_limit > 0 ||
483+
_tablet_reader_params.general_read_limit > 0) {
484+
DORIS_CHECK(can_push_down_segment_limit);
485+
DORIS_CHECK(_conjuncts.empty());
486+
}
487+
488+
// A key TopN scan cannot share the plain LIMIT early-stop counter. If
489+
// storage TopN is pushed down, each scanner must produce its full local
490+
// candidates. If it is not pushed down for any reason, the upper TopN
491+
// still needs all rows from the scan.
492+
if (has_key_topn) {
493+
_shared_scan_limit = nullptr;
494+
if (_tablet_reader_params.read_orderby_key_limit == 0) {
495+
_limit = -1;
496+
}
497+
}
498+
// Note: _shared_scan_limit is intentionally not pushed into the
499+
// storage layer. SegmentIterator's _process_eof() is irreversible,
500+
// so a concurrently-decremented atomic could reach 0 while a segment
501+
// still has data needed by other scanners.
502+
486503
// set push down topn filter
487504
_tablet_reader_params.topn_filter_source_node_ids =
488505
olap_scan_local_state->get_topn_filter_source_node_ids(_state, true);

‎be/src/exec/scan/scanner.cpp‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,12 @@ Scanner::Scanner(RuntimeState* state, ScanLocalStateBase* local_state, int64_t l
4747
}
4848

4949
Status Scanner::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) {
50+
// All scanners share a remaining-limit counter so a LIMIT query can
51+
// stop once enough rows have been collected across scanners.
52+
// Key TopN scans have no ordinary scan LIMIT, so each scanner can
53+
// independently produce its full local top-N.
54+
_shared_scan_limit = _local_state->shared_scan_limit_ptr();
55+
5056
if (!conjuncts.empty()) {
5157
_conjuncts.resize(conjuncts.size());
5258
for (size_t i = 0; i != conjuncts.size(); ++i) {
@@ -126,6 +132,15 @@ Status Scanner::get_block_after_projects(RuntimeState* state, Block* block, bool
126132
Status Scanner::get_block(RuntimeState* state, Block* block, bool* eof) {
127133
// only empty block should be here
128134
DCHECK(block->rows() == 0);
135+
136+
// Stop early if other scanners have already collected enough rows
137+
// for the SQL LIMIT. Skipped when _shared_scan_limit is null (topn
138+
// path or no LIMIT).
139+
if (_shared_scan_limit && _shared_scan_limit->load(std::memory_order_acquire) <= 0) {
140+
*eof = true;
141+
return Status::OK();
142+
}
143+
129144
// scanner running time
130145
SCOPED_RAW_TIMER(&_per_scanner_timer);
131146
int64_t rows_read_threshold = _num_rows_read + config::doris_scanner_row_num;
@@ -159,6 +174,13 @@ Status Scanner::get_block(RuntimeState* state, Block* block, bool* eof) {
159174
}
160175
// record rows return (after filter) for _limit check
161176
_num_rows_return += block->rows();
177+
// Publish progress to the shared counter so peer scanners can
178+
// observe it. The counter may go negative when several scanners
179+
// subtract concurrently; that is harmless because the operator's
180+
// reached_limit() makes the final cut.
181+
if (_shared_scan_limit && block->rows() > 0) {
182+
_shared_scan_limit->fetch_sub(block->rows(), std::memory_order_acq_rel);
183+
}
162184
} while (!_should_stop && !state->is_cancelled() && block->rows() == 0 && !(*eof) &&
163185
_num_rows_read < rows_read_threshold);
164186
}
@@ -171,6 +193,7 @@ Status Scanner::get_block(RuntimeState* state, Block* block, bool* eof) {
171193
// set eof to true if per scanner limit is reached
172194
// currently for query: ORDER BY key LIMIT n
173195
*eof = *eof || (_limit > 0 && _num_rows_return >= _limit);
196+
*eof = *eof || (_shared_scan_limit && _shared_scan_limit->load(std::memory_order_acquire) <= 0);
174197

175198
return Status::OK();
176199
}

‎be/src/exec/scan/scanner.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,10 @@ class Scanner {
274274
int64_t _projection_timer = 0;
275275

276276
bool _should_stop = false;
277+
278+
// Cached pointer to ScanOperator's remaining-limit counter. Null when
279+
// this scanner is on the topn path or the query has no LIMIT.
280+
std::atomic<int64_t>* _shared_scan_limit = nullptr;
277281
};
278282

279283
using ScannerSPtr = std::shared_ptr<Scanner>;

0 commit comments

Comments
 (0)