Skip to content

Commit 01e735b

Browse files
authored
[fix](be) Fix condition cache and COUNT_ON_INDEX predicate handling (apache#69087)
### What problem does this PR solve? ### 1. Segment TopN leaves an incomplete condition cache SQL: ```sql -- c2 is a DUPLICATE KEY table with multiple overlapping segments. SELECT k, v FROM c2 WHERE abs(v) > 0 ORDER BY k LIMIT 1; SELECT count(*), sum(k) FROM c2 WHERE abs(v) > 0; SELECT k, v FROM c2 WHERE abs(v) > 1 ORDER BY k DESC LIMIT 1; SELECT count(*), sum(k) FROM c2 WHERE abs(v) > 1; Before this fix: - abs(v) > 0 returns 7022 rows instead of 14043. - abs(v) > 1 returns 4681 rows instead of 9362. The segment reaches its local read limit and writes an incomplete condition-cache entry. A later query reuses the entry and skips valid row ranges. After this fix: - abs(v) > 0 returns 14043 rows. - abs(v) > 1 returns 9362 rows. A segment that reaches its local read limit no longer writes the incomplete condition-cache entry. ### 2. COUNT_ON_INDEX rejects valid slotless predicates SQL: -- t8 is a DUPLICATE KEY table with an inverted index on v. SELECT count(*) FROM t8 WHERE random() < 2; SELECT count(*) FROM t8 WHERE v MATCH_ANY 'apple' AND random() < 2; Before this fix: Both queries fail with: INVALID_ARGUMENT: COUNT_ON_INDEX pushdown cannot be used with residual scan predicates After this fix: - The first query returns 3. - The second query returns 2. COUNT_ON_INDEX now rejects only residual predicates that reference columns. Slotless predicates are evaluated by the scanner. Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [ ] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent 15eb3f4 commit 01e735b

6 files changed

Lines changed: 165 additions & 7 deletions

File tree

‎be/src/exec/operator/olap_scan_operator.cpp‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include <memory>
2424
#include <numeric>
2525
#include <optional>
26+
#include <set>
2627
#include <shared_mutex>
2728

2829
#include "cloud/cloud_meta_mgr.h"
@@ -499,13 +500,18 @@ static Status validate_residual_scan_conjuncts(RuntimeState* state,
499500
}
500501
}
501502

502-
if (push_down_agg_type == TPushAggOp::COUNT_ON_INDEX && !conjuncts.empty()) {
503+
// Reject only residual predicates that reference columns: COUNT_ON_INDEX fills columns with
504+
// defaults, while a predicate such as random() < 2 still filters each row correctly.
505+
if (push_down_agg_type == TPushAggOp::COUNT_ON_INDEX &&
506+
std::ranges::any_of(conjuncts, [](const VExprContextSPtr& conjunct) {
507+
std::set<int> column_ids;
508+
conjunct->root()->collect_slot_column_ids(column_ids);
509+
return !column_ids.empty();
510+
})) {
503511
return Status::InvalidArgument(
504-
"COUNT_ON_INDEX pushdown cannot be used with residual scan predicates. "
505-
"Residual predicates must be evaluated before COUNT_ON_INDEX counts rows; "
506-
"otherwise the query may return incorrect results. "
507-
"enable_segment_limit_pushdown only controls SegmentIterator LIMIT pushdown and "
508-
"does not make COUNT_ON_INDEX safe with residual predicates. Set "
512+
"COUNT_ON_INDEX pushdown cannot be used with residual scan predicates that "
513+
"reference columns. Residual predicates must be evaluated before COUNT_ON_INDEX "
514+
"counts rows; otherwise the query may return incorrect results. Set "
509515
"enable_count_on_index_pushdown=false to disable COUNT_ON_INDEX pushdown.");
510516
}
511517
return Status::OK();

‎be/src/storage/segment/segment_iterator.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2892,7 +2892,8 @@ Status SegmentIterator::next_batch(Block* block) {
28922892
block->replace_by_position(cid, type->create_column());
28932893
}
28942894

2895-
if (_opts.condition_cache_digest && !_find_condition_cache) {
2895+
if (_opts.condition_cache_digest && !_find_condition_cache &&
2896+
!(_opts.read_limit > 0 && _rows_returned >= _opts.read_limit)) {
28962897
auto* condition_cache = ConditionCache::instance();
28972898
ConditionCache::CacheKey cache_key(_opts.rowset_id, _segment->id(),
28982899
_opts.condition_cache_digest);
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !topn_asc --
3+
0 -3
4+
5+
-- !all_after_asc --
6+
14043 115029723
7+
8+
-- !topn_desc --
9+
16381 -2
10+
11+
-- !all_after_desc --
12+
9362 76686481
13+
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !all_rows --
3+
3
4+
5+
-- !no_rows --
6+
0
7+
8+
-- !match_all_rows --
9+
2
10+
11+
-- !match_no_rows --
12+
0
13+
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
// A storage TopN (ORDER BY key LIMIT n) stops each segment after n rows. It must not leave
19+
// a condition cache entry behind, or a later query with the same predicate skips row ranges
20+
// that do have matching rows.
21+
//
22+
// The bug needs a rowset with overlapping segments, which is read by VMergeIterator: it asks
23+
// every segment for one more batch, so a segment that hit its LIMIT reaches EOF.
24+
// MemTable.need_flush writes one segment per inserted block.
25+
suite("test_segment_limit_condition_cache", "nonConcurrent") {
26+
def tbl = "test_segment_limit_condition_cache"
27+
sql "DROP TABLE IF EXISTS ${tbl}"
28+
sql """
29+
CREATE TABLE ${tbl} (k INT, v INT)
30+
DUPLICATE KEY(k)
31+
DISTRIBUTED BY HASH(k) BUCKETS 1
32+
PROPERTIES ('replication_num' = '1', 'disable_auto_compaction' = 'true')
33+
"""
34+
35+
try {
36+
GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush")
37+
// 4 blocks of 4096 rows -> 4 segments, each larger than one 2048-row cache granule.
38+
// Every block has keys from the whole range, so the segments overlap.
39+
sql "SET batch_size = 4096"
40+
sql """
41+
INSERT INTO ${tbl}
42+
SELECT (number * 7919) % 16384, (number * 7919) % 16384 % 7 - 3
43+
FROM numbers('number' = '16384')
44+
"""
45+
} finally {
46+
GetDebugPoint().disableDebugPointForAllBEs("MemTable.need_flush")
47+
}
48+
49+
def tablets = sql_return_maparray("SHOW TABLETS FROM ${tbl}")
50+
def (code, out, err) = curl("GET", tablets[0].MetaUrl)
51+
assertEquals(0, code)
52+
def hasMergeLayout = parseJson(out.trim()).rs_metas.any { meta ->
53+
meta.segments_overlap_pb == "OVERLAPPING" && meta.num_segments > 1
54+
}
55+
assertTrue(hasMergeLayout, "expected an OVERLAPPING rowset with more than one segment")
56+
57+
// Both switches are randomized in fuzzy runs; this case needs both on.
58+
sql "SET enable_condition_cache = true"
59+
sql "SET enable_segment_limit_pushdown = true"
60+
// Disable TopN runtime filters so the queries exercise storage TopN and condition cache.
61+
sql "SET topn_filter_ratio = 0"
62+
// Keep each segment on the merge path that requests EOF after its local LIMIT.
63+
sql "SET enable_parallel_scan = false"
64+
explain {
65+
sql "SELECT k, v FROM ${tbl} WHERE abs(v) > 0 ORDER BY k LIMIT 1"
66+
contains "SORT LIMIT: 1"
67+
notContains "TOPN filter targets"
68+
}
69+
70+
// Each case runs the TopN first, then the same predicate without LIMIT.
71+
qt_topn_asc "SELECT k, v FROM ${tbl} WHERE abs(v) > 0 ORDER BY k LIMIT 1"
72+
qt_all_after_asc "SELECT count(*), sum(k) FROM ${tbl} WHERE abs(v) > 0"
73+
74+
qt_topn_desc "SELECT k, v FROM ${tbl} WHERE abs(v) > 1 ORDER BY k DESC LIMIT 1"
75+
qt_all_after_desc "SELECT count(*), sum(k) FROM ${tbl} WHERE abs(v) > 1"
76+
}
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
// A predicate that references no column, such as random() < 2, is not pushed into the storage
19+
// layer and stays in the scanner. COUNT_ON_INDEX must still count with it instead of failing.
20+
suite("test_count_on_index_slotless_predicate", "p0") {
21+
def tbl = "test_count_on_index_slotless_predicate"
22+
sql "DROP TABLE IF EXISTS ${tbl}"
23+
sql """
24+
CREATE TABLE ${tbl} (
25+
k INT,
26+
v STRING,
27+
INDEX idx_v (v) USING INVERTED PROPERTIES ('parser' = 'english')
28+
) DUPLICATE KEY(k)
29+
DISTRIBUTED BY HASH(k) BUCKETS 1
30+
PROPERTIES ('replication_num' = '1')
31+
"""
32+
sql "INSERT INTO ${tbl} VALUES (1, 'apple pie'), (2, 'banana'), (3, 'apple juice')"
33+
34+
sql "SET enable_count_on_index_pushdown = true"
35+
explain {
36+
sql "SELECT count(*) FROM ${tbl} WHERE random() < 2"
37+
contains "pushAggOp=COUNT_ON_INDEX"
38+
}
39+
explain {
40+
sql "SELECT count(*) FROM ${tbl} WHERE v MATCH_ANY 'apple' AND random() < 2"
41+
contains "pushAggOp=COUNT_ON_INDEX"
42+
}
43+
44+
// random() returns a value in [0, 1), so "< 2" keeps every row and "< 0" keeps none.
45+
qt_all_rows "SELECT count(*) FROM ${tbl} WHERE random() < 2"
46+
qt_no_rows "SELECT count(*) FROM ${tbl} WHERE random() < 0"
47+
qt_match_all_rows "SELECT count(*) FROM ${tbl} WHERE v MATCH_ANY 'apple' AND random() < 2"
48+
qt_match_no_rows "SELECT count(*) FROM ${tbl} WHERE v MATCH_ANY 'apple' AND random() < 0"
49+
}

0 commit comments

Comments
 (0)