Skip to content

Commit 916b4e7

Browse files
committed
[fix](storage) storage should aggregate on replace column
We can skip aggregate on replace column, otherwise it would generate wrong result. e.g. a row in UNIQUE is deleted by delte_sign_column, then it would be returned.
1 parent eb3e0b2 commit 916b4e7

6 files changed

Lines changed: 117 additions & 28 deletions

File tree

‎be/src/exec/olap_scanner.cpp‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ Status OlapScanner::_init_tablet_reader_params(
167167
const std::vector<std::pair<string, std::shared_ptr<IBloomFilterFuncBase>>>& bloom_filters,
168168
const std::vector<FunctionFilter>& function_filters) {
169169
// if the table with rowset [0-x] or [0-1] [2-y], and [0-1] is empty
170-
bool single_version =
170+
bool single_non_overlapping_version =
171171
(_tablet_reader_params.rs_readers.size() == 1 &&
172172
_tablet_reader_params.rs_readers[0]->rowset()->start_version() == 0 &&
173173
!_tablet_reader_params.rs_readers[0]
@@ -182,14 +182,14 @@ Status OlapScanner::_init_tablet_reader_params(
182182
->rowset_meta()
183183
->is_segments_overlapping());
184184

185-
_tablet_reader_params.direct_mode = single_version || _aggregation;
186-
187-
RETURN_IF_ERROR(_init_return_columns(!_tablet_reader_params.direct_mode));
185+
bool has_replace_col = false;
186+
RETURN_IF_ERROR(_init_return_columns(!single_non_overlapping_version, has_replace_col));
188187

188+
_tablet_reader_params.aggregation = has_replace_col ? false : _aggregation;
189+
_tablet_reader_params.direct_mode = single_non_overlapping_version || _tablet_reader_params.aggregation;
189190
_tablet_reader_params.tablet = _tablet;
190191
_tablet_reader_params.tablet_schema = _tablet_schema;
191192
_tablet_reader_params.reader_type = READER_QUERY;
192-
_tablet_reader_params.aggregation = _aggregation;
193193
_tablet_reader_params.version = Version(0, _version);
194194

195195
// Condition
@@ -271,7 +271,7 @@ Status OlapScanner::_init_tablet_reader_params(
271271
return Status::OK();
272272
}
273273

274-
Status OlapScanner::_init_return_columns(bool need_seq_col) {
274+
Status OlapScanner::_init_return_columns(bool need_seq_col, bool &has_replace_col) {
275275
for (auto slot : _tuple_desc->slots()) {
276276
if (!slot->is_materialized()) {
277277
continue;
@@ -291,20 +291,23 @@ Status OlapScanner::_init_return_columns(bool need_seq_col) {
291291
_query_slots.push_back(slot);
292292
}
293293

294-
// expand the sequence column
295-
if (_tablet_schema->has_sequence_col() && need_seq_col) {
296-
bool has_replace_col = false;
294+
has_replace_col = false;
295+
if (_tablet->keys_type() == UNIQUE_KEYS || _tablet->keys_type() == AGG_KEYS) {
297296
for (auto col : _return_columns) {
298297
if (_tablet_schema->column(col).aggregation() ==
299298
FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE) {
300299
has_replace_col = true;
301300
break;
302301
}
303302
}
304-
if (auto sequence_col_idx = _tablet_schema->sequence_col_idx();
305-
has_replace_col && std::find(_return_columns.begin(), _return_columns.end(),
306-
sequence_col_idx) == _return_columns.end()) {
307-
_return_columns.push_back(sequence_col_idx);
303+
304+
// handle SEQ
305+
if (_tablet_schema->has_sequence_col() && need_seq_col && has_replace_col) {
306+
auto sequence_col_idx = _tablet_schema->sequence_col_idx();
307+
if (std::find(_return_columns.begin(), _return_columns.end(),
308+
sequence_col_idx) == _return_columns.end()) {
309+
_return_columns.push_back(sequence_col_idx);
310+
}
308311
}
309312
}
310313

‎be/src/exec/olap_scanner.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ class OlapScanner {
9898
const std::vector<std::pair<string, std::shared_ptr<IBloomFilterFuncBase>>>&
9999
bloom_filters,
100100
const std::vector<FunctionFilter>& function_filters);
101-
Status _init_return_columns(bool need_seq_col);
101+
Status _init_return_columns(bool need_seq_col, bool& has_replace_col);
102102
void _convert_row_to_tuple(Tuple* tuple);
103103

104104
// Update profile that need to be reported in realtime.

‎be/src/vec/exec/volap_scanner.cpp‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ Status VOlapScanner::_init_tablet_reader_params(
156156
const std::vector<std::pair<string, std::shared_ptr<IBloomFilterFuncBase>>>& bloom_filters,
157157
const std::vector<FunctionFilter>& function_filters) {
158158
// if the table with rowset [0-x] or [0-1] [2-y], and [0-1] is empty
159-
bool single_version =
159+
bool single_non_overlapping_version =
160160
(_tablet_reader_params.rs_readers.size() == 1 &&
161161
_tablet_reader_params.rs_readers[0]->rowset()->start_version() == 0 &&
162162
!_tablet_reader_params.rs_readers[0]
@@ -171,14 +171,14 @@ Status VOlapScanner::_init_tablet_reader_params(
171171
->rowset_meta()
172172
->is_segments_overlapping());
173173

174-
_tablet_reader_params.direct_mode = _aggregation || single_version;
175-
176-
RETURN_IF_ERROR(_init_return_columns(!_tablet_reader_params.direct_mode));
174+
bool has_replace_col = false;
175+
RETURN_IF_ERROR(_init_return_columns(!single_non_overlapping_version, has_replace_col));
177176

177+
_tablet_reader_params.aggregation = has_replace_col ? false : _aggregation;
178+
_tablet_reader_params.direct_mode = single_non_overlapping_version || _tablet_reader_params.aggregation;
178179
_tablet_reader_params.tablet = _tablet;
179180
_tablet_reader_params.tablet_schema = _tablet_schema;
180181
_tablet_reader_params.reader_type = READER_QUERY;
181-
_tablet_reader_params.aggregation = _aggregation;
182182
_tablet_reader_params.version = Version(0, _version);
183183

184184
// Condition
@@ -266,7 +266,7 @@ Status VOlapScanner::_init_tablet_reader_params(
266266
return Status::OK();
267267
}
268268

269-
Status VOlapScanner::_init_return_columns(bool need_seq_col) {
269+
Status VOlapScanner::_init_return_columns(bool need_seq_col, bool& has_replace_col) {
270270
for (auto slot : _tuple_desc->slots()) {
271271
if (!slot->is_materialized()) {
272272
continue;
@@ -288,20 +288,23 @@ Status VOlapScanner::_init_return_columns(bool need_seq_col) {
288288
}
289289
}
290290

291-
// expand the sequence column
292-
if (_tablet_schema->has_sequence_col() && need_seq_col) {
293-
bool has_replace_col = false;
291+
has_replace_col = false;
292+
if (_tablet->keys_type() == UNIQUE_KEYS || _tablet->keys_type() == AGG_KEYS) {
294293
for (auto col : _return_columns) {
295294
if (_tablet_schema->column(col).aggregation() ==
296295
FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE) {
297296
has_replace_col = true;
298297
break;
299298
}
300299
}
301-
if (auto sequence_col_idx = _tablet_schema->sequence_col_idx();
302-
has_replace_col && std::find(_return_columns.begin(), _return_columns.end(),
303-
sequence_col_idx) == _return_columns.end()) {
304-
_return_columns.push_back(sequence_col_idx);
300+
301+
// handle SEQ
302+
if (_tablet_schema->has_sequence_col() && need_seq_col && has_replace_col) {
303+
auto sequence_col_idx = _tablet_schema->sequence_col_idx();
304+
if (std::find(_return_columns.begin(), _return_columns.end(),
305+
sequence_col_idx) == _return_columns.end()) {
306+
_return_columns.push_back(sequence_col_idx);
307+
}
305308
}
306309
}
307310

‎be/src/vec/exec/volap_scanner.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ class VOlapScanner {
9797
const std::vector<std::pair<string, std::shared_ptr<IBloomFilterFuncBase>>>&
9898
bloom_filters,
9999
const std::vector<FunctionFilter>& function_filters);
100-
Status _init_return_columns(bool need_seq_col);
100+
Status _init_return_columns(bool need_seq_col, bool &has_replace_col);
101101

102102
// Update profile that need to be reported in realtime.
103103
void _update_realtime_counter();
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !test_delete_sign --
3+
0
4+
5+
-- !test_delete_sign_2 --
6+
0
7+
8+
-- !test_delete_sign_3 --
9+
1
10+
11+
-- !test_delete_sign_4 --
12+
1
13+
14+
-- !test_delete_sign_5 --
15+
1
16+
17+
-- !test_delete_sign_6 --
18+
1
19+
20+
-- !test_delete_sign_7 --
21+
1
22+
23+
-- !test_delete_sign_8 --
24+
1
25+
26+
-- !test_delete_sign_9 --
27+
1
28+
29+
-- !test_delete_sign_10 --
30+
2
31+
32+
-- !test_delete_sign_11 --
33+
2
34+
35+
-- !test_delete_sign_12 --
36+
1
37+
38+
-- !test_delete_sign_13 --
39+
1
40+
41+
-- !test_delete_sign_14 --
42+
1
43+
44+
-- !test_delete_sign_15 --
45+
1
46+
47+
-- !test_delete_sign_16 --
48+
0
49+
50+
-- !test_delete_sign_17 --
51+
0
52+
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
DROP TABLE IF EXISTS delete_sign_test;
2+
3+
CREATE TABLE delete_sign_test (
4+
`uid` BIGINT NULL,
5+
`v1` BIGINT NULL
6+
)
7+
UNIQUE KEY(uid)
8+
DISTRIBUTED BY HASH(uid) BUCKETS 3
9+
PROPERTIES (
10+
"replication_num" = "1"
11+
);
12+
13+
insert into delete_sign_test values(1, 1);
14+
select count(uid) from delete_sign_test;
15+
select count(distinct uid) from delete_sign_test;
16+
17+
insert into delete_sign_test values(1, 1);
18+
select count(uid) from delete_sign_test;
19+
select count(distinct uid) from delete_sign_test;
20+
21+
insert into delete_sign_test values(2, 1);
22+
select count(uid) from delete_sign_test;
23+
select count(distinct uid) from delete_sign_test;
24+
25+
insert into delete_sign_test(uid, v1, __DORIS_DELETE_SIGN__) values(1, 1, 1);
26+
select count(uid) from delete_sign_test;
27+
select count(distinct uid) from delete_sign_test;
28+
29+
insert into delete_sign_test(uid, v1, __DORIS_DELETE_SIGN__) values(2, 1, 1);
30+
select count(uid) from delete_sign_test;
31+
select count(distinct uid) from delete_sign_test;

0 commit comments

Comments
 (0)