Skip to content

Commit 652ccc7

Browse files
authored
[pick](topn) Sync and pick topn from doris master (apache#1431)
Issue Number: close http://jira.selectdb-in.cc/browse/CORE-1462 Describe the overview of changes. commit e1697741a82f875ca42b0d18caa7972eaa225bee Author: Kang <[email protected]> Date: Thu Jan 19 22:59:29 2023 +0800 [opt](test) scalar_types_p0 use 100k lines dataset and scalar_types_p2 use 1000k (apache#16104) commit 33a47e8d02644123ffd8c5c4353653c1c175e96a Author: Kang <[email protected]> Date: Wed Jan 18 14:17:24 2023 +0800 [testcase](bitmap index)bitmap index testcase (apache#15975) * add bitmap index testcases for all scalar types commit 260a631441834ca7e23da4b77c922eb818eddca7 Author: Kang <[email protected]> Date: Mon Jan 16 16:49:59 2023 +0800 [regression-test](topn)add test cases for nonkey topn query for each scalar type (apache#15790) related to apache#15558 apache#15693 1. dup key table with 17 scalar datatypes 2. unique key table with mow enabled 3. unique key table with mow disabled commit 81cea5219ae86df950f10aa123072df78c7cdf23 Author: Kang <[email protected]> Date: Sun Feb 19 23:28:33 2023 +0800 [bugfix](topn) fix topn read_orderby_key_columns nullptr (apache#16896) The SQL `SELECT nationkey FROM regression_test_query_p0_limit.tpch_tiny_nation ORDER BY nationkey DESC LIMIT 5` make be core dump since dereference a nullptr `read_orderby_key_columns in VCollectIterator::_topn_next`, triggered by skipping _colname_to_value_range init in apache#16818 . This PR makes two changes: 1. avoid read_orderby_key_columns nullptr in TabletReader::_init_orderby_keys_param 2. return error if read_orderby_key_columns is nullptr unexpected in VCollectIterator::_topn_next to avoid core dump commit 2fee1d1d79942e49eddaafdc2b49e49b0651b109 Author: Kang <[email protected]> Date: Fri Feb 10 12:56:33 2023 +0800 [Improvement](topn) add limit threashold session variable and fuzzy for topn optimizations (apache#16514) 1. add limit threshold for topn runtime pushdown and key topn optimization 2. use unified session variable topn_opt_limit_threshold for all topn optimizations 3. add fuzzy support for topn_opt_limit_threshold commit 1696bed39129fcc891f32f64ff1fb43f9531fcd4 Author: Kang <[email protected]> Date: Thu Feb 2 09:13:32 2023 +0800 [bugfix](topn) fix topn runtime predicate getting value bug for decimal type (apache#16331) * fix topn runtime predicate getting value bug for decimal type * fix cast_to_string bug for TYPE_DECIMALV2 commit d70cdf61521a23417c9bc734a3cdb668265a15b0 Author: Kang <[email protected]> Date: Wed Feb 22 16:18:46 2023 +0800 topn sync doris order by key topn query optimization apache#15663 commit 1df514c8f0b66ae9a8438617163a31848e519949 Author: Kang <[email protected]> Date: Wed Feb 22 15:14:43 2023 +0800 sync with doris runtime prune for topn query apache#15558
1 parent 159a48f commit 652ccc7

705 files changed

Lines changed: 25377 additions & 208 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/olap/reader.cpp‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -414,10 +414,6 @@ Status TabletReader::_init_keys_param(const ReaderParams& read_params) {
414414
}
415415

416416
Status TabletReader::_init_orderby_keys_param(const ReaderParams& read_params) {
417-
if (read_params.start_key.empty()) {
418-
return Status::OK();
419-
}
420-
421417
// UNIQUE_KEYS will compare all keys as before
422418
if (_tablet_schema->keys_type() == DUP_KEYS ||
423419
(_tablet_schema->keys_type() == UNIQUE_KEYS &&

‎be/src/runtime/runtime_predicate.cpp‎

Lines changed: 88 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -16,160 +16,142 @@
1616
// under the License.
1717

1818
#include "runtime/runtime_predicate.h"
19+
1920
#include "olap/predicate_creator.h"
20-
#include "olap/column_predicate.h"
2121

2222
namespace doris {
2323

2424
namespace vectorized {
2525

26-
std::shared_ptr<ColumnPredicate> RuntimePredicate::get_predictate() {
27-
std::shared_lock<std::shared_mutex> rlock(_rwlock);
28-
return _predictate;
29-
}
30-
31-
Status RuntimePredicate::update(std::vector<Field>& values, const String& col_name,
32-
const TypeIndex type, bool is_reverse) {
26+
Status RuntimePredicate::init(const PrimitiveType type) {
3327
std::unique_lock<std::shared_mutex> wlock(_rwlock);
3428

35-
if (!_predicate_mem_pool) {
36-
_predicate_mem_pool.reset(new MemPool());
37-
}
38-
39-
bool updated = false;
40-
41-
if (UNLIKELY(_orderby_extrems.size() == 0)) {
42-
_orderby_extrems.push_back(values[0]);
43-
updated = true;
44-
} else if (is_reverse) {
45-
if (values[0] > _orderby_extrems[0]) {
46-
_orderby_extrems[0] = values[0];
47-
updated = true;
48-
}
49-
} else {
50-
if (values[0] < _orderby_extrems[0]) {
51-
_orderby_extrems[0] = values[0];
52-
updated = true;
53-
}
54-
}
55-
56-
if (!updated)
29+
if (_inited) {
5730
return Status::OK();
31+
}
5832

59-
TCondition condition;
60-
condition.__set_column_name(col_name);
61-
condition.__set_column_unique_id(_tablet_schema->column(col_name).unique_id());
62-
condition.__set_condition_op(is_reverse ? ">=" : "<=");
63-
64-
Field& field = _orderby_extrems[0];
65-
String str_value;
66-
int scale = 0;
33+
_predicate_mem_pool.reset(new MemPool());
6734

35+
// set get value function
6836
switch (type) {
69-
case TypeIndex::UInt8: {
70-
using ValueType = typename PrimitiveTypeTraits<TYPE_BOOLEAN>::CppType;
71-
str_value = cast_to_string<TYPE_BOOLEAN, ValueType>(field.get<ValueType>(), scale);
37+
case PrimitiveType::TYPE_BOOLEAN: {
38+
_get_value_fn = get_bool_value;
39+
break;
40+
}
41+
case PrimitiveType::TYPE_TINYINT: {
42+
_get_value_fn = get_tinyint_value;
7243
break;
7344
}
74-
case TypeIndex::Int8: {
75-
using ValueType = typename PrimitiveTypeTraits<TYPE_TINYINT>::CppType;
76-
str_value = cast_to_string<TYPE_TINYINT, ValueType>(field.get<ValueType>(), scale);
45+
case PrimitiveType::TYPE_SMALLINT: {
46+
_get_value_fn = get_smallint_value;
7747
break;
7848
}
79-
case TypeIndex::Int16: {
80-
using ValueType = typename PrimitiveTypeTraits<TYPE_SMALLINT>::CppType;
81-
str_value = cast_to_string<TYPE_SMALLINT, ValueType>(field.get<ValueType>(), scale);
49+
case PrimitiveType::TYPE_INT: {
50+
_get_value_fn = get_int_value;
8251
break;
8352
}
84-
case TypeIndex::Int32: {
85-
using ValueType = typename PrimitiveTypeTraits<TYPE_INT>::CppType;
86-
str_value = cast_to_string<TYPE_INT, ValueType>(field.get<ValueType>(), scale);
53+
case PrimitiveType::TYPE_BIGINT: {
54+
_get_value_fn = get_bigint_value;
8755
break;
8856
}
89-
case TypeIndex::Int64: {
90-
using ValueType = typename PrimitiveTypeTraits<TYPE_BIGINT>::CppType;
91-
str_value = cast_to_string<TYPE_BIGINT, ValueType>(field.get<ValueType>(), scale);
57+
case PrimitiveType::TYPE_LARGEINT: {
58+
_get_value_fn = get_largeint_value;
9259
break;
9360
}
94-
case TypeIndex::Int128: {
95-
using ValueType = typename PrimitiveTypeTraits<TYPE_LARGEINT>::CppType;
96-
str_value = cast_to_string<TYPE_LARGEINT, ValueType>(field.get<ValueType>(), scale);
61+
case PrimitiveType::TYPE_FLOAT: {
62+
_get_value_fn = get_float_value;
9763
break;
9864
}
99-
case TypeIndex::Float32: {
100-
using ValueType = typename PrimitiveTypeTraits<TYPE_FLOAT>::CppType;
101-
str_value = cast_to_string<TYPE_FLOAT, ValueType>(field.get<ValueType>(), scale);
65+
case PrimitiveType::TYPE_DOUBLE: {
66+
_get_value_fn = get_double_value;
10267
break;
10368
}
104-
case TypeIndex::Float64: {
105-
using ValueType = typename PrimitiveTypeTraits<TYPE_DOUBLE>::CppType;
106-
str_value = cast_to_string<TYPE_DOUBLE, ValueType>(field.get<ValueType>(), scale);
69+
case PrimitiveType::TYPE_STRING: {
70+
_get_value_fn = get_string_value;
10771
break;
10872
}
109-
case TypeIndex::String: {
110-
str_value = field.get<String>();
73+
case PrimitiveType::TYPE_DATEV2: {
74+
_get_value_fn = get_datev2_value;
11175
break;
11276
}
113-
case TypeIndex::DateV2: {
114-
using ValueType = typename PrimitiveTypeTraits<TYPE_DATEV2>::CppType;
115-
str_value = cast_to_string<TYPE_DATEV2, ValueType>(
116-
binary_cast<UInt32, ValueType>(field.get<UInt32>()), scale);
77+
case PrimitiveType::TYPE_DATETIMEV2: {
78+
_get_value_fn = get_datetimev2_value;
11779
break;
11880
}
119-
case TypeIndex::DateTimeV2: {
120-
using ValueType = typename PrimitiveTypeTraits<TYPE_DATETIMEV2>::CppType;
121-
str_value = cast_to_string<TYPE_DATETIMEV2, ValueType>(
122-
binary_cast<UInt64, ValueType>(field.get<UInt64>()), scale);
81+
case PrimitiveType::TYPE_DATE: {
82+
_get_value_fn = get_date_value;
12383
break;
12484
}
125-
case TypeIndex::Date: {
126-
using ValueType = typename PrimitiveTypeTraits<TYPE_DATE>::CppType;
127-
ValueType value;
128-
Int64 v = field.get<Int64>();
129-
VecDateTimeValue* p = (VecDateTimeValue*)&v;
130-
value.from_olap_date(p->to_olap_date());
131-
value.cast_to_date();
132-
str_value = cast_to_string<TYPE_DATE, ValueType>(value, scale);
85+
case PrimitiveType::TYPE_DATETIME: {
86+
_get_value_fn = get_datetime_value;
13387
break;
13488
}
135-
case TypeIndex::DateTime: {
136-
using ValueType = typename PrimitiveTypeTraits<TYPE_DATETIME>::CppType;
137-
ValueType value;
138-
Int64 v = field.get<Int64>();
139-
VecDateTimeValue* p = (VecDateTimeValue*)&v;
140-
value.from_olap_datetime(p->to_olap_datetime());
141-
value.to_datetime();
142-
str_value = cast_to_string<TYPE_DATETIME, ValueType>(value, scale);
89+
case PrimitiveType::TYPE_DECIMAL32: {
90+
_get_value_fn = get_decimal32_value;
14391
break;
14492
}
145-
case TypeIndex::Decimal128: {
146-
using ValueType = typename PrimitiveTypeTraits<TYPE_DECIMALV2>::CppType;
147-
ValueType value;
148-
auto v = field.get<DecimalField<Decimal128>>();
149-
value.from_olap_decimal(v.get_value(), v.get_scale());
150-
scale = v.get_scale();
151-
str_value = cast_to_string<TYPE_DECIMALV2, ValueType>(value, scale);
93+
case PrimitiveType::TYPE_DECIMAL64: {
94+
_get_value_fn = get_decimal64_value;
15295
break;
15396
}
154-
case TypeIndex::Decimal32: {
155-
using ValueType = typename PrimitiveTypeTraits<TYPE_DECIMAL32>::CppType;
156-
ValueType value = field.get<ValueType>();
157-
str_value = cast_to_string<TYPE_DECIMAL32, ValueType>(value, scale);
97+
case PrimitiveType::TYPE_DECIMALV2: {
98+
_get_value_fn = get_decimalv2_value;
15899
break;
159100
}
160-
case TypeIndex::Decimal64: {
161-
using ValueType = typename PrimitiveTypeTraits<TYPE_DECIMAL64>::CppType;
162-
ValueType value = field.get<ValueType>();
163-
str_value = cast_to_string<TYPE_DECIMAL64, ValueType>(value, scale);
101+
case PrimitiveType::TYPE_DECIMAL128I: {
102+
_get_value_fn = get_decimal128_value;
164103
break;
165104
}
166105
default:
167-
return Status::InvalidArgument("unsupported type {}", type);
106+
return Status::InvalidArgument("unsupported runtime predicate type {}", type);
168107
}
169108

170-
condition.condition_values.push_back(str_value);
109+
_inited = true;
110+
return Status::OK();
111+
}
112+
113+
Status RuntimePredicate::update(const Field& value, const String& col_name, bool is_reverse) {
114+
std::unique_lock<std::shared_mutex> wlock(_rwlock);
115+
116+
// TODO why null
117+
if (!_tablet_schema) {
118+
return Status::OK();
119+
}
120+
121+
bool updated = false;
122+
123+
if (UNLIKELY(_orderby_extrem.is_null())) {
124+
_orderby_extrem = value;
125+
updated = true;
126+
} else if (is_reverse) {
127+
if (value > _orderby_extrem) {
128+
_orderby_extrem = value;
129+
updated = true;
130+
}
131+
} else {
132+
if (value < _orderby_extrem) {
133+
_orderby_extrem = value;
134+
updated = true;
135+
}
136+
}
137+
138+
if (!updated) {
139+
return Status::OK();
140+
}
141+
142+
TCondition condition;
143+
condition.__set_column_name(col_name);
144+
condition.__set_column_unique_id(_tablet_schema->column(col_name).unique_id());
145+
condition.__set_condition_op(is_reverse ? ">=" : "<=");
146+
147+
// get value string from _orderby_extrem and push back to condition_values
148+
condition.condition_values.push_back(_get_value_fn(_orderby_extrem));
149+
150+
VLOG_DEBUG << "update runtime predicate condition " << condition;
171151

172-
_predictate.reset(parse_to_predicate(_tablet_schema, condition, _predicate_mem_pool.get(), false));
152+
// update _predictate
153+
_predictate.reset(
154+
parse_to_predicate(_tablet_schema, condition, _predicate_mem_pool.get(), false));
173155

174156
return Status::OK();
175157
}

0 commit comments

Comments
 (0)