Skip to content

Commit 1093017

Browse files
zclllyybbpull[bot]
authored andcommitted
Revert "[Feature](partitions) Support auto partition" (#24024)
* Revert "[Feature](partitions) Support auto partition (#23236)" This reverts commit 6c544dd. * Update config.h
1 parent ffb6d4d commit 1093017

47 files changed

Lines changed: 602 additions & 2411 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/common/config.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ DECLARE_mString(process_full_gc_size);
168168
// used memory and the exec_mem_limit will be canceled.
169169
// If false, cancel query when the memory used exceeds exec_mem_limit, same as before.
170170
DECLARE_mBool(enable_query_memory_overcommit);
171-
171+
//waibibabu
172172
// gc will release cache, cancel task, and task will wait for gc to release memory,
173173
// default gc strategy is conservative, if you want to exclude the interference of gc, let it be true
174174
DECLARE_mBool(disable_memory_gc);

‎be/src/common/status.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,6 @@ E(NOT_INITIALIZED, -236);
113113
E(ALREADY_CANCELLED, -237);
114114
E(TOO_MANY_SEGMENTS, -238);
115115
E(ALREADY_CLOSED, -239);
116-
E(NEED_SEND_AGAIN, -240);
117116
E(CE_CMD_PARAMS_ERROR, -300);
118117
E(CE_BUFFER_TOO_SMALL, -301);
119118
E(CE_CMD_NOT_VALID, -302);
@@ -285,7 +284,6 @@ constexpr bool capture_stacktrace(int code) {
285284
&& code != ErrorCode::TOO_MANY_VERSION
286285
&& code != ErrorCode::ALREADY_CANCELLED
287286
&& code != ErrorCode::ALREADY_CLOSED
288-
&& code != ErrorCode::NEED_SEND_AGAIN
289287
&& code != ErrorCode::PUSH_TRANSACTION_ALREADY_EXIST
290288
&& code != ErrorCode::BE_NO_SUITABLE_VERSION
291289
&& code != ErrorCode::CUMULATIVE_NO_SUITABLE_VERSION
@@ -429,7 +427,6 @@ class Status {
429427
ERROR_CTOR(DataQualityError, DATA_QUALITY_ERROR)
430428
ERROR_CTOR(NotAuthorized, NOT_AUTHORIZED)
431429
ERROR_CTOR(HttpError, HTTP_ERROR)
432-
ERROR_CTOR(NeedSendAgain, NEED_SEND_AGAIN)
433430
#undef ERROR_CTOR
434431

435432
template <int code>

‎be/src/exec/tablet_info.cpp‎

Lines changed: 59 additions & 223 deletions
Large diffs are not rendered by default.

‎be/src/exec/tablet_info.h‎

Lines changed: 27 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
#include <map>
2727
#include <memory>
2828
#include <string>
29-
#include <tuple>
3029
#include <unordered_map>
3130
#include <utility>
3231
#include <vector>
@@ -36,8 +35,6 @@
3635
#include "vec/columns/column.h"
3736
#include "vec/core/block.h"
3837
#include "vec/core/column_with_type_and_name.h"
39-
#include "vec/exprs/vexpr.h"
40-
#include "vec/exprs/vexpr_context.h"
4138
#include "vec/exprs/vexpr_fwd.h"
4239

4340
namespace doris {
@@ -113,8 +110,7 @@ using OlapTableIndexTablets = TOlapTableIndexTablets;
113110
// }
114111

115112
using BlockRow = std::pair<vectorized::Block*, int32_t>;
116-
using BlockRowWithIndicator =
117-
std::tuple<vectorized::Block*, int32_t, bool>; // [block, column, is_transformed]
113+
using VecBlock = vectorized::Block;
118114

119115
struct VOlapTablePartition {
120116
int64_t id = 0;
@@ -129,20 +125,32 @@ struct VOlapTablePartition {
129125
: start_key {partition_block, -1}, end_key {partition_block, -1} {}
130126
};
131127

132-
// this is only used by tablet_sink. so we can assume it's inited by its' descriptor.
133128
class VOlapTablePartKeyComparator {
134129
public:
135-
VOlapTablePartKeyComparator(const std::vector<uint16_t>& slot_locs,
136-
const std::vector<uint16_t>& params_locs)
137-
: _slot_locs(slot_locs), _param_locs(params_locs) {}
130+
VOlapTablePartKeyComparator(const std::vector<uint16_t>& slot_locs) : _slot_locs(slot_locs) {}
138131

139132
// return true if lhs < rhs
140-
// 'row' is -1 mean maximal boundary
141-
bool operator()(const BlockRowWithIndicator lhs, const BlockRowWithIndicator rhs) const;
133+
// 'row' is -1 mean
134+
bool operator()(const BlockRow* lhs, const BlockRow* rhs) const {
135+
if (lhs->second == -1) {
136+
return false;
137+
} else if (rhs->second == -1) {
138+
return true;
139+
}
140+
141+
for (auto slot_loc : _slot_locs) {
142+
auto res = lhs->first->get_by_position(slot_loc).column->compare_at(
143+
lhs->second, rhs->second, *rhs->first->get_by_position(slot_loc).column, -1);
144+
if (res != 0) {
145+
return res < 0;
146+
}
147+
}
148+
// equal, return false
149+
return false;
150+
}
142151

143152
private:
144153
const std::vector<uint16_t>& _slot_locs;
145-
const std::vector<uint16_t>& _param_locs;
146154
};
147155

148156
// store an olap table's tablet information
@@ -166,26 +174,6 @@ class VOlapTablePartitionParam {
166174

167175
const std::vector<VOlapTablePartition*>& get_partitions() const { return _partitions; }
168176

169-
// it's same with auto now because we only support transformed partition in auto partition. may expand in future
170-
bool is_projection_partition() const { return _is_auto_partiton; }
171-
bool is_auto_partition() const { return _is_auto_partiton; }
172-
173-
std::vector<uint16_t> get_partition_keys() const { return _partition_slot_locs; }
174-
175-
Status add_partitions(const std::vector<TOlapTablePartition>& partitions);
176-
177-
//TODO: use vector when we support multi partition column for auto-partition
178-
vectorized::VExprContextSPtr get_part_func_ctx() { return _part_func_ctx; }
179-
vectorized::VExprSPtr get_partition_function() { return _partition_function; }
180-
181-
// which will affect _partition_block
182-
Status generate_partition_from(const TOlapTablePartition& t_part,
183-
VOlapTablePartition*& part_result);
184-
185-
void set_transformed_slots(const std::vector<uint16_t>& new_slots) {
186-
_transformed_slot_locs = new_slots;
187-
}
188-
189177
private:
190178
Status _create_partition_keys(const std::vector<TExprNode>& t_exprs, BlockRow* part_key);
191179

@@ -194,40 +182,33 @@ class VOlapTablePartitionParam {
194182
std::function<uint32_t(BlockRow*, int64_t)> _compute_tablet_index;
195183

196184
// check if this partition contain this key
197-
bool _part_contains(VOlapTablePartition* part, BlockRowWithIndicator key) const;
185+
bool _part_contains(VOlapTablePartition* part, BlockRow* key) const {
186+
// start_key.second == -1 means only single partition
187+
VOlapTablePartKeyComparator comparator(_partition_slot_locs);
188+
return part->start_key.second == -1 || !comparator(key, &part->start_key);
189+
}
198190

199191
// this partition only valid in this schema
200192
std::shared_ptr<OlapTableSchemaParam> _schema;
201193
TOlapTablePartitionParam _t_param;
202194

203195
const std::vector<SlotDescriptor*>& _slots;
204196
std::vector<uint16_t> _partition_slot_locs;
205-
std::vector<uint16_t> _transformed_slot_locs;
206197
std::vector<uint16_t> _distributed_slot_locs;
207198

208199
ObjectPool _obj_pool;
209200
vectorized::Block _partition_block;
210201
std::unique_ptr<MemTracker> _mem_tracker;
211202
std::vector<VOlapTablePartition*> _partitions;
212-
// For all partition value rows saved in this map, indicator is false. whenever we use a value to find in it, the param is true.
213-
// so that we can distinguish which column index to use (origin slots or transformed slots).
214-
std::unique_ptr<
215-
std::map<BlockRowWithIndicator, VOlapTablePartition*, VOlapTablePartKeyComparator>>
203+
std::unique_ptr<std::map<BlockRow*, VOlapTablePartition*, VOlapTablePartKeyComparator>>
216204
_partitions_map;
217205

218206
bool _is_in_partition = false;
219207
uint32_t _mem_usage = 0;
220208
// only works when using list partition, the resource is owned by _partitions
221209
VOlapTablePartition* _default_partition = nullptr;
222-
223-
// for auto partition, now only support 1 column. TODO: use vector to save them when we support multi column auto-partition.
224-
bool _is_auto_partiton = false;
225-
vectorized::VExprContextSPtr _part_func_ctx = nullptr;
226-
vectorized::VExprSPtr _partition_function = nullptr;
227-
TPartitionType::type _part_type; // support list or range
228210
};
229211

230-
// indicate where's the tablet and all its replications (node-wise)
231212
using TabletLocation = TTabletLocation;
232213
// struct TTabletLocation {
233214
// 1: required i64 tablet_id
@@ -254,17 +235,9 @@ class OlapTableLocationParam {
254235
return nullptr;
255236
}
256237

257-
void add_locations(std::vector<TTabletLocation>& locations) {
258-
for (auto& location : locations) {
259-
if (_tablets.find(location.tablet_id) == _tablets.end()) {
260-
_tablets[location.tablet_id] = &location;
261-
}
262-
}
263-
}
264-
265238
private:
266239
TOlapTableLocationParam _t_param;
267-
// [tablet_id, tablet]. tablet has id, also.
240+
268241
std::unordered_map<int64_t, TabletLocation*> _tablets;
269242
};
270243

@@ -305,15 +278,6 @@ class DorisNodesInfo {
305278
return nullptr;
306279
}
307280

308-
void add_nodes(const std::vector<TNodeInfo>& t_nodes) {
309-
for (const auto& node : t_nodes) {
310-
auto node_info = find_node(node.id);
311-
if (node_info == nullptr) {
312-
_nodes.emplace(node.id, node);
313-
}
314-
}
315-
}
316-
317281
const std::unordered_map<int64_t, NodeInfo>& nodes_info() { return _nodes; }
318282

319283
private:

‎be/src/exprs/runtime_filter.cpp‎

Lines changed: 83 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,89 @@ PFilterType get_type(RuntimeFilterType type) {
193193
}
194194

195195
Status create_literal(const TypeDescriptor& type, const void* data, vectorized::VExprSPtr& expr) {
196-
TExprNode node = create_texpr_node_from(data, type.type, type.precision, type.scale);
196+
TExprNode node;
197+
198+
switch (type.type) {
199+
case TYPE_BOOLEAN: {
200+
create_texpr_literal_node<TYPE_BOOLEAN>(data, &node);
201+
break;
202+
}
203+
case TYPE_TINYINT: {
204+
create_texpr_literal_node<TYPE_TINYINT>(data, &node);
205+
break;
206+
}
207+
case TYPE_SMALLINT: {
208+
create_texpr_literal_node<TYPE_SMALLINT>(data, &node);
209+
break;
210+
}
211+
case TYPE_INT: {
212+
create_texpr_literal_node<TYPE_INT>(data, &node);
213+
break;
214+
}
215+
case TYPE_BIGINT: {
216+
create_texpr_literal_node<TYPE_BIGINT>(data, &node);
217+
break;
218+
}
219+
case TYPE_LARGEINT: {
220+
create_texpr_literal_node<TYPE_LARGEINT>(data, &node);
221+
break;
222+
}
223+
case TYPE_FLOAT: {
224+
create_texpr_literal_node<TYPE_FLOAT>(data, &node);
225+
break;
226+
}
227+
case TYPE_DOUBLE: {
228+
create_texpr_literal_node<TYPE_DOUBLE>(data, &node);
229+
break;
230+
}
231+
case TYPE_DATEV2: {
232+
create_texpr_literal_node<TYPE_DATEV2>(data, &node);
233+
break;
234+
}
235+
case TYPE_DATETIMEV2: {
236+
create_texpr_literal_node<TYPE_DATETIMEV2>(data, &node);
237+
break;
238+
}
239+
case TYPE_DATE: {
240+
create_texpr_literal_node<TYPE_DATE>(data, &node);
241+
break;
242+
}
243+
case TYPE_DATETIME: {
244+
create_texpr_literal_node<TYPE_DATETIME>(data, &node);
245+
break;
246+
}
247+
case TYPE_DECIMALV2: {
248+
create_texpr_literal_node<TYPE_DECIMALV2>(data, &node, type.precision, type.scale);
249+
break;
250+
}
251+
case TYPE_DECIMAL32: {
252+
create_texpr_literal_node<TYPE_DECIMAL32>(data, &node, type.precision, type.scale);
253+
break;
254+
}
255+
case TYPE_DECIMAL64: {
256+
create_texpr_literal_node<TYPE_DECIMAL64>(data, &node, type.precision, type.scale);
257+
break;
258+
}
259+
case TYPE_DECIMAL128I: {
260+
create_texpr_literal_node<TYPE_DECIMAL128I>(data, &node, type.precision, type.scale);
261+
break;
262+
}
263+
case TYPE_CHAR: {
264+
create_texpr_literal_node<TYPE_CHAR>(data, &node);
265+
break;
266+
}
267+
case TYPE_VARCHAR: {
268+
create_texpr_literal_node<TYPE_VARCHAR>(data, &node);
269+
break;
270+
}
271+
case TYPE_STRING: {
272+
create_texpr_literal_node<TYPE_STRING>(data, &node);
273+
break;
274+
}
275+
default:
276+
DCHECK(false);
277+
return Status::InvalidArgument("Invalid type!");
278+
}
197279

198280
try {
199281
expr = vectorized::VLiteral::create_shared(node);

0 commit comments

Comments
 (0)