Skip to content

Commit c538edd

Browse files
committed
1. Add MemTracker In Sort/Scan/Exchange/Corss Join Node (apache#90)
2. Opt the mem usage of TOPN node
1 parent 259728a commit c538edd

18 files changed

Lines changed: 120 additions & 169 deletions

‎be/src/exec/cross_join_node.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ Status CrossJoinNode::construct_build_side(RuntimeState* state) {
6060
RETURN_IF_CANCELLED(state);
6161
// TODO(zhaochun):
6262
// RETURN_IF_ERROR(state->CheckQueryState());
63-
bool eos = true;
63+
bool eos = false;
6464
RETURN_IF_ERROR(child(1)->get_next(state, batch, &eos));
6565

6666
// to prevent use too many memory

‎be/src/vec/aggregate_functions/aggregate_function_count.h‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,7 @@
1717

1818
#pragma once
1919

20-
// #include <IO/VarInt.h>
21-
// #include <IO/WriteHelpers.h>
20+
#include <array>
2221

2322
#include "common/logging.h"
2423
#include <vec/aggregate_functions/aggregate_function.h>
@@ -27,8 +26,6 @@
2726
#include <vec/data_types/data_type_number.h>
2827
#include <vec/io/io_helper.h>
2928

30-
#include <array>
31-
3229
namespace doris::vectorized {
3330

3431
struct AggregateFunctionCountData {

‎be/src/vec/aggregate_functions/aggregate_function_distinct.cpp‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,15 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
#include "vec/aggregate_functions/aggregate_function_distinct.h"
19-
2018
#include <algorithm>
19+
#include <boost/algorithm/string.hpp>
2120

22-
#include "boost/algorithm/string.hpp"
2321
#include "vec/aggregate_functions/aggregate_function_combinator.h"
22+
#include "vec/aggregate_functions/aggregate_function_distinct.h"
2423
#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
2524
#include "vec/aggregate_functions/helpers.h"
2625
#include "vec/common/typeid_cast.h"
2726
#include "vec/data_types/data_type_nullable.h"
28-
// #include "registerAggregateFunctions.h"
2927

3028
namespace doris::vectorized {
3129
namespace ErrorCodes {
@@ -89,7 +87,6 @@ void register_aggregate_function_combinator_distinct(AggregateFunctionSimpleFact
8987
auto nested_function = factory.get(nested_function_name, transform_arguments, params);
9088
return function_combinator->transform_aggregate_function(nested_function, types, params);
9189
};
92-
factory.registerDistinctFunctionCombinator(creator, DISTINCT_FUNCTION_PREFIX);
93-
// factory.registerCombinator(std::make_shared<AggregateFunctionCombinatorDistinct>());
90+
factory.register_distinct_function_combinator(creator, DISTINCT_FUNCTION_PREFIX);
9491
}
9592
} // namespace doris::vectorized

‎be/src/vec/aggregate_functions/aggregate_function_distinct.h‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,6 @@
1919

2020
#include "vec/aggregate_functions/aggregate_function.h"
2121
#include "vec/aggregate_functions/key_holder_helpers.h"
22-
23-
// #include <Columns/ColumnArray.h>
24-
// #include <DataTypes/DataTypeArray.h>
2522
#include "vec/common/aggregation_common.h"
2623
#include "vec/common/assert_cast.h"
2724
#include "vec/common/field_visitors.h"
@@ -213,8 +210,6 @@ class AggregateFunctionDistinct
213210
bool allocates_memory_in_arena() const override { return true; }
214211

215212
const char* get_header_file_path() const override { return __FILE__; }
216-
217-
// AggregateFunctionPtr getNestedFunction() const override { return nested_func; }
218213
};
219214

220215
} // namespace doris::vectorized

‎be/src/vec/aggregate_functions/aggregate_function_simple_factory.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ class AggregateFunctionSimpleFactory {
6464
}
6565
}
6666

67-
void registerDistinctFunctionCombinator(Creator creator, const std::string& prefix) {
67+
void register_distinct_function_combinator(Creator creator, const std::string& prefix) {
6868
std::vector<std::string> need_insert;
6969
for (auto entity : aggregate_functions) {
7070
std::string target_value = prefix + entity.first;

‎be/src/vec/core/block.cpp‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -682,17 +682,13 @@ Status Block::filter_block(Block* block, int filter_column_id, int column_to_kee
682682
}
683683
filter_block_internal(block, filter, column_to_keep);
684684
} else if (auto* const_column = check_and_get_column<ColumnConst>(*filter_column)) {
685-
UInt64 ret = const_column->get_uint(0);
686-
if (ret == 0) {
687-
block->get_by_position(0).column = block->get_by_position(0).column->clone_empty();
688-
} else if (ret == 1) {
685+
bool ret = const_column->get_bool(0);
686+
if (ret) {
689687
for (size_t i = column_to_keep; i < block->columns(); ++i) {
690688
block->erase(i);
691689
}
692690
} else {
693-
std::stringstream ss;
694-
ss << "invalid const value for filter block, value = " << ret;
695-
return Status::InternalError(ss.str());
691+
block->get_by_position(0).column = block->get_by_position(0).column->clone_empty();
696692
}
697693
} else {
698694
const IColumn::Filter& filter =

‎be/src/vec/core/sort_block.cpp‎

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -114,12 +114,6 @@ void sort_block(Block& block, const SortDescription& description, UInt64 limit)
114114
: block.safe_get_by_position(description[0].column_number).column.get();
115115

116116
IColumn::Permutation perm;
117-
// if (needCollation(column, description[0]))
118-
// {
119-
// const ColumnString & column_string = typeid_cast<const ColumnString &>(*column);
120-
// column_string.get_permutation_with_collation(*description[0].collator, reverse, limit, perm);
121-
// }
122-
// else
123117
column->get_permutation(reverse, limit, description[0].nulls_direction, perm);
124118

125119
size_t columns = block.columns();
@@ -132,29 +126,8 @@ void sort_block(Block& block, const SortDescription& description, UInt64 limit)
132126

133127
if (limit >= size) limit = 0;
134128

135-
bool need_collation = false;
136129
ColumnsWithSortDescriptions columns_with_sort_desc =
137130
get_columns_with_sort_description(block, description);
138-
139-
// for (size_t i = 0, num_sort_columns = description.size(); i < num_sort_columns; ++i)
140-
// {
141-
// if (needCollation(columns_with_sort_desc[i].first, description[i]))
142-
// {
143-
// need_collation = true;
144-
// break;
145-
// }
146-
// }
147-
148-
// if (need_collation)
149-
// {
150-
// PartialSortingLessWithCollation less_with_collation(columns_with_sort_desc);
151-
//
152-
// if (limit)
153-
// std::partial_sort(perm.begin(), perm.begin() + limit, perm.end(), less_with_collation);
154-
// else
155-
// pdqsort(perm.begin(), perm.end(), less_with_collation);
156-
// }
157-
// else
158131
{
159132
PartialSortingLess less(columns_with_sort_desc);
160133

‎be/src/vec/core/sort_cursor.h‎

Lines changed: 44 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -41,14 +41,6 @@ struct SortCursorImpl {
4141
size_t pos = 0;
4242
size_t rows = 0;
4343

44-
/** Determines order if comparing columns are equal.
45-
* Order is determined by number of cursor.
46-
*
47-
* Cursor number (always?) equals to number of merging part.
48-
* Therefore this field can be used to determine part number of current row (see ColumnGathererStream).
49-
*/
50-
size_t order = 0;
51-
5244
using NeedCollationFlags = std::vector<UInt8>;
5345

5446
/** Should we use Collator to sort a column? */
@@ -59,18 +51,16 @@ struct SortCursorImpl {
5951

6052
SortCursorImpl() = default;
6153

62-
SortCursorImpl(const Block& block, const SortDescription& desc_, size_t order_ = 0)
54+
SortCursorImpl(const Block& block, const SortDescription& desc_)
6355
: desc(desc_),
6456
sort_columns_size(desc.size()),
65-
order(order_),
6657
need_collation(desc.size()) {
6758
reset(block);
6859
}
6960

70-
SortCursorImpl(const Columns& columns, const SortDescription& desc_, size_t order_ = 0)
61+
SortCursorImpl(const Columns& columns, const SortDescription& desc_)
7162
: desc(desc_),
7263
sort_columns_size(desc.size()),
73-
order(order_),
7464
need_collation(desc.size()) {
7565
for (auto& column_desc : desc) {
7666
if (!column_desc.column_name.empty()) {
@@ -101,9 +91,6 @@ struct SortCursorImpl {
10191
? block.get_position_by_name(column_desc.column_name)
10292
: column_desc.column_number;
10393
sort_columns.push_back(columns[column_number].get());
104-
105-
// need_collation[j] = desc[j].collator != nullptr && typeid_cast<const ColumnString *>(sort_columns.back()); /// TODO Nullable(String)
106-
// has_collation |= need_collation[j];
10794
}
10895

10996
pos = 0;
@@ -175,84 +162,67 @@ struct SortCursor {
175162
const SortCursorImpl* operator->() const { return impl; }
176163

177164
/// The specified row of this cursor is greater than the specified row of another cursor.
178-
bool greater_at(const SortCursor& rhs, size_t lhs_pos, size_t rhs_pos) const {
165+
int8_t greater_at(const SortCursor& rhs, size_t lhs_pos, size_t rhs_pos) const {
179166
for (size_t i = 0; i < impl->sort_columns_size; ++i) {
180167
int direction = impl->desc[i].direction;
181168
int nulls_direction = impl->desc[i].nulls_direction;
182169
int res = direction * impl->sort_columns[i]->compare_at(lhs_pos, rhs_pos,
183170
*(rhs.impl->sort_columns[i]),
184171
nulls_direction);
185-
if (res > 0) return true;
186-
if (res < 0) return false;
172+
if (res > 0) return 1;
173+
if (res < 0) return -1;
187174
}
188-
return impl->order > rhs.impl->order;
175+
return 0;
189176
}
190177

191178
/// Checks that all rows in the current block of this cursor are less than or equal to all the rows of the current block of another cursor.
192-
bool totally_less_or_equals(const SortCursor& rhs) const {
179+
bool totally_less(const SortCursor& rhs) const {
193180
if (impl->rows == 0 || rhs.impl->rows == 0) return false;
194181

195182
/// The last row of this cursor is no larger than the first row of the another cursor.
196-
return !greater_at(rhs, impl->rows - 1, 0);
183+
return greater_at(rhs, impl->rows - 1, 0) == -1;
197184
}
198185

199-
bool greater(const SortCursor& rhs) const { return greater_at(rhs, impl->pos, rhs.impl->pos); }
186+
bool greater(const SortCursor& rhs) const { return greater_at(rhs, impl->pos, rhs.impl->pos) > 0; }
200187

201188
/// Inverted so that the priority queue elements are removed in ascending order.
202189
bool operator<(const SortCursor& rhs) const { return greater(rhs); }
203190
};
204191

205-
/// Separate comparator for locale-sensitive string comparisons
206-
//struct SortCursorWithCollation
207-
//{
208-
// SortCursorImpl * impl;
209-
//
210-
// SortCursorWithCollation(SortCursorImpl * impl_) : impl(impl_) {}
211-
// SortCursorImpl * operator-> () { return impl; }
212-
// const SortCursorImpl * operator-> () const { return impl; }
213-
//
214-
// bool greater_at(const SortCursorWithCollation & rhs, size_t lhs_pos, size_t rhs_pos) const
215-
// {
216-
// for (size_t i = 0; i < impl->sort_columns_size; ++i)
217-
// {
218-
// int direction = impl->desc[i].direction;
219-
// int nulls_direction = impl->desc[i].nulls_direction;
220-
// int res;
221-
// if (impl->need_collation[i])
222-
// {
223-
// const ColumnString & column_string = assert_cast<const ColumnString &>(*impl->sort_columns[i]);
224-
// res = column_string.compare_at_with_collation(lhs_pos, rhs_pos, *(rhs.impl->sort_columns[i]), *impl->desc[i].collator);
225-
// }
226-
// else
227-
// res = impl->sort_columns[i]->compare_at(lhs_pos, rhs_pos, *(rhs.impl->sort_columns[i]), nulls_direction);
228-
//
229-
// res *= direction;
230-
// if (res > 0)
231-
// return true;
232-
// if (res < 0)
233-
// return false;
234-
// }
235-
// return impl->order > rhs.impl->order;
236-
// }
237-
//
238-
// bool totally_less_or_equals(const SortCursorWithCollation & rhs) const
239-
// {
240-
// if (impl->rows == 0 || rhs.impl->rows == 0)
241-
// return false;
242-
//
243-
// /// The last row of this cursor is no larger than the first row of the another cursor.
244-
// return !greater_at(rhs, impl->rows - 1, 0);
245-
// }
246-
//
247-
// bool greater(const SortCursorWithCollation & rhs) const
248-
// {
249-
// return greater_at(rhs, impl->pos, rhs.impl->pos);
250-
// }
251-
//
252-
// bool operator< (const SortCursorWithCollation & rhs) const
253-
// {
254-
// return greater(rhs);
255-
// }
256-
//};
192+
/// For easy copying.
193+
struct SortBlockCursor {
194+
SortCursorImpl* impl;
195+
196+
SortBlockCursor(SortCursorImpl* impl_) : impl(impl_) {}
197+
SortCursorImpl* operator->() { return impl; }
198+
const SortCursorImpl* operator->() const { return impl; }
199+
200+
// ~SortBlockCursor() { delete impl;}
201+
202+
/// The specified row of this cursor is greater than the specified row of another cursor.
203+
int8_t less_at(const SortBlockCursor& rhs, int rows) const {
204+
for (size_t i = 0; i < impl->sort_columns_size; ++i) {
205+
int direction = impl->desc[i].direction;
206+
int nulls_direction = impl->desc[i].nulls_direction;
207+
int res = direction * impl->sort_columns[i]->compare_at(rows, rhs->rows - 1,
208+
*(rhs.impl->sort_columns[i]),
209+
nulls_direction);
210+
if (res < 0) return 1;
211+
if (res > 0) return -1;
212+
}
213+
return 0;
214+
}
215+
216+
/// Checks that all rows in the current block of this cursor are less than or equal to all the rows of the current block of another cursor.
217+
bool totally_greater(const SortBlockCursor& rhs) const {
218+
if (impl->rows == 0 || rhs.impl->rows == 0) return false;
219+
220+
/// The last row of this cursor is no larger than the first row of the another cursor.
221+
return less_at(rhs, 0) == -1;
222+
}
223+
224+
/// Inverted so that the priority queue elements are removed in ascending order.
225+
bool operator<(const SortBlockCursor& rhs) const { return less_at(rhs, impl->rows - 1) >= 0; }
226+
};
257227

258228
} // namespace doris::vectorized

‎be/src/vec/data_types/data_type.h‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -233,13 +233,8 @@ class IDataType : private boost::noncopyable {
233233
virtual size_t serialize(const IColumn& column, PColumn* pcolumn) const = 0;
234234
virtual void deserialize(const PColumn& pcolumn, IColumn* column) const = 0;
235235

236-
// static String getFileNameForStream(const String & column_name, const SubstreamPath & path);
237-
238236
private:
239237
friend class DataTypeFactory;
240-
/** Customize this DataType
241-
*/
242-
// void setCustomization(DataTypeCustomDescPtr custom_desc_) const;
243238

244239
private:
245240
/** This is mutable to allow setting custom name and serialization on `const IDataType` post construction.

‎be/src/vec/data_types/data_type_nullable.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,8 @@ std::string DataTypeNullable::to_string(const IColumn& column, size_t row_num) c
5757
}
5858

5959
size_t DataTypeNullable::serialize(const IColumn& column, PColumn* pcolumn) const {
60-
const ColumnNullable& col =
61-
assert_cast<const ColumnNullable&>(*column.convert_to_full_column_if_const().get());
60+
auto ptr = column.convert_to_full_column_if_const();
61+
const ColumnNullable& col = assert_cast<const ColumnNullable&>(*ptr.get());
6262
pcolumn->mutable_is_null()->Reserve(column.size());
6363

6464
for (size_t i = 0; i < column.size(); ++i) {

0 commit comments

Comments
 (0)