Skip to content

Commit 64446af

Browse files
committed
� This is a combination of 2 commits.
� This is the 1st commit message: Support Grouping Sets, Rollup and Cube to extend group by statement (apache#2039) [ADD] compatible with hive grouping clause (apache#2039) [ADD] support grouping functions in expr like grouping(a) + grouping(b) (apache#2039) [FIX] fix analyzer error in window function(apache#2039) � This is the commit message apache#2: Support Grouping Sets, Rollup and Cube to extend group by statement (apache#2039) [ADD] compatible with hive grouping clause (apache#2039) [ADD] support grouping functions in expr like grouping(a) + grouping(b) (apache#2039) [FIX] fix analyzer error in window function(apache#2039) clean some code style clean some code style [FIX] fix grouping functions not substitute in subquery(apache#2039) [ADD] add design docs [FIX] fix expr in groupby clause remove duplicate error[apache#2039] [FIX] fix code style problems add some comments [ADD] add fe unit test [FIX] fix according comments and add syntax check in where clause remve unused code fix by comments
1 parent e65a645 commit 64446af

53 files changed

Lines changed: 3870 additions & 414 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/common/daemon.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
#include "exprs/utility_functions.h"
5252
#include "exprs/json_functions.h"
5353
#include "exprs/hll_hash_function.h"
54+
#include "exprs/grouping_sets_functions.h"
5455
#include "exprs/timezone_db.h"
5556
#include "exprs/bitmap_function.h"
5657
#include "exprs/hll_function.h"
@@ -272,6 +273,7 @@ void init_daemon(int argc, char** argv, const std::vector<StorePath>& paths) {
272273
HllHashFunctions::init();
273274
ESFunctions::init();
274275
GeoFunctions::init();
276+
GroupingSetsFunctions::init();
275277
TimezoneDatabase::init();
276278
BitmapFunctions::init();
277279
HllFunctions::init();

‎be/src/exec/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ set(EXEC_FILES
7171
spill_sort_node.cc
7272
union_node.cpp
7373
union_node_ir.cpp
74+
repeat_node.cpp
7475
schema_scanner.cpp
7576
schema_scan_node.cpp
7677
schema_scanner/schema_tables_scanner.cpp

‎be/src/exec/exec_node.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
#include "exec/analytic_eval_node.h"
5151
#include "exec/select_node.h"
5252
#include "exec/union_node.h"
53+
#include "exec/repeat_node.h"
5354
#include "exec/assert_num_rows_node.h"
5455
#include "runtime/exec_env.h"
5556
#include "runtime/descriptors.h"
@@ -452,10 +453,14 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN
452453
*node = pool->add(new BrokerScanNode(pool, tnode, descs));
453454
return Status::OK();
454455

456+
case TPlanNodeType::REPEAT_NODE:
457+
*node = pool->add(new RepeatNode(pool, tnode, descs));
458+
return Status::OK();
459+
455460
case TPlanNodeType::ASSERT_NUM_ROWS_NODE:
456461
*node = pool->add(new AssertNumRowsNode(pool, tnode, descs));
457462
return Status::OK();
458-
463+
459464
default:
460465
map<int, const char*>::const_iterator i =
461466
_TPlanNodeType_VALUES_TO_NAMES.find(tnode.node_type);

‎be/src/exec/repeat_node.cpp‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "exec/repeat_node.h"
19+
20+
#include "exprs/expr.h"
21+
#include "runtime/raw_value.h"
22+
#include "runtime/row_batch.h"
23+
#include "runtime/runtime_state.h"
24+
#include "util/runtime_profile.h"
25+
#include "gutil/strings/join.h"
26+
27+
namespace doris {
28+
29+
RepeatNode::RepeatNode(ObjectPool* pool, const TPlanNode& tnode,
30+
const DescriptorTbl& descs)
31+
: ExecNode(pool, tnode, descs),
32+
_slot_id_set_list(tnode.repeat_node.slot_id_set_list),
33+
_repeat_id_list(tnode.repeat_node.repeat_id_list),
34+
_grouping_list(tnode.repeat_node.grouping_list),
35+
_output_tuple_id(tnode.repeat_node.output_tuple_id),
36+
_tuple_desc(nullptr),
37+
_child_row_batch(nullptr),
38+
_child_eos(false),
39+
_repeat_id_idx(0),
40+
_runtime_state(nullptr) {
41+
}
42+
43+
RepeatNode::~RepeatNode() {
44+
}
45+
46+
Status RepeatNode::prepare(RuntimeState* state) {
47+
SCOPED_TIMER(_runtime_profile->total_time_counter());
48+
RETURN_IF_ERROR(ExecNode::prepare(state));
49+
DCHECK(_repeat_id_idx >= 0);
50+
for (const std::vector<int64_t>& v : _grouping_list) {
51+
DCHECK(_repeat_id_idx <= (int)v.size());
52+
}
53+
_runtime_state = state;
54+
_tuple_desc = state->desc_tbl().get_tuple_descriptor(_output_tuple_id);
55+
if (_tuple_desc == NULL) {
56+
return Status::InternalError("Failed to get tuple descriptor.");
57+
}
58+
59+
return Status::OK();
60+
}
61+
62+
Status RepeatNode::open(RuntimeState* state) {
63+
SCOPED_TIMER(_runtime_profile->total_time_counter());
64+
RETURN_IF_ERROR(ExecNode::open(state));
65+
RETURN_IF_CANCELLED(state);
66+
RETURN_IF_ERROR(child(0)->open(state));
67+
return Status::OK();
68+
}
69+
70+
/**
71+
* copy the rows to new tuple based on repeat_id_idx and _repeat_id_list and fill in row_batch,
72+
* and then set grouping_id and other grouping function slot in child_row_batch
73+
* e.g. _repeat_id_list = [0, 3, 1, 2], _repeat_id_idx = 2, _grouping_list [[0, 3, 1, 2], [0, 1, 1, 0]],
74+
* row_batch tuple 0 ['a', 'b', 1] -> [['a', null, 1] tuple 1 [1, 1]]
75+
*/
76+
Status RepeatNode::get_repeated_batch(
77+
RowBatch* child_row_batch, int repeat_id_idx, RowBatch* row_batch) {
78+
79+
DCHECK(child_row_batch != nullptr);
80+
DCHECK_EQ(row_batch->num_rows(), 0);
81+
82+
// Fill all slots according to child
83+
MemPool* tuple_pool = row_batch->tuple_data_pool();
84+
const vector<TupleDescriptor*>& src_tuple_descs = child_row_batch->row_desc().tuple_descriptors();
85+
const vector<TupleDescriptor*>& dst_tuple_descs = row_batch->row_desc().tuple_descriptors();
86+
vector<Tuple*> dst_tuples(src_tuple_descs.size(), nullptr);
87+
for (int i = 0; i < child_row_batch->num_rows(); ++i) {
88+
int row_idx = row_batch->add_row();
89+
TupleRow* dst_row = row_batch->get_row(row_idx);
90+
TupleRow* src_row = child_row_batch->get_row(i);
91+
92+
auto src_it = src_tuple_descs.begin();
93+
auto dst_it = dst_tuple_descs.begin();
94+
for (int j = 0; src_it != src_tuple_descs.end() && dst_it != dst_tuple_descs.end();
95+
++src_it, ++dst_it, ++j) {
96+
Tuple* src_tuple = src_row->get_tuple(j);
97+
if (src_tuple == NULL) {
98+
continue;
99+
}
100+
101+
if (dst_tuples[j] == nullptr) {
102+
int size = row_batch->capacity() * (*dst_it)->byte_size();
103+
void* tuple_buffer = tuple_pool->allocate(size);
104+
if (tuple_buffer == nullptr) {
105+
return Status::InternalError("Allocate memory for row batch failed.");
106+
}
107+
dst_tuples[j] = reinterpret_cast<Tuple*>(tuple_buffer);
108+
} else {
109+
char* new_tuple = reinterpret_cast<char*>(dst_tuples[j]);
110+
new_tuple += (*dst_it)->byte_size();
111+
dst_tuples[j] = reinterpret_cast<Tuple*>(new_tuple);
112+
}
113+
dst_row->set_tuple(j, dst_tuples[j]);
114+
memset(dst_tuples[j], 0, (*dst_it)->num_null_bytes());
115+
src_tuple->deep_copy(dst_tuples[j], **dst_it, tuple_pool);
116+
for (int k = 0; k < (*src_it)->slots().size(); k++) {
117+
SlotDescriptor* src_slot_desc = (*src_it)->slots()[k];
118+
SlotDescriptor* dst_slot_desc = (*dst_it)->slots()[k];
119+
DCHECK_EQ(src_slot_desc->type().type, dst_slot_desc->type().type);
120+
DCHECK_EQ(src_slot_desc->col_name(), dst_slot_desc->col_name());
121+
// set null base on repeated list
122+
if (_slot_id_set_list[0].find(src_slot_desc->id()) != _slot_id_set_list[0].end()) {
123+
std::set<SlotId>& repeat_ids = _slot_id_set_list[repeat_id_idx];
124+
if (repeat_ids.find(src_slot_desc->id()) == repeat_ids.end()) {
125+
dst_tuples[j]->set_null(dst_slot_desc->null_indicator_offset());
126+
continue;
127+
}
128+
}
129+
}
130+
}
131+
row_batch->commit_last_row();
132+
}
133+
Tuple *tuple = nullptr;
134+
// Fill grouping ID to tuple
135+
for (int i = 0; i < child_row_batch->num_rows(); ++i) {
136+
int row_idx = i;
137+
TupleRow *row = row_batch->get_row(row_idx);
138+
139+
if (tuple == nullptr) {
140+
int size = row_batch->capacity() * _tuple_desc->byte_size();
141+
void *tuple_buffer = tuple_pool->allocate(size);
142+
if (tuple_buffer == nullptr) {
143+
return Status::InternalError("Allocate memory for row batch failed.");
144+
}
145+
tuple = reinterpret_cast<Tuple *>(tuple_buffer);
146+
} else {
147+
char *new_tuple = reinterpret_cast<char *>(tuple);
148+
new_tuple += _tuple_desc->byte_size();
149+
tuple = reinterpret_cast<Tuple *>(new_tuple);
150+
}
151+
152+
row->set_tuple(src_tuple_descs.size(), tuple);
153+
memset(tuple, 0, _tuple_desc->num_null_bytes());
154+
155+
for(size_t slot_idx = 0; slot_idx < _grouping_list.size(); ++slot_idx) {
156+
int64_t val = _grouping_list[slot_idx][repeat_id_idx];
157+
DCHECK_LT(slot_idx, _tuple_desc->slots().size()) << "TupleDescriptor: " << _tuple_desc->debug_string();
158+
const SlotDescriptor *slot_desc = _tuple_desc->slots()[slot_idx];
159+
tuple->set_not_null(slot_desc->null_indicator_offset());
160+
RawValue::write(&val, tuple, slot_desc, tuple_pool);
161+
}
162+
}
163+
164+
return Status::OK();
165+
}
166+
167+
Status RepeatNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) {
168+
SCOPED_TIMER(_runtime_profile->total_time_counter());
169+
RETURN_IF_CANCELLED(state);
170+
// current child has finished its repeat, get child's next batch
171+
if (_child_row_batch.get() == nullptr) {
172+
if (_child_eos) {
173+
*eos = true;
174+
return Status::OK();
175+
}
176+
177+
_child_row_batch.reset(
178+
new RowBatch(child(0)->row_desc(), state->batch_size(), mem_tracker()));
179+
RETURN_IF_ERROR(child(0)->get_next(state, _child_row_batch.get(), &_child_eos));
180+
181+
if (_child_row_batch->num_rows() <= 0) {
182+
_child_row_batch.reset(nullptr);
183+
*eos = true;
184+
return Status::OK();
185+
}
186+
}
187+
188+
DCHECK_EQ(row_batch->num_rows(), 0);
189+
RETURN_IF_ERROR(get_repeated_batch(_child_row_batch.get(), _repeat_id_idx, row_batch));
190+
_repeat_id_idx++;
191+
192+
int size = _repeat_id_list.size();
193+
if (_repeat_id_idx >= size) {
194+
_child_row_batch.reset(nullptr);
195+
_repeat_id_idx = 0;
196+
}
197+
198+
return Status::OK();
199+
}
200+
201+
Status RepeatNode::close(RuntimeState* state) {
202+
if (is_closed()) {
203+
return Status::OK();
204+
}
205+
_child_row_batch.reset(nullptr);
206+
RETURN_IF_ERROR(child(0)->close(state));
207+
return ExecNode::close(state);
208+
}
209+
210+
void RepeatNode::debug_string(int indentation_level, std::stringstream* out) const {
211+
*out << string(indentation_level * 2, ' ');
212+
*out << "RepeatNode(";
213+
*out << "repeat pattern: [" << JoinElements(_repeat_id_list, ",") << "]\n";
214+
*out << "add " << _grouping_list.size() << " columns. \n";
215+
*out << "added column values: ";
216+
for (const std::vector<int64_t> &v : _grouping_list ){
217+
*out << "[" << JoinElements(v, ",") << "] ";
218+
}
219+
*out << "\n";
220+
ExecNode::debug_string(indentation_level, out);
221+
*out << ")";
222+
}
223+
224+
}

‎be/src/exec/repeat_node.h‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include "exec/exec_node.h"
21+
22+
namespace doris {
23+
24+
class Tuple;
25+
class RuntimeState;
26+
class RowBatch;
27+
28+
// repeat tuple of children and set given slots to null, this class generates tuple rows according to the given
29+
// _repeat_id_list, and sets the value of the slot corresponding to the grouping function according to _grouping_list
30+
class RepeatNode : public ExecNode {
31+
public:
32+
RepeatNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs);
33+
~RepeatNode();
34+
35+
virtual Status prepare(RuntimeState* state) override;
36+
virtual Status open(RuntimeState* state) override;
37+
virtual Status get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) override;
38+
virtual Status close(RuntimeState* state) override;
39+
40+
protected:
41+
virtual void debug_string(int indentation_level, std::stringstream* out) const override;
42+
43+
private:
44+
Status get_repeated_batch(RowBatch* child_row_batch, int repeat_id_idx, RowBatch* row_batch);
45+
46+
// Slot id set used to indicate those slots need to set to null.
47+
std::vector<std::set<SlotId>> _slot_id_set_list;
48+
// An integer bitmap list, it indicates the bit position of the exprs not null.
49+
std::vector<int64_t> _repeat_id_list;
50+
std::vector<std::vector<int64_t>> _grouping_list;
51+
// Tulple id used for output, it has new slots.
52+
TupleId _output_tuple_id;
53+
const TupleDescriptor* _tuple_desc;
54+
55+
std::unique_ptr<RowBatch> _child_row_batch;
56+
bool _child_eos;
57+
int _repeat_id_idx;
58+
RuntimeState* _runtime_state;
59+
};
60+
61+
}

‎be/src/exprs/CMakeLists.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,4 +65,5 @@ add_library(Exprs
6565
agg_fn.cc
6666
new_agg_fn_evaluator.cc
6767
bitmap_function.cpp
68-
hll_function.cpp)
68+
hll_function.cpp
69+
grouping_sets_functions.cpp)
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "exprs/grouping_sets_functions.h"
19+
20+
namespace doris {
21+
22+
void GroupingSetsFunctions::init() {
23+
}
24+
25+
doris_udf::BigIntVal GroupingSetsFunctions::grouping_id(
26+
doris_udf::FunctionContext* ctx, const doris_udf::BigIntVal& grouping_id) {
27+
return grouping_id;
28+
}
29+
30+
BigIntVal GroupingSetsFunctions::grouping(
31+
doris_udf::FunctionContext* ctx, const doris_udf::BigIntVal& grouping) {
32+
return grouping;
33+
}
34+
35+
} // doris
36+

0 commit comments

Comments
 (0)