Skip to content

Commit 2a99f63

Browse files
authored
[Function] Support vectorized like and regexp and function context (apache#182)
1 parent af3a3c1 commit 2a99f63

56 files changed

Lines changed: 1136 additions & 243 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/udf/udf.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,10 @@ void FunctionContextImpl::set_constant_args(const std::vector<doris_udf::AnyVal*
131131
_constant_args = constant_args;
132132
}
133133

134+
void FunctionContextImpl::set_constant_cols(const std::vector<doris::ColumnPtrWrapper*>& constant_cols) {
135+
_constant_cols = constant_cols;
136+
}
137+
134138
bool FunctionContextImpl::check_allocations_empty() {
135139
if (_allocations.empty() && _external_bytes_tracked == 0) {
136140
return true;
@@ -187,6 +191,7 @@ FunctionContext* FunctionContextImpl::clone(MemPool* pool) {
187191
create_context(_state, pool, _intermediate_type, _return_type, _arg_types,
188192
_varargs_buffer_size, _debug);
189193
new_context->_impl->_constant_args = _constant_args;
194+
new_context->_impl->_constant_cols = _constant_cols;
190195
new_context->_impl->_fragment_local_fn_state = _fragment_local_fn_state;
191196
return new_context;
192197
}

‎be/src/udf/udf.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
// object serves as the interface object between the UDF/UDA and the doris process.
2929
namespace doris {
3030
class FunctionContextImpl;
31+
class ColumnPtrWrapper;
3132
}
3233

3334
namespace doris_udf {
@@ -219,12 +220,16 @@ class FunctionContext {
219220
// FunctionContext* argument) is a constant (e.g. 5, "string", 1 + 1).
220221
bool is_arg_constant(int arg_idx) const;
221222

223+
bool is_col_constant(int arg_idx) const;
224+
222225
// Returns a pointer to the value of the arg_idx-th input argument (0 indexed, not
223226
// including the FunctionContext* argument). Returns NULL if the argument is not
224227
// constant. This function can be used to obtain user-specified constants in a UDF's
225228
// Init() or Close() functions.
226229
AnyVal* get_constant_arg(int arg_idx) const;
227230

231+
doris::ColumnPtrWrapper* get_constant_col(int arg_idx) const;
232+
228233
// Create a test FunctionContext object. The caller is responsible for calling delete
229234
// on it. This context has additional debugging validation enabled.
230235
static FunctionContext* create_test_context();

‎be/src/udf/udf_internal.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ namespace doris {
3232
class FreePool;
3333
class MemPool;
3434
class RuntimeState;
35+
class ColumnPtrWrapper;
3536

3637
// This class actually implements the interface of FunctionContext. This is split to
3738
// hide the details from the external header.
@@ -67,6 +68,8 @@ class FunctionContextImpl {
6768

6869
void set_constant_args(const std::vector<doris_udf::AnyVal*>& constant_args);
6970

71+
void set_constant_cols(const std::vector<doris::ColumnPtrWrapper*>& cols);
72+
7073
uint8_t* varargs_buffer() { return _varargs_buffer; }
7174

7275
std::vector<doris_udf::AnyVal*>* staging_input_vals() { return &_staging_input_vals; }
@@ -169,6 +172,8 @@ class FunctionContextImpl {
169172
// value of the argument.
170173
std::vector<doris_udf::AnyVal*> _constant_args;
171174

175+
std::vector<doris::ColumnPtrWrapper*> _constant_cols;
176+
172177
// Used by ScalarFnCall to store the arguments when running without codegen. Allows us
173178
// to pass AnyVal* arguments to the scalar function directly, rather than codegening a
174179
// call that passes the correct AnyVal subclass pointer type.

‎be/src/udf/udf_ir.cpp‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,27 @@ bool FunctionContext::is_arg_constant(int i) const {
2525
return _impl->_constant_args[i] != NULL;
2626
}
2727

28+
bool FunctionContext::is_col_constant(int i) const {
29+
if (i < 0 || i >= _impl->_constant_cols.size()) {
30+
return false;
31+
}
32+
return _impl->_constant_cols[i] != nullptr;
33+
}
34+
2835
AnyVal* FunctionContext::get_constant_arg(int i) const {
2936
if (i < 0 || i >= _impl->_constant_args.size()) {
3037
return NULL;
3138
}
3239
return _impl->_constant_args[i];
3340
}
3441

42+
doris::ColumnPtrWrapper* FunctionContext::get_constant_col(int i) const {
43+
if (i < 0 || i >= _impl->_constant_cols.size()) {
44+
return nullptr;
45+
}
46+
return _impl->_constant_cols[i];
47+
}
48+
3549
int FunctionContext::get_num_args() const {
3650
return _impl->_arg_types.size();
3751
}

‎be/src/vec/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ set(VEC_FILES
113113
functions/is_null.cpp
114114
functions/is_not_null.cpp
115115
functions/in.cpp
116+
functions/like.cpp
116117
functions/to_time_function.cpp
117118
functions/time_of_function.cpp
118119
functions/if.cpp

‎be/src/vec/columns/column.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -437,3 +437,12 @@ bool is_column_const(const IColumn& column);
437437
bool is_column_nullable(const IColumn& column);
438438

439439
} // namespace doris::vectorized
440+
441+
// Wrap `ColumnPtr` because `ColumnPtr` can't be used in forward declaration.
442+
namespace doris {
443+
struct ColumnPtrWrapper {
444+
vectorized::ColumnPtr column_ptr;
445+
446+
ColumnPtrWrapper(vectorized::ColumnPtr col) : column_ptr(col) {};
447+
};
448+
} // namespace doris

‎be/src/vec/exprs/vcast_expr.cpp‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,12 @@ void VCastExpr::close(doris::RuntimeState* state, VExprContext* context) {
6464
VExpr::close(state, context);
6565
}
6666

67-
doris::Status VCastExpr::execute(doris::vectorized::Block* block, int* result_column_id) {
67+
doris::Status VCastExpr::execute(VExprContext* context, doris::vectorized::Block* block,
68+
int* result_column_id) {
6869
// for each child call execute
6970
doris::vectorized::ColumnNumbers arguments(2);
7071
int column_id = -1;
71-
_children[0]->execute(block, &column_id);
72+
_children[0]->execute(context, block, &column_id);
7273
arguments[0] = column_id;
7374

7475
size_t const_param_id = block->columns();
@@ -79,7 +80,8 @@ doris::Status VCastExpr::execute(doris::vectorized::Block* block, int* result_co
7980
size_t num_columns_without_result = block->columns();
8081
// prepare a column to save result
8182
block->insert({nullptr, _data_type, _expr_name});
82-
_function->execute(*block, arguments, num_columns_without_result, block->rows(), false);
83+
_function->execute(context->fn_context(_fn_context_index), *block, arguments,
84+
num_columns_without_result, block->rows(), false);
8385
*result_column_id = num_columns_without_result;
8486
return Status::OK();
8587
}

‎be/src/vec/exprs/vcast_expr.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ class VCastExpr final: public VExpr {
2424
public:
2525
VCastExpr(const TExprNode& node) : VExpr(node) {}
2626
~VCastExpr() = default;
27-
virtual doris::Status execute(doris::vectorized::Block* block, int* result_column_id);
27+
virtual doris::Status execute(VExprContext* context, doris::vectorized::Block* block,
28+
int* result_column_id);
2829
virtual doris::Status prepare(doris::RuntimeState* state, const doris::RowDescriptor& desc,
2930
VExprContext* context);
3031
virtual doris::Status open(doris::RuntimeState* state, VExprContext* context);

‎be/src/vec/exprs/vectorized_fn_call.cpp‎

Lines changed: 44 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,10 @@
1919

2020
#include <string_view>
2121

22+
#include "exprs/anyval_util.h"
2223
#include "fmt/format.h"
2324
#include "fmt/ranges.h"
25+
#include "udf/udf_internal.h"
2426
#include "vec/data_types/data_type_nullable.h"
2527
#include "vec/data_types/data_type_number.h"
2628
#include "vec/functions/simple_function_factory.h"
@@ -47,30 +49,64 @@ doris::Status VectorizedFnCall::prepare(doris::RuntimeState* state,
4749
fmt::format("Function {} is not implemented", _fn.name.function_name));
4850
}
4951
_expr_name = fmt::format("{}({})", _fn.name.function_name, child_expr_name);
52+
53+
FunctionContext::TypeDesc return_type = AnyValUtil::column_type_to_type_desc(_type);
54+
std::vector<FunctionContext::TypeDesc> arg_types;
55+
for (int i = 0; i < _children.size(); ++i) {
56+
arg_types.push_back(AnyValUtil::column_type_to_type_desc(_children[i]->type()));
57+
}
58+
59+
_fn_context_index = context->register_func(state, return_type, arg_types, 0);
5060
return Status::OK();
5161
}
52-
doris::Status VectorizedFnCall::open(doris::RuntimeState* state, VExprContext* context) {
53-
RETURN_IF_ERROR(VExpr::open(state, context));
62+
63+
doris::Status VectorizedFnCall::open(doris::RuntimeState* state, VExprContext* context,
64+
FunctionContext::FunctionStateScope scope) {
65+
RETURN_IF_ERROR(VExpr::open(state, context, scope));
66+
67+
FunctionContext* fn_ctx = context->fn_context(_fn_context_index);
68+
if (scope == FunctionContext::FRAGMENT_LOCAL) {
69+
std::vector<ColumnPtrWrapper*> constant_cols;
70+
for (int i = 0; i < _children.size(); ++i) {
71+
constant_cols.push_back(_children[i]->get_const_col(context));
72+
}
73+
fn_ctx->impl()->set_constant_cols(constant_cols);
74+
}
75+
76+
if (scope == FunctionContext::FRAGMENT_LOCAL) {
77+
RETURN_IF_ERROR(_function->prepare(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
78+
}
79+
RETURN_IF_ERROR(_function->prepare(fn_ctx, FunctionContext::THREAD_LOCAL));
5480
return Status::OK();
5581
}
56-
void VectorizedFnCall::close(doris::RuntimeState* state, VExprContext* context) {
57-
VExpr::close(state, context);
82+
83+
void VectorizedFnCall::close(doris::RuntimeState* state, VExprContext* context,
84+
FunctionContext::FunctionStateScope scope) {
85+
if (_fn_context_index != -1) {
86+
FunctionContext* fn_ctx = context->fn_context(_fn_context_index);
87+
_function->close(fn_ctx, FunctionContext::THREAD_LOCAL);
88+
if (scope == FunctionContext::FRAGMENT_LOCAL) {
89+
_function->close(fn_ctx, FunctionContext::FRAGMENT_LOCAL);
90+
}
91+
}
92+
VExpr::close(state, context, scope);
5893
}
5994

60-
doris::Status VectorizedFnCall::execute(doris::vectorized::Block* block, int* result_column_id) {
95+
doris::Status VectorizedFnCall::execute(VExprContext* context, doris::vectorized::Block* block,
96+
int* result_column_id) {
6197
// for each child call execute
6298
doris::vectorized::ColumnNumbers arguments(_children.size());
6399
for (int i = 0; i < _children.size(); ++i) {
64100
int column_id = -1;
65-
_children[i]->execute(block, &column_id);
101+
_children[i]->execute(context, block, &column_id);
66102
arguments[i] = column_id;
67103
}
68104
// call function
69105
size_t num_columns_without_result = block->columns();
70106
// prepare a column to save result
71107
block->insert({nullptr, _data_type, _expr_name});
72-
RETURN_IF_ERROR(_function->execute(*block, arguments, num_columns_without_result, block->rows(),
73-
false));
108+
RETURN_IF_ERROR(_function->execute(context->fn_context(_fn_context_index), *block, arguments,
109+
num_columns_without_result, block->rows(), false));
74110
*result_column_id = num_columns_without_result;
75111
return Status::OK();
76112
}

‎be/src/vec/exprs/vectorized_fn_call.h‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,14 @@ namespace doris::vectorized {
2424
class VectorizedFnCall final : public VExpr {
2525
public:
2626
VectorizedFnCall(const doris::TExprNode& node);
27-
virtual doris::Status execute(doris::vectorized::Block* block, int* result_column_id);
27+
virtual doris::Status execute(VExprContext* context, doris::vectorized::Block* block,
28+
int* result_column_id);
2829
virtual doris::Status prepare(doris::RuntimeState* state, const doris::RowDescriptor& desc,
2930
VExprContext* context);
30-
virtual doris::Status open(doris::RuntimeState* state, VExprContext* context);
31-
virtual void close(doris::RuntimeState* state, VExprContext* context);
31+
virtual doris::Status open(doris::RuntimeState* state, VExprContext* context,
32+
FunctionContext::FunctionStateScope scope);
33+
virtual void close(doris::RuntimeState* state, VExprContext* context,
34+
FunctionContext::FunctionStateScope scope);
3235
virtual VExpr* clone(doris::ObjectPool* pool) const override {
3336
return pool->add(new VectorizedFnCall(*this));
3437
}

0 commit comments

Comments
 (0)