Skip to content

Commit 38f73c6

Browse files
authored
[Function] Support parse_url function on vectorized engine (apache#212)
1 parent 518e87f commit 38f73c6

5 files changed

Lines changed: 183 additions & 8 deletions

File tree

‎be/src/vec/functions/function_string.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -674,6 +674,7 @@ void register_function_string(SimpleFunctionFactory& factory) {
674674
factory.register_function<FunctionFromBase64>();
675675
factory.register_function<FunctionSplitPart>();
676676
factory.register_function<FunctionStringMd5sum>();
677+
factory.register_function<FunctionStringParseUrl>();
677678

678679
factory.register_alias(FunctionLeft::name, "strleft");
679680
factory.register_alias(FunctionRight::name, "strright");

‎be/src/vec/functions/function_string.h‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,18 @@
1717

1818
#pragma once
1919

20+
#include "exprs/anyval_util.h"
21+
2022
#include <fmt/core.h>
2123
#include <fmt/format.h>
2224
#include <fmt/ranges.h>
2325

26+
#include "runtime/string_value.hpp"
27+
2428
#include <string_view>
2529

2630
#include "util/md5.h"
31+
#include "util/url_parser.h"
2732

2833
#include "vec/columns/column_nullable.h"
2934
#include "vec/columns/column_string.h"
@@ -882,4 +887,96 @@ class FunctionStringMd5sum : public IFunction {
882887
return Status::OK();
883888
}
884889
};
890+
891+
class FunctionStringParseUrl : public IFunction {
892+
public:
893+
static constexpr auto name = "parse_url";
894+
static FunctionPtr create() { return std::make_shared<FunctionStringParseUrl>(); }
895+
String get_name() const override { return name; }
896+
size_t get_number_of_arguments() const override { return 0; }
897+
bool is_variadic() const override { return true; }
898+
899+
DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
900+
return make_nullable(std::make_shared<DataTypeString>());
901+
}
902+
bool use_default_implementation_for_nulls() const override { return false; }
903+
904+
Status execute_impl(Block& block, const ColumnNumbers& arguments, size_t result,
905+
size_t input_rows_count) override {
906+
auto null_map = ColumnUInt8::create(input_rows_count, 0);
907+
auto& null_map_data = null_map->get_data();
908+
909+
auto res = ColumnString::create();
910+
auto& res_offsets = res->get_offsets();
911+
auto& res_chars = res->get_chars();
912+
res_offsets.resize(input_rows_count);
913+
914+
size_t argument_size = arguments.size();
915+
bool has_key = argument_size >= 3;
916+
917+
ColumnPtr argument_columns[argument_size];
918+
for (size_t i = 0; i < argument_size; ++i) {
919+
argument_columns[i] =
920+
block.get_by_position(arguments[i]).column->convert_to_full_column_if_const();
921+
if (auto* nullable = check_and_get_column<const ColumnNullable>(*argument_columns[i])) {
922+
argument_columns[i] = nullable->get_nested_column_ptr();
923+
VectorizedUtils::update_null_map(null_map_data, nullable->get_null_map_data());
924+
}
925+
}
926+
927+
auto url_col = assert_cast<const ColumnString*>(argument_columns[0].get());
928+
auto part_col = assert_cast<const ColumnString*>(argument_columns[1].get());
929+
const ColumnString* key_col;
930+
if (has_key) {
931+
key_col = assert_cast<const ColumnString*>(argument_columns[2].get());
932+
}
933+
934+
for (size_t i = 0; i < input_rows_count; ++i) {
935+
if (null_map_data[i]) {
936+
StringOP::push_null_string(i, res_chars, res_offsets, null_map_data);
937+
continue;
938+
}
939+
940+
auto part = part_col->get_data_at(i);
941+
StringValue p(const_cast<char *>(part.data), part.size);
942+
UrlParser::UrlPart url_part = UrlParser::get_url_part(p);
943+
StringValue url_key;
944+
if (has_key) {
945+
auto key = key_col->get_data_at(i);
946+
url_key = StringValue(const_cast<char *>(key.data), key.size);
947+
}
948+
949+
auto source = url_col->get_data_at(i);
950+
StringValue url_val(const_cast<char *>(source.data), source.size);
951+
952+
StringValue parse_res;
953+
bool success = false;
954+
if (has_key) {
955+
success = UrlParser::parse_url_key(url_val, url_part, url_key, &parse_res);
956+
} else {
957+
success = UrlParser::parse_url(url_val, url_part, &parse_res);
958+
}
959+
960+
if (!success) {
961+
// url is malformed, or url_part is invalid.
962+
if (url_part == UrlParser::INVALID) {
963+
return Status::RuntimeError(fmt::format(
964+
"Invalid URL part: {}\n{}", std::string(part.data, part.size),
965+
"(Valid URL parts are 'PROTOCOL', 'HOST', 'PATH', 'REF', 'AUTHORITY', "
966+
"'FILE', 'USERINFO', 'PORT' and 'QUERY')"));
967+
} else {
968+
StringOP::push_null_string(i, res_chars, res_offsets, null_map_data);
969+
continue;
970+
}
971+
}
972+
973+
StringOP::push_value_string(std::string_view(parse_res.ptr, parse_res.len),
974+
i, res_chars, res_offsets);
975+
}
976+
block.get_by_position(result).column =
977+
ColumnNullable::create(std::move(res), std::move(null_map));
978+
return Status::OK();
979+
}
980+
};
981+
885982
} // namespace doris::vectorized

‎be/test/vec/function/function_string_test.cpp‎

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,22 +14,17 @@
1414
// KIND, either express or implied. See the License for the
1515
// specific language governing permissions and limitations
1616
// under the License.
17-
#include "vec/functions/function_string.h"
1817

1918
#include <gtest/gtest.h>
2019
#include <time.h>
2120

22-
#include <iostream>
2321
#include <string>
2422

25-
#include "exec/schema_scanner.h"
2623
#include "function_test_util.h"
27-
#include "runtime/row_batch.h"
2824
#include "runtime/tuple_row.h"
2925
#include "util/aes_util.h"
3026
#include "util/url_coding.h"
3127
#include "vec/core/field.h"
32-
#include "vec/functions/simple_function_factory.h"
3328

3429
namespace doris {
3530

@@ -615,9 +610,74 @@ TEST(function_string_test, function_aes_decrypt_test) {
615610
vectorized::check_function<vectorized::DataTypeString, true>(func_name, input_types, data_set);
616611
}
617612

613+
TEST(function_string_test, function_parse_url_test) {
614+
std::string func_name = "parse_url";
615+
616+
{
617+
std::vector<std::any> input_types = {TypeIndex::String, TypeIndex::String};
618+
DataSet data_set = {
619+
{{std::string("zhangsan"), std::string("HOST")},
620+
{Null()}},
621+
{{std::string("facebook.com/path/p1"), std::string("HOST")},
622+
{Null()}},
623+
{{std::string("http://fb.com/path/p1.p?q=1#f"), std::string("HOST")},
624+
{std::string("fb.com")}},
625+
{{std::string("http://facebook.com/path/p1.php?query=1"), std::string("AUTHORITY")},
626+
{std::string("facebook.com")}},
627+
{{std::string("http://facebook.com/path/p1.php?query=1"), std::string("authority")},
628+
{std::string("facebook.com")}},
629+
{{std::string("http://www.baidu.com:9090/a/b/c.php"), std::string("FILE")},
630+
{std::string("/a/b/c.php")}},
631+
{{std::string("http://www.baidu.com:9090/a/b/c.php"), std::string("file")},
632+
{std::string("/a/b/c.php")}},
633+
{{std::string("http://www.baidu.com:9090/a/b/c.php"), std::string("PATH")},
634+
{std::string("/a/b/c.php")}},
635+
{{std::string("http://www.baidu.com:9090/a/b/c.php"), std::string("path")},
636+
{std::string("/a/b/c.php")}},
637+
{{std::string("http://facebook.com/path/p1.php?query=1"), std::string("PROTOCOL")},
638+
{std::string("http")}},
639+
{{std::string("http://facebook.com/path/p1.php?query=1"), std::string("protocol")},
640+
{std::string("http")}},
641+
{{std::string("http://www.baidu.com:9090?a=b"), std::string("QUERY")},
642+
{std::string("a=b")}},
643+
{{std::string("http://www.baidu.com:9090?a=b"), std::string("query")},
644+
{std::string("a=b")}},
645+
{{std::string("http://www.baidu.com:9090?a=b"), std::string("REF")},
646+
{Null()}},
647+
{{std::string("http://www.baidu.com:9090?a=b"), std::string("ref")},
648+
{Null()}},
649+
{{std::string("http://www.baidu.com:9090/a/b/c?a=b"), std::string("PORT")},
650+
{std::string("9090")}},
651+
{{std::string("http://www.baidu.com/a/b/c?a=b"), std::string("PORT")},
652+
{Null()}},
653+
{{std::string("http://fb.com/path/p1.p?q=1#f"), std::string("QUERY")},
654+
{std::string("q=1")}}
655+
};
656+
657+
vectorized::check_function<vectorized::DataTypeString, true>(func_name, input_types, data_set);
658+
}
659+
660+
{
661+
std::vector<std::any> input_types = {TypeIndex::String, TypeIndex::String, TypeIndex::String};
662+
DataSet data_set = {
663+
{{std::string("http://fb.com/path/p1.p?q=1#f"), std::string("QUERY"), std::string("q")},
664+
{std::string("1")}},
665+
{{std::string("fb.com/path/p1.p?q=1#f"), std::string("QUERY"), std::string("q")},
666+
{std::string("1")}},
667+
{{std::string("http://facebook.com/path/p1"), std::string("QUERY"), std::string("q")},
668+
{Null()}},
669+
{{std::string("http://fb.com/path/p1.p?q=1#f"), std::string("HOST"), std::string("q")},
670+
{Null()}}
671+
};
672+
673+
vectorized::check_function<vectorized::DataTypeString, true>(func_name, input_types, data_set);
674+
}
675+
}
676+
618677
} // namespace doris
619678

620679
int main(int argc, char** argv) {
680+
doris::CpuInfo::init();
621681
::testing::InitGoogleTest(&argc, argv);
622682
return RUN_ALL_TESTS();
623683
}

‎be/test/vec/runtime/vdata_stream_test.cpp‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,20 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
118
#include "bthread/id.h"
219
#include "common/object_pool.h"
320
#include "gen_cpp/internal_service.pb.h"

‎gensrc/script/doris_builtins_functions.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -920,14 +920,14 @@
920920
'_ZN5doris15StringFunctions17parse_url_prepareEPN9doris_udf'
921921
'15FunctionContextENS2_18FunctionStateScopeE',
922922
'_ZN5doris15StringFunctions15parse_url_closeEPN9doris_udf'
923-
'15FunctionContextENS2_18FunctionStateScopeE', '', ''],
923+
'15FunctionContextENS2_18FunctionStateScopeE', 'vec', 'ALWAYS_NULLABLE'],
924924
[['parse_url'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'VARCHAR'],
925925
'_ZN5doris15StringFunctions13parse_url_keyEPN9doris_udf'
926926
'15FunctionContextERKNS1_9StringValES6_S6_',
927927
'_ZN5doris15StringFunctions17parse_url_prepareEPN9doris_udf'
928928
'15FunctionContextENS2_18FunctionStateScopeE',
929929
'_ZN5doris15StringFunctions15parse_url_closeEPN9doris_udf'
930-
'15FunctionContextENS2_18FunctionStateScopeE', '', ''],
930+
'15FunctionContextENS2_18FunctionStateScopeE', 'vec', 'ALWAYS_NULLABLE'],
931931
[['money_format'], 'VARCHAR', ['BIGINT'],
932932
'_ZN5doris15StringFunctions12money_formatEPN9doris_udf15FunctionContextERKNS1_9BigIntValE',
933933
'', '', '', ''],
@@ -1210,7 +1210,7 @@
12101210
'15FunctionContextERKNS1_9StringValES6_', '', '', 'vec', 'ALWAYS_NULLABLE'],
12111211
[['from_base64'], 'STRING', ['STRING'],
12121212
'_ZN5doris19EncryptionFunctions11from_base64EPN9doris_udf'
1213-
'15FunctionContextERKNS1_9StringValE', '', '', '', ''],
1213+
'15FunctionContextERKNS1_9StringValE', '', '', 'vec', 'ALWAYS_NULLABLE'],
12141214
[['to_base64'], 'STRING', ['STRING'],
12151215
'_ZN5doris19EncryptionFunctions9to_base64EPN9doris_udf'
12161216
'15FunctionContextERKNS1_9StringValE', '', '', 'vec', 'ALWAYS_NULLABLE'],

0 commit comments

Comments
 (0)