Skip to content

Commit 01ee595

Browse files
authored
[feature](information_schema)add metadata_name_ids for quickly get catlogs,db,table and add profiling table in order to Compatible with mysql (#22702) (#23980)
1 parent 5240e90 commit 01ee595

19 files changed

Lines changed: 748 additions & 52 deletions

File tree

‎be/src/exec/schema_scanner.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@
3131
#include "exec/schema_scanner/schema_columns_scanner.h"
3232
#include "exec/schema_scanner/schema_dummy_scanner.h"
3333
#include "exec/schema_scanner/schema_files_scanner.h"
34+
#include "exec/schema_scanner/schema_metadata_name_ids_scanner.h"
3435
#include "exec/schema_scanner/schema_partitions_scanner.h"
36+
#include "exec/schema_scanner/schema_profiling_scanner.h"
3537
#include "exec/schema_scanner/schema_rowsets_scanner.h"
3638
#include "exec/schema_scanner/schema_schema_privileges_scanner.h"
3739
#include "exec/schema_scanner/schema_schemata_scanner.h"
@@ -144,6 +146,10 @@ std::unique_ptr<SchemaScanner> SchemaScanner::create(TSchemaTableType::type type
144146
return SchemaPartitionsScanner::create_unique();
145147
case TSchemaTableType::SCH_ROWSETS:
146148
return SchemaRowsetsScanner::create_unique();
149+
case TSchemaTableType::SCH_METADATA_NAME_IDS:
150+
return SchemaMetadataNameIdsScanner::create_unique();
151+
case TSchemaTableType::SCH_PROFILING:
152+
return SchemaProfilingScanner::create_unique();
147153
default:
148154
return SchemaDummyScanner::create_unique();
149155
break;

‎be/src/exec/schema_scanner/schema_helper.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,14 @@ Status SchemaHelper::list_table_status(const std::string& ip, const int32_t port
6060
client->listTableStatus(*result, request);
6161
});
6262
}
63+
Status SchemaHelper::list_table_metadata_name_ids(const std::string& ip, const int32_t port,
64+
const doris::TGetTablesParams& request,
65+
TListTableMetadataNameIdsResult* result) {
66+
return ThriftRpcHelper::rpc<FrontendServiceClient>(
67+
ip, port, [&request, &result](FrontendServiceConnection& client) {
68+
client->listTableMetadataNameIds(*result, request);
69+
});
70+
}
6371

6472
Status SchemaHelper::describe_table(const std::string& ip, const int32_t port,
6573
const TDescribeTableParams& request,

‎be/src/exec/schema_scanner/schema_helper.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ class TGetTablesParams;
3434
class TGetTablesResult;
3535
class TListPrivilegesResult;
3636
class TListTableStatusResult;
37+
class TListTableMetadataNameIdsResult;
3738
class TShowVariableRequest;
3839
class TShowVariableResult;
3940

@@ -50,6 +51,9 @@ class SchemaHelper {
5051
static Status list_table_status(const std::string& ip, const int32_t port,
5152
const TGetTablesParams& table_params,
5253
TListTableStatusResult* table_result);
54+
static Status list_table_metadata_name_ids(const std::string& ip, const int32_t port,
55+
const doris::TGetTablesParams& request,
56+
TListTableMetadataNameIdsResult* result);
5357

5458
static Status describe_table(const std::string& ip, const int32_t port,
5559
const TDescribeTableParams& desc_params,
Lines changed: 244 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,244 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "exec/schema_scanner/schema_metadata_name_ids_scanner.h"
19+
20+
#include <gen_cpp/Descriptors_types.h>
21+
#include <gen_cpp/FrontendService_types.h>
22+
#include <stdint.h>
23+
24+
#include <string>
25+
26+
#include "common/status.h"
27+
#include "exec/schema_scanner/schema_helper.h"
28+
#include "runtime/decimalv2_value.h"
29+
#include "runtime/define_primitive_type.h"
30+
#include "util/runtime_profile.h"
31+
#include "util/timezone_utils.h"
32+
#include "vec/common/string_ref.h"
33+
#include "vec/runtime/vdatetime_value.h"
34+
35+
namespace doris {
36+
class RuntimeState;
37+
38+
namespace vectorized {
39+
class Block;
40+
} // namespace vectorized
41+
42+
std::vector<SchemaScanner::ColumnDesc> SchemaMetadataNameIdsScanner::_s_tbls_columns = {
43+
// name, type, size, is_null
44+
{"CATALOG_ID", TYPE_BIGINT, sizeof(int64_t), true},
45+
{"CATALOG_NAME", TYPE_VARCHAR, sizeof(StringRef), true},
46+
{"DATABASE_ID", TYPE_BIGINT, sizeof(int64_t), true},
47+
{"DATABASE_NAME", TYPE_VARCHAR, sizeof(StringRef), true},
48+
{"TABLE_ID", TYPE_BIGINT, sizeof(int64_t), true},
49+
{"TABLE_NAME", TYPE_VARCHAR, sizeof(StringRef), true},
50+
};
51+
52+
SchemaMetadataNameIdsScanner::SchemaMetadataNameIdsScanner()
53+
: SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_METADATA_NAME_IDS), _db_index(0) {}
54+
55+
SchemaMetadataNameIdsScanner::~SchemaMetadataNameIdsScanner() {}
56+
57+
Status SchemaMetadataNameIdsScanner::start(RuntimeState* state) {
58+
if (!_is_init) {
59+
return Status::InternalError("used before initialized.");
60+
}
61+
SCOPED_TIMER(_get_db_timer);
62+
TGetDbsParams db_params;
63+
if (nullptr != _param->db) {
64+
db_params.__set_pattern(*(_param->db));
65+
}
66+
if (nullptr != _param->catalog) {
67+
db_params.__set_catalog(*(_param->catalog));
68+
}
69+
if (nullptr != _param->current_user_ident) {
70+
db_params.__set_current_user_ident(*(_param->current_user_ident));
71+
} else {
72+
if (nullptr != _param->user) {
73+
db_params.__set_user(*(_param->user));
74+
}
75+
if (nullptr != _param->user_ip) {
76+
db_params.__set_user_ip(*(_param->user_ip));
77+
}
78+
}
79+
db_params.__set_get_null_catalog(true);
80+
if (nullptr != _param->ip && 0 != _param->port) {
81+
RETURN_IF_ERROR(
82+
SchemaHelper::get_db_names(*(_param->ip), _param->port, db_params, &_db_result));
83+
} else {
84+
return Status::InternalError("IP or port doesn't exists");
85+
}
86+
return Status::OK();
87+
}
88+
89+
Status SchemaMetadataNameIdsScanner::_get_new_table() {
90+
SCOPED_TIMER(_get_table_timer);
91+
if (_db_result.db_ids[_db_index] == -1 &&
92+
_db_result.dbs[_db_index] == "NULL") { //catalog is empty.
93+
_db_index++;
94+
_table_result.tables.clear();
95+
_table_result.tables.push_back(TTableMetadataNameIds());
96+
97+
return Status::OK();
98+
}
99+
TGetTablesParams table_params;
100+
table_params.__set_db(_db_result.dbs[_db_index]);
101+
if (_db_result.__isset.catalogs) {
102+
table_params.__set_catalog(_db_result.catalogs[_db_index]);
103+
}
104+
_db_index++;
105+
if (nullptr != _param->wild) {
106+
table_params.__set_pattern(*(_param->wild));
107+
}
108+
if (nullptr != _param->current_user_ident) {
109+
table_params.__set_current_user_ident(*(_param->current_user_ident));
110+
} else {
111+
if (nullptr != _param->user) {
112+
table_params.__set_user(*(_param->user));
113+
}
114+
if (nullptr != _param->user_ip) {
115+
table_params.__set_user_ip(*(_param->user_ip));
116+
}
117+
}
118+
119+
if (nullptr != _param->ip && 0 != _param->port) {
120+
RETURN_IF_ERROR(SchemaHelper::list_table_metadata_name_ids(*(_param->ip), _param->port,
121+
table_params, &_table_result));
122+
} else {
123+
return Status::InternalError("IP or port doesn't exists");
124+
}
125+
return Status::OK();
126+
}
127+
128+
Status SchemaMetadataNameIdsScanner::_fill_block_impl(vectorized::Block* block) {
129+
SCOPED_TIMER(_fill_block_timer);
130+
auto table_num = _table_result.tables.size();
131+
if (table_num == 0) { //database is null
132+
table_num = 1;
133+
_table_result.tables.push_back(TTableMetadataNameIds());
134+
}
135+
std::vector<void*> null_datas(table_num, nullptr);
136+
std::vector<void*> datas(table_num);
137+
138+
// catalog_id
139+
{
140+
int64_t srcs[table_num];
141+
if (_db_result.__isset.catalog_ids) {
142+
int64_t id = _db_result.catalog_ids[_db_index - 1];
143+
for (int i = 0; i < table_num; ++i) {
144+
srcs[i] = id;
145+
datas[i] = srcs + i;
146+
}
147+
fill_dest_column_for_range(block, 0, datas);
148+
} else {
149+
fill_dest_column_for_range(block, 0, null_datas);
150+
}
151+
}
152+
153+
// catalog_name
154+
{
155+
if (_db_result.__isset.catalogs) {
156+
std::string catalog_name = _db_result.catalogs[_db_index - 1];
157+
StringRef str_slot = StringRef(catalog_name.c_str(), catalog_name.size());
158+
for (int i = 0; i < table_num; ++i) {
159+
datas[i] = &str_slot;
160+
}
161+
fill_dest_column_for_range(block, 1, datas);
162+
} else {
163+
fill_dest_column_for_range(block, 1, null_datas);
164+
}
165+
}
166+
167+
// database_id
168+
{
169+
int64_t srcs[table_num];
170+
if (_db_result.__isset.db_ids) {
171+
int64_t id = _db_result.db_ids[_db_index - 1];
172+
for (int i = 0; i < table_num; ++i) {
173+
srcs[i] = id;
174+
datas[i] = srcs + i;
175+
}
176+
fill_dest_column_for_range(block, 2, datas);
177+
} else {
178+
fill_dest_column_for_range(block, 2, null_datas);
179+
}
180+
}
181+
182+
// database_name
183+
{
184+
if (_db_result.__isset.dbs) {
185+
std::string db_name = SchemaHelper::extract_db_name(_db_result.dbs[_db_index - 1]);
186+
StringRef str_slot = StringRef(db_name.c_str(), db_name.size());
187+
for (int i = 0; i < table_num; ++i) {
188+
datas[i] = &str_slot;
189+
}
190+
fill_dest_column_for_range(block, 3, datas);
191+
} else {
192+
fill_dest_column_for_range(block, 3, null_datas);
193+
}
194+
}
195+
// table_id
196+
{
197+
int64_t srcs[table_num];
198+
for (int i = 0; i < table_num; ++i) {
199+
if (_table_result.tables[i].__isset.id) {
200+
srcs[i] = _table_result.tables[i].id;
201+
datas[i] = &srcs;
202+
} else {
203+
datas[i] = nullptr;
204+
}
205+
}
206+
fill_dest_column_for_range(block, 4, datas);
207+
}
208+
209+
//table_name
210+
{
211+
StringRef strs[table_num];
212+
for (int i = 0; i < table_num; ++i) {
213+
if (_table_result.tables[i].__isset.name) {
214+
const std::string* src = &_table_result.tables[i].name;
215+
strs[i] = StringRef(src->c_str(), src->size());
216+
datas[i] = strs + i;
217+
} else {
218+
datas[i] = nullptr;
219+
}
220+
}
221+
fill_dest_column_for_range(block, 5, datas);
222+
}
223+
224+
return Status::OK();
225+
}
226+
227+
Status SchemaMetadataNameIdsScanner::get_next_block(vectorized::Block* block, bool* eos) {
228+
if (!_is_init) {
229+
return Status::InternalError("Used before initialized.");
230+
}
231+
if (nullptr == block || nullptr == eos) {
232+
return Status::InternalError("input pointer is nullptr.");
233+
}
234+
if (_db_index < _db_result.dbs.size()) {
235+
RETURN_IF_ERROR(_get_new_table());
236+
} else {
237+
*eos = true;
238+
return Status::OK();
239+
}
240+
*eos = false;
241+
return _fill_block_impl(block);
242+
}
243+
244+
} // namespace doris
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include <gen_cpp/FrontendService_types.h>
21+
22+
#include <vector>
23+
24+
#include "common/status.h"
25+
#include "exec/schema_scanner.h"
26+
27+
namespace doris {
28+
class RuntimeState;
29+
30+
namespace vectorized {
31+
class Block;
32+
} // namespace vectorized
33+
34+
class SchemaMetadataNameIdsScanner : public SchemaScanner {
35+
ENABLE_FACTORY_CREATOR(SchemaMetadataNameIdsScanner);
36+
37+
public:
38+
SchemaMetadataNameIdsScanner();
39+
~SchemaMetadataNameIdsScanner() override;
40+
41+
Status start(RuntimeState* state) override;
42+
Status get_next_block(vectorized::Block* block, bool* eos) override;
43+
44+
private:
45+
Status _get_new_table();
46+
Status _fill_block_impl(vectorized::Block* block);
47+
48+
int _db_index;
49+
TGetDbsResult _db_result;
50+
TListTableMetadataNameIdsResult _table_result;
51+
static std::vector<SchemaScanner::ColumnDesc> _s_tbls_columns;
52+
};
53+
54+
} // namespace doris

0 commit comments

Comments
 (0)