Skip to content

Commit 1054891

Browse files
morningmanpull[bot]
authored andcommitted
[improvement](catalog) reduce the size thrift params for external table query (apache#21771)
### 1 In previous implementation, for each FileSplit, there will be a `TFileScanRange`, and each `TFileScanRange` contains a list of `TFileRangeDesc` and a `TFileScanRangeParams`. So if there are thousands of FileSplit, there will be thousands of `TFileScanRange`, which cause the thrift data send to BE too large, resulting in: 1. the rpc of sending fragment may fail due to timeout 2. FE will OOM For a certain query request, the `TFileScanRangeParams` is the common part and is same of all `TFileScanRange`. So I move this to the `TExecPlanFragmentParams`. After that, for each FileSplit, there is only a list of `TFileRangeDesc`. In my test, to query a hive table with 100000 partitions, the size of thrift data reduced from 151MB to 15MB, and the above 2 issues are gone. ### 2 Support when setting `max_external_file_meta_cache_num` <=0, the file meta cache for parquet footer will not be used. Because I found that for some wide table, the footer is too large(1MB after compact, and much more after deserialized to thrift), it will consuming too much memory of BE when there are many files. This will be optimized later, here I just support to disable this cache.
1 parent a1790ec commit 1054891

18 files changed

Lines changed: 211 additions & 88 deletions

File tree

‎be/src/olap/push_handler.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -322,7 +322,6 @@ PushBrokerReader::PushBrokerReader(const Schema* schema, const TBrokerScanRange&
322322
if (0 == _ranges.size()) {
323323
return;
324324
}
325-
_file_params.file_type = _ranges[0].file_type;
326325
_file_params.format_type = _ranges[0].format_type;
327326
_file_params.src_tuple_id = _params.src_tuple_id;
328327
_file_params.dest_tuple_id = _params.dest_tuple_id;
@@ -336,6 +335,12 @@ PushBrokerReader::PushBrokerReader(const Schema* schema, const TBrokerScanRange&
336335

337336
for (int i = 0; i < _ranges.size(); ++i) {
338337
TFileRangeDesc file_range;
338+
// TODO(cmy): in previous implementation, the file_type is set in _file_params
339+
// and it use _ranges[0].file_type.
340+
// Later, this field is moved to TFileRangeDesc, but here we still only use _ranges[0]'s
341+
// file_type.
342+
// Because I don't know if other range has this field, so just keep it same as before.
343+
file_range.file_type = _ranges[0].file_type;
339344
file_range.load_id = _ranges[i].load_id;
340345
file_range.path = _ranges[i].path;
341346
file_range.start_offset = _ranges[i].start_offset;

‎be/src/runtime/fragment_mgr.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -676,6 +676,12 @@ Status FragmentMgr::_get_query_ctx(const Params& params, TUniqueId query_id, boo
676676
query_ctx->query_id = query_id;
677677
RETURN_IF_ERROR(DescriptorTbl::create(&(query_ctx->obj_pool), params.desc_tbl,
678678
&(query_ctx->desc_tbl)));
679+
680+
// set file scan range params
681+
if (params.__isset.file_scan_params) {
682+
query_ctx->file_scan_range_params_map = params.file_scan_params;
683+
}
684+
679685
query_ctx->coord_addr = params.coord;
680686
LOG(INFO) << "query_id: " << UniqueId(query_ctx->query_id.hi, query_ctx->query_id.lo)
681687
<< " coord_addr " << query_ctx->coord_addr

‎be/src/runtime/query_context.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,10 @@ class QueryContext {
197197

198198
std::vector<TUniqueId> fragment_ids;
199199

200+
// plan node id -> TFileScanRangeParams
201+
// only for file scan node
202+
std::map<int, TFileScanRangeParams> file_scan_range_params_map;
203+
200204
private:
201205
ExecEnv* _exec_env;
202206
vectorized::VecDateTimeValue _start_time;

‎be/src/vec/exec/format/csv/csv_reader.cpp‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,12 @@ CsvReader::CsvReader(RuntimeState* state, RuntimeProfile* profile, ScannerCounte
8888
_io_ctx(io_ctx) {
8989
_file_format_type = _params.format_type;
9090
_is_proto_format = _file_format_type == TFileFormatType::FORMAT_PROTO;
91-
_file_compress_type = _params.compress_type;
91+
if (_range.__isset.compress_type) {
92+
// for compatibility
93+
_file_compress_type = _range.compress_type;
94+
} else {
95+
_file_compress_type = _params.compress_type;
96+
}
9297
_size = _range.size;
9398

9499
_split_values.reserve(sizeof(Slice) * _file_slot_descs.size());
@@ -110,7 +115,12 @@ CsvReader::CsvReader(RuntimeProfile* profile, const TFileScanRangeParams& params
110115
_decompressor(nullptr),
111116
_io_ctx(io_ctx) {
112117
_file_format_type = _params.format_type;
113-
_file_compress_type = _params.compress_type;
118+
if (_range.__isset.compress_type) {
119+
// for compatibility
120+
_file_compress_type = _range.compress_type;
121+
} else {
122+
_file_compress_type = _params.compress_type;
123+
}
114124
_size = _range.size;
115125
_init_system_properties();
116126
_init_file_description();
@@ -119,7 +129,12 @@ CsvReader::CsvReader(RuntimeProfile* profile, const TFileScanRangeParams& params
119129
CsvReader::~CsvReader() = default;
120130

121131
void CsvReader::_init_system_properties() {
122-
_system_properties.system_type = _params.file_type;
132+
if (_range.__isset.file_type) {
133+
// for compatibility
134+
_system_properties.system_type = _range.file_type;
135+
} else {
136+
_system_properties.system_type = _params.file_type;
137+
}
123138
_system_properties.properties = _params.properties;
124139
_system_properties.hdfs_params = _params.hdfs_params;
125140
if (_params.__isset.broker_addresses) {

‎be/src/vec/exec/format/json/new_json_reader.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,12 @@ NewJsonReader::NewJsonReader(RuntimeProfile* profile, const TFileScanRangeParams
135135
}
136136

137137
void NewJsonReader::_init_system_properties() {
138-
_system_properties.system_type = _params.file_type;
138+
if (_range.__isset.file_type) {
139+
// for compatibility
140+
_system_properties.system_type = _range.file_type;
141+
} else {
142+
_system_properties.system_type = _params.file_type;
143+
}
139144
_system_properties.properties = _params.properties;
140145
_system_properties.hdfs_params = _params.hdfs_params;
141146
if (_params.__isset.broker_addresses) {

‎be/src/vec/exec/format/orc/vorc_reader.cpp‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,9 @@ void OrcReader::_collect_profile_on_close() {
181181
COUNTER_UPDATE(_orc_profile.read_bytes, _statistics.fs_read_bytes);
182182
COUNTER_UPDATE(_orc_profile.column_read_time, _statistics.column_read_time);
183183
COUNTER_UPDATE(_orc_profile.get_batch_time, _statistics.get_batch_time);
184-
COUNTER_UPDATE(_orc_profile.parse_meta_time, _statistics.parse_meta_time);
184+
COUNTER_UPDATE(_orc_profile.create_reader_time, _statistics.create_reader_time);
185+
COUNTER_UPDATE(_orc_profile.init_column_time, _statistics.init_column_time);
186+
COUNTER_UPDATE(_orc_profile.set_fill_column_time, _statistics.set_fill_column_time);
185187
COUNTER_UPDATE(_orc_profile.decode_value_time, _statistics.decode_value_time);
186188
COUNTER_UPDATE(_orc_profile.decode_null_map_time, _statistics.decode_null_map_time);
187189
}
@@ -200,7 +202,11 @@ void OrcReader::_init_profile() {
200202
_orc_profile.read_bytes = ADD_COUNTER(_profile, "FileReadBytes", TUnit::BYTES);
201203
_orc_profile.column_read_time = ADD_CHILD_TIMER(_profile, "ColumnReadTime", orc_profile);
202204
_orc_profile.get_batch_time = ADD_CHILD_TIMER(_profile, "GetBatchTime", orc_profile);
203-
_orc_profile.parse_meta_time = ADD_CHILD_TIMER(_profile, "ParseMetaTime", orc_profile);
205+
_orc_profile.create_reader_time =
206+
ADD_CHILD_TIMER(_profile, "CreateReaderTime", orc_profile);
207+
_orc_profile.init_column_time = ADD_CHILD_TIMER(_profile, "InitColumnTime", orc_profile);
208+
_orc_profile.set_fill_column_time =
209+
ADD_CHILD_TIMER(_profile, "SetFillColumnTime", orc_profile);
204210
_orc_profile.decode_value_time = ADD_CHILD_TIMER(_profile, "DecodeValueTime", orc_profile);
205211
_orc_profile.decode_null_map_time =
206212
ADD_CHILD_TIMER(_profile, "DecodeNullMapTime", orc_profile);
@@ -254,9 +260,14 @@ Status OrcReader::init_reader(
254260
not_single_slot_filter_conjuncts->end());
255261
}
256262
_obj_pool = std::make_shared<ObjectPool>();
257-
SCOPED_RAW_TIMER(&_statistics.parse_meta_time);
258-
RETURN_IF_ERROR(_create_file_reader());
259-
RETURN_IF_ERROR(_init_read_columns());
263+
{
264+
SCOPED_RAW_TIMER(&_statistics.create_reader_time);
265+
RETURN_IF_ERROR(_create_file_reader());
266+
}
267+
{
268+
SCOPED_RAW_TIMER(&_statistics.init_column_time);
269+
RETURN_IF_ERROR(_init_read_columns());
270+
}
260271
return Status::OK();
261272
}
262273

@@ -646,7 +657,7 @@ Status OrcReader::set_fill_columns(
646657
const std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>&
647658
partition_columns,
648659
const std::unordered_map<std::string, VExprContextSPtr>& missing_columns) {
649-
SCOPED_RAW_TIMER(&_statistics.parse_meta_time);
660+
SCOPED_RAW_TIMER(&_statistics.set_fill_column_time);
650661

651662
// std::unordered_map<column_name, std::pair<col_id, slot_id>>
652663
std::unordered_map<std::string, std::pair<uint32_t, int>> predicate_columns;
@@ -864,7 +875,12 @@ void OrcReader::_init_bloom_filter(
864875
}
865876

866877
void OrcReader::_init_system_properties() {
867-
_system_properties.system_type = _scan_params.file_type;
878+
if (_scan_range.__isset.file_type) {
879+
// for compatibility
880+
_system_properties.system_type = _scan_range.file_type;
881+
} else {
882+
_system_properties.system_type = _scan_params.file_type;
883+
}
868884
_system_properties.properties = _scan_params.properties;
869885
_system_properties.hdfs_params = _scan_params.hdfs_params;
870886
if (_scan_params.__isset.broker_addresses) {

‎be/src/vec/exec/format/orc/vorc_reader.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,9 @@ class OrcReader : public GenericReader {
126126
int64_t fs_read_bytes = 0;
127127
int64_t column_read_time = 0;
128128
int64_t get_batch_time = 0;
129-
int64_t parse_meta_time = 0;
129+
int64_t create_reader_time = 0;
130+
int64_t init_column_time = 0;
131+
int64_t set_fill_column_time = 0;
130132
int64_t decode_value_time = 0;
131133
int64_t decode_null_map_time = 0;
132134
};
@@ -200,7 +202,9 @@ class OrcReader : public GenericReader {
200202
RuntimeProfile::Counter* read_bytes;
201203
RuntimeProfile::Counter* column_read_time;
202204
RuntimeProfile::Counter* get_batch_time;
203-
RuntimeProfile::Counter* parse_meta_time;
205+
RuntimeProfile::Counter* create_reader_time;
206+
RuntimeProfile::Counter* init_column_time;
207+
RuntimeProfile::Counter* set_fill_column_time;
204208
RuntimeProfile::Counter* decode_value_time;
205209
RuntimeProfile::Counter* decode_null_map_time;
206210
};

‎be/src/vec/exec/format/parquet/vparquet_reader.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,12 @@ Status ParquetReader::open() {
269269
}
270270

271271
void ParquetReader::_init_system_properties() {
272-
_system_properties.system_type = _scan_params.file_type;
272+
if (_scan_range.__isset.file_type) {
273+
// for compatibility
274+
_system_properties.system_type = _scan_range.file_type;
275+
} else {
276+
_system_properties.system_type = _scan_params.file_type;
277+
}
273278
_system_properties.properties = _scan_params.properties;
274279
_system_properties.hdfs_params = _scan_params.hdfs_params;
275280
if (_scan_params.__isset.broker_addresses) {

‎be/src/vec/exec/scan/avro_jni_reader.cpp‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,13 @@ Status AvroJNIReader::init_fetch_table_reader(
7474
index++;
7575
}
7676

77-
TFileType::type type = _params.file_type;
77+
TFileType::type type;
78+
if (_range.__isset.file_type) {
79+
// for compatibility
80+
type = _range.file_type;
81+
} else {
82+
type = _params.file_type;
83+
}
7884
std::map<String, String> required_param = {
7985
{"required_fields", required_fields.str()},
8086
{"columns_types", columns_types.str()},

‎be/src/vec/exec/scan/new_file_scan_node.cpp‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,12 @@ Status NewFileScanNode::init(const TPlanNode& tnode, RuntimeState* state) {
4949

5050
Status NewFileScanNode::prepare(RuntimeState* state) {
5151
RETURN_IF_ERROR(VScanNode::prepare(state));
52+
if (state->get_query_ctx() != nullptr &&
53+
state->get_query_ctx()->file_scan_range_params_map.count(id()) > 0) {
54+
TFileScanRangeParams& params = state->get_query_ctx()->file_scan_range_params_map[id()];
55+
_input_tuple_id = params.src_tuple_id;
56+
_output_tuple_id = params.dest_tuple_id;
57+
}
5258
return Status::OK();
5359
}
5460

@@ -74,7 +80,10 @@ void NewFileScanNode::set_scan_ranges(const std::vector<TScanRangeParams>& scan_
7480
_scan_ranges.shrink_to_fit();
7581
LOG(INFO) << "Merge " << scan_ranges.size() << " scan ranges to " << _scan_ranges.size();
7682
}
77-
if (scan_ranges.size() > 0) {
83+
if (scan_ranges.size() > 0 &&
84+
scan_ranges[0].scan_range.ext_scan_range.file_scan_range.__isset.params) {
85+
// for compatibility.
86+
// in new implement, the tuple id is set in prepare phase
7887
_input_tuple_id =
7988
scan_ranges[0].scan_range.ext_scan_range.file_scan_range.params.src_tuple_id;
8089
_output_tuple_id =

0 commit comments

Comments
 (0)