Skip to content

Commit a4b5008

Browse files
authored
[Exec](performance) Topn lazy materialize
1 parent d3a247e commit a4b5008

178 files changed

Lines changed: 8979 additions & 1495 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/common/consts.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ const std::string CSV_WITH_NAMES_AND_TYPES = "csv_with_names_and_types";
2727
const std::string BLOCK_TEMP_COLUMN_PREFIX = "__TEMP__";
2828
const std::string BLOCK_TEMP_COLUMN_SCANNER_FILTERED = "__TEMP__scanner_filtered";
2929
const std::string ROWID_COL = "__DORIS_ROWID_COL__";
30+
const std::string GLOBAL_ROWID_COL = "__DORIS_GLOBAL_ROWID_COL__";
3031
const std::string ROW_STORE_COL = "__DORIS_ROW_STORE_COL__";
3132
const std::string DYNAMIC_COLUMN_NAME = "__DORIS_DYNAMIC_COL__";
3233
const std::string PARTIAL_UPDATE_AUTO_INC_COL = "__PARTIAL_UPDATE_AUTO_INC_COLUMN__";

‎be/src/exec/rowid_fetcher.cpp‎

Lines changed: 346 additions & 9 deletions
Large diffs are not rendered by default.

‎be/src/exec/rowid_fetcher.h‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727

2828
#include "common/status.h"
2929
#include "exec/tablet_info.h" // DorisNodesInfo
30+
#include "olap/id_manager.h"
3031
#include "vec/core/block.h"
3132
#include "vec/data_types/data_type.h"
3233

@@ -36,6 +37,11 @@ class DorisNodesInfo;
3637
class RuntimeState;
3738
class TupleDescriptor;
3839

40+
struct FileMapping;
41+
struct IteratorKey;
42+
struct IteratorItem;
43+
struct HashOfIteratorKey;
44+
3945
namespace vectorized {
4046
template <typename T>
4147
class ColumnStr;
@@ -70,9 +76,52 @@ class RowIDFetcher {
7076
FetchOption _fetch_option;
7177
};
7278

79+
struct RowStoreReadStruct {
80+
RowStoreReadStruct(std::string& buffer) : row_store_buffer(buffer) {};
81+
std::string& row_store_buffer;
82+
vectorized::DataTypeSerDeSPtrs serdes;
83+
std::unordered_map<uint32_t, uint32_t> col_uid_to_idx;
84+
std::vector<std::string> default_values;
85+
};
86+
7387
class RowIdStorageReader {
7488
public:
7589
static Status read_by_rowids(const PMultiGetRequest& request, PMultiGetResponse* response);
90+
static Status read_by_rowids(const PMultiGetRequestV2& request, PMultiGetResponseV2* response);
91+
92+
private:
93+
static Status read_doris_format_row(
94+
const std::shared_ptr<IdFileMap>& id_file_map,
95+
const std::shared_ptr<FileMapping>& file_mapping, int64_t row_id,
96+
std::vector<SlotDescriptor>& slots, const TabletSchema& full_read_schema,
97+
RowStoreReadStruct& row_store_read_struct, OlapReaderStatistics& stats,
98+
int64_t* acquire_tablet_ms, int64_t* acquire_rowsets_ms, int64_t* acquire_segments_ms,
99+
int64_t* lookup_row_data_ms,
100+
std::unordered_map<IteratorKey, IteratorItem, HashOfIteratorKey>& iterator_map,
101+
vectorized::Block& result_block);
102+
103+
static Status read_batch_doris_format_row(
104+
const PRequestBlockDesc& request_block_desc, std::shared_ptr<IdFileMap> id_file_map,
105+
std::vector<SlotDescriptor>& slots, const TUniqueId& query_id,
106+
vectorized::Block& result_block, OlapReaderStatistics& stats,
107+
int64_t* acquire_tablet_ms, int64_t* acquire_rowsets_ms, int64_t* acquire_segments_ms,
108+
int64_t* lookup_row_data_ms);
109+
110+
static Status read_batch_external_row(const PRequestBlockDesc& request_block_desc,
111+
std::shared_ptr<IdFileMap> id_file_map,
112+
std::vector<SlotDescriptor>& slots,
113+
std::shared_ptr<FileMapping> first_file_mapping,
114+
const TUniqueId& query_id,
115+
vectorized::Block& result_block, int64_t* init_reader_ms,
116+
int64_t* get_block_ms);
76117
};
77118

119+
template <typename Func>
120+
auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
121+
MonotonicStopWatch watch;
122+
watch.start();
123+
auto res = fn();
124+
*cost += watch.elapsed_time() / 1000 / 1000;
125+
return res;
126+
}
78127
} // namespace doris

‎be/src/olap/id_manager.h‎

Lines changed: 264 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,264 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#include <butil/macros.h>
21+
#include <gen_cpp/BackendService_types.h>
22+
#include <gen_cpp/Types_types.h>
23+
#include <stddef.h>
24+
#include <stdint.h>
25+
26+
#include <functional>
27+
#include <map>
28+
#include <memory>
29+
#include <mutex>
30+
#include <set>
31+
#include <shared_mutex>
32+
#include <string>
33+
#include <string_view>
34+
#include <unordered_map>
35+
#include <unordered_set>
36+
#include <utility>
37+
#include <vector>
38+
39+
#include "common/status.h"
40+
#include "olap/olap_common.h"
41+
#include "olap/tablet.h"
42+
#include "olap/tablet_meta.h"
43+
44+
namespace doris {
45+
46+
enum class FileMappingType {
47+
INTERNAL, // for doris format file {tablet_id}{rowset_id}{segment_id}
48+
EXTERNAL, // for external table.
49+
};
50+
51+
struct InternalFileMappingInfo {
52+
int64_t tablet_id;
53+
RowsetId rowset_id;
54+
uint32_t segment_id;
55+
56+
std::string to_string() const {
57+
std::string value;
58+
value.resize(sizeof(tablet_id) + sizeof(rowset_id) + sizeof(segment_id));
59+
auto* ptr = value.data();
60+
61+
memcpy(ptr, &tablet_id, sizeof(tablet_id));
62+
ptr += sizeof(tablet_id);
63+
memcpy(ptr, &rowset_id, sizeof(rowset_id));
64+
ptr += sizeof(rowset_id);
65+
memcpy(ptr, &segment_id, sizeof(segment_id));
66+
return value;
67+
}
68+
};
69+
70+
struct ExternalFileMappingInfo {
71+
/* By recording the plan_node_id in fileMapping, the TFileScanRangeParams used in the scan phase can be found
72+
* from QueryContext according to the plan_node_id. Because there are some important information in
73+
* TFileScanRangeParams (needed when creating hdfs/s3 reader):
74+
* 8: optional THdfsParams hdfs_params;
75+
* 9: optional map<string, string> properties;
76+
*/
77+
int plan_node_id;
78+
79+
/*
80+
* Record TFileRangeDesc external_scan_range_desc in fileMapping, usage:
81+
* 1. If the file belongs to a partition, columns_from_path_keys and columns_from_path in TFileRangeDesc are needed when materializing the partition column
82+
* 2. path, file_type, modification_time,compress_type .... used to read the file
83+
* 3. TFileFormatType can distinguish whether it is iceberg/hive/hudi/paimon
84+
*/
85+
TFileRangeDesc scan_range_desc;
86+
bool enable_file_meta_cache;
87+
88+
ExternalFileMappingInfo(int plan_node_id, const TFileRangeDesc& scan_range,
89+
bool file_meta_cache)
90+
: plan_node_id(plan_node_id),
91+
scan_range_desc(scan_range),
92+
enable_file_meta_cache(file_meta_cache) {}
93+
94+
std::string to_string() const {
95+
std::string value;
96+
value.resize(scan_range_desc.path.size() + sizeof(plan_node_id) +
97+
sizeof(scan_range_desc.start_offset));
98+
auto* ptr = value.data();
99+
100+
memcpy(ptr, &plan_node_id, sizeof(plan_node_id));
101+
ptr += sizeof(plan_node_id);
102+
memcpy(ptr, &scan_range_desc.start_offset, sizeof(scan_range_desc.start_offset));
103+
ptr += sizeof(scan_range_desc.start_offset);
104+
memcpy(ptr, scan_range_desc.path.data(), scan_range_desc.path.size());
105+
return value;
106+
}
107+
};
108+
109+
struct FileMapping {
110+
ENABLE_FACTORY_CREATOR(FileMapping);
111+
112+
FileMappingType type;
113+
std::variant<InternalFileMappingInfo, ExternalFileMappingInfo> value;
114+
115+
FileMapping(int64_t tablet_id, RowsetId rowset_id, uint32_t segment_id)
116+
: type(FileMappingType::INTERNAL),
117+
value(std::in_place_type<InternalFileMappingInfo>, tablet_id, rowset_id, segment_id) {
118+
}
119+
120+
FileMapping(int plan_node_id, const TFileRangeDesc& scan_range, bool enable_file_meta_cache)
121+
: type(FileMappingType::EXTERNAL),
122+
value(std::in_place_type<ExternalFileMappingInfo>, plan_node_id, scan_range,
123+
enable_file_meta_cache) {}
124+
125+
std::tuple<int64_t, RowsetId, uint32_t> get_doris_format_info() const {
126+
DCHECK(type == FileMappingType::INTERNAL);
127+
auto info = std::get<InternalFileMappingInfo>(value);
128+
return std::make_tuple(info.tablet_id, info.rowset_id, info.segment_id);
129+
}
130+
131+
ExternalFileMappingInfo& get_external_file_info() {
132+
DCHECK(type == FileMappingType::EXTERNAL);
133+
return std::get<ExternalFileMappingInfo>(value);
134+
}
135+
136+
static std::string file_mapping_info_to_string(
137+
const std::variant<InternalFileMappingInfo, ExternalFileMappingInfo>& info) {
138+
return std::visit(
139+
[](const auto& info) -> std::string {
140+
using T = std::decay_t<decltype(info)>;
141+
142+
if constexpr (std::is_same_v<T, InternalFileMappingInfo>) {
143+
return info.to_string();
144+
145+
} else if constexpr (std::is_same_v<T, ExternalFileMappingInfo>) {
146+
return info.to_string();
147+
}
148+
},
149+
info);
150+
}
151+
152+
std::string file_mapping_info_to_string() { return file_mapping_info_to_string(value); }
153+
};
154+
155+
class IdFileMap {
156+
public:
157+
IdFileMap(uint64_t expired_timestamp) : delayed_expired_timestamp(expired_timestamp) {}
158+
159+
std::shared_ptr<FileMapping> get_file_mapping(uint32_t id) {
160+
std::shared_lock lock(_mtx);
161+
auto it = _id_map.find(id);
162+
if (it == _id_map.end()) {
163+
return nullptr;
164+
}
165+
return it->second;
166+
}
167+
168+
uint32 get_file_mapping_id(const std::shared_ptr<FileMapping>& mapping) {
169+
DCHECK(mapping.get() != nullptr);
170+
auto value = mapping->file_mapping_info_to_string();
171+
172+
std::unique_lock lock(_mtx);
173+
auto it = _mapping_to_id.find(value);
174+
if (it != _mapping_to_id.end()) {
175+
return it->second;
176+
}
177+
_id_map[_init_id++] = mapping;
178+
_mapping_to_id[value] = _init_id - 1;
179+
180+
return _init_id - 1;
181+
}
182+
183+
void add_temp_rowset(const RowsetSharedPtr& rowset) {
184+
std::unique_lock lock(_mtx);
185+
_temp_rowset_maps[{rowset->rowset_meta()->tablet_id(), rowset->rowset_id()}] = rowset;
186+
}
187+
188+
RowsetSharedPtr get_temp_rowset(const int64_t tablet_id, const RowsetId& rowset_id) {
189+
std::shared_lock lock(_mtx);
190+
auto it = _temp_rowset_maps.find({tablet_id, rowset_id});
191+
if (it == _temp_rowset_maps.end()) {
192+
return nullptr;
193+
}
194+
return it->second;
195+
}
196+
197+
int64_t get_delayed_expired_timestamp() { return delayed_expired_timestamp; }
198+
199+
private:
200+
std::shared_mutex _mtx;
201+
uint32_t _init_id = 0;
202+
std::unordered_map<std::string, uint32_t> _mapping_to_id;
203+
std::unordered_map<uint32_t, std::shared_ptr<FileMapping>> _id_map;
204+
205+
// use in Doris Format to keep temp rowsets, preventing them from being deleted by compaction
206+
std::unordered_map<std::pair<int64_t, RowsetId>, RowsetSharedPtr> _temp_rowset_maps;
207+
uint64_t delayed_expired_timestamp = 0;
208+
};
209+
210+
class IdManager {
211+
public:
212+
static constexpr uint8_t ID_VERSION = 0;
213+
214+
IdManager() = default;
215+
216+
~IdManager() {
217+
std::unique_lock lock(_query_to_id_file_map_mtx);
218+
_query_to_id_file_map.clear();
219+
}
220+
221+
std::shared_ptr<IdFileMap> add_id_file_map(const UniqueId& query_id, int timeout) {
222+
std::unique_lock lock(_query_to_id_file_map_mtx);
223+
auto it = _query_to_id_file_map.find(query_id);
224+
if (it == _query_to_id_file_map.end()) {
225+
auto id_file_map = std::make_shared<IdFileMap>(UnixSeconds() + timeout);
226+
_query_to_id_file_map[query_id] = id_file_map;
227+
return id_file_map;
228+
}
229+
return it->second;
230+
}
231+
232+
void gc_expired_id_file_map(int64_t now) {
233+
std::unique_lock lock(_query_to_id_file_map_mtx);
234+
for (auto it = _query_to_id_file_map.begin(); it != _query_to_id_file_map.end();) {
235+
if (it->second->get_delayed_expired_timestamp() <= now) {
236+
it = _query_to_id_file_map.erase(it);
237+
} else {
238+
++it;
239+
}
240+
}
241+
}
242+
243+
void remove_id_file_map(const UniqueId& query_id) {
244+
std::unique_lock lock(_query_to_id_file_map_mtx);
245+
_query_to_id_file_map.erase(query_id);
246+
}
247+
248+
std::shared_ptr<IdFileMap> get_id_file_map(const UniqueId& query_id) {
249+
std::shared_lock lock(_query_to_id_file_map_mtx);
250+
auto it = _query_to_id_file_map.find(query_id);
251+
if (it == _query_to_id_file_map.end()) {
252+
return nullptr;
253+
}
254+
return it->second;
255+
}
256+
257+
private:
258+
DISALLOW_COPY_AND_ASSIGN(IdManager);
259+
260+
phmap::flat_hash_map<UniqueId, std::shared_ptr<IdFileMap>> _query_to_id_file_map;
261+
std::shared_mutex _query_to_id_file_map_mtx;
262+
};
263+
264+
} // namespace doris

‎be/src/olap/rowset/segment_v2/column_reader.cpp‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1775,4 +1775,31 @@ Status DefaultNestedColumnIterator::read_by_rowids(const rowid_t* rowids, const
17751775
return Status::OK();
17761776
}
17771777

1778+
Status RowIdColumnIteratorV2::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
1779+
bool* has_null) {
1780+
auto* string_column = assert_cast<vectorized::ColumnString*>(dst.get());
1781+
1782+
for (size_t i = 0; i < *n; ++i) {
1783+
uint32_t row_id = _current_rowid + i;
1784+
GlobalRowLoacationV2 location(_version, _backend_id, _file_id, row_id);
1785+
string_column->insert_data(reinterpret_cast<const char*>(&location),
1786+
sizeof(GlobalRowLoacationV2));
1787+
}
1788+
_current_rowid += *n;
1789+
return Status::OK();
1790+
}
1791+
1792+
Status RowIdColumnIteratorV2::read_by_rowids(const rowid_t* rowids, const size_t count,
1793+
vectorized::MutableColumnPtr& dst) {
1794+
auto* string_column = assert_cast<vectorized::ColumnString*>(dst.get());
1795+
1796+
for (size_t i = 0; i < count; ++i) {
1797+
uint32_t row_id = rowids[i];
1798+
GlobalRowLoacationV2 location(_version, _backend_id, _file_id, row_id);
1799+
string_column->insert_data(reinterpret_cast<const char*>(&location),
1800+
sizeof(GlobalRowLoacationV2));
1801+
}
1802+
return Status::OK();
1803+
}
1804+
17781805
} // namespace doris::segment_v2

0 commit comments

Comments
 (0)