Skip to content

Commit 4b827eb

Browse files
committed
[Schema Change] support fast add/drop column (apache#49)
* [feature](schema-change) support fast schema change. coauthor: yixiutt * [schema change] Using columns desc from fe to read data. coauthor: Lchangliang * [feature](schema change) schema change optimize for add/drop columns. 1.add uniqueId field for class column. 2.schema change for add/drop columns directly update schema meta Co-authored-by: yixiutt <[email protected]> Co-authored-by: SWJTU-ZhangLei <[email protected]> [Feature](schema change) fix write and add regression test (apache#69) Co-authored-by: yixiutt <[email protected]> [schema change] be ssupport that delete use newest schema add delete regression test fix regression case (apache#107) tmp [feature](schema change) light schema change exclude rollup and agg/uniq/dup key type. [feature](schema change) fe olapTable maxUniqueId write in disk. [feature](schema change) add rpc iface for sc add column. [feature](schema change) add columnsDesc to TPushReq for ligtht sc. resolve the deadlock when schema change (apache#124) fix columns from fe don't has bitmap_index flag (apache#134) add update/delete case construct MATERIALIZED schema from origin schema when insert fix not vectorized compaction coredump use segment cache choose newest schema by schema version when compaction (apache#182) [bugfix](schema change) fix ligth schema change problem. [feature](schema change) light schema change add alter job. (#1) fix be ut [bug] (schema change) unique drop key column should not light schema change [feature](schema change) add schema change regression-test. fix regression test [bugfix](schema change) fix multi alter clauses for light schema change. (#2) [bugfix](schema change) fix multi clauses calculate column unique id (#3) modify PushTask process (apache#217) [Bugfix](schema change) fix jobId replay cause bdbje exception. [bug](schema change) fix max col unique id repeatitive. (apache#232) [optimize](schema change) modify pendingMaxColUniqueId generate rule. fix compaction error * fix be ut * fix snapshot load core fix unique_id error (apache#278) [refact](fe) remove redundant code for light schema change. (#4) [refact](fe) remove redundant code for light schema change. (#4) format fe core format be core fix be ut modify fe meta version fix rebase error
1 parent c936abd commit 4b827eb

98 files changed

Lines changed: 5191 additions & 750 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/exec/olap_scanner.cpp‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@
2323
#include "exprs/expr_context.h"
2424
#include "gen_cpp/PaloInternalService_types.h"
2525
#include "olap/decimal12.h"
26+
#include "olap/field.h"
2627
#include "olap/storage_engine.h"
28+
#include "olap/tablet_schema.h"
2729
#include "olap/uint24.h"
2830
#include "olap_scan_node.h"
2931
#include "olap_utils.h"
@@ -86,6 +88,14 @@ Status OlapScanner::prepare(
8688
LOG(WARNING) << ss.str();
8789
return Status::InternalError(ss.str());
8890
}
91+
_tablet_schema = _tablet->tablet_schema();
92+
if (!_parent->_olap_scan_node.columns_desc.empty() &&
93+
_parent->_olap_scan_node.columns_desc[0].col_unique_id >= 0) {
94+
_tablet_schema.clear_columns();
95+
for (const auto& column_desc : _parent->_olap_scan_node.columns_desc) {
96+
_tablet_schema.append_column(TabletColumn(column_desc));
97+
}
98+
}
8999
{
90100
std::shared_lock rdlock(_tablet->get_header_lock());
91101
const RowsetSharedPtr rowset = _tablet->rowset_with_max_version();
@@ -168,6 +178,7 @@ Status OlapScanner::_init_tablet_reader_params(
168178
RETURN_IF_ERROR(_init_return_columns(!_tablet_reader_params.direct_mode));
169179

170180
_tablet_reader_params.tablet = _tablet;
181+
_tablet_reader_params.tablet_schema = &_tablet_schema;
171182
_tablet_reader_params.reader_type = READER_QUERY;
172183
_tablet_reader_params.aggregation = _aggregation;
173184
_tablet_reader_params.version = Version(0, _version);
@@ -208,7 +219,7 @@ Status OlapScanner::_init_tablet_reader_params(
208219
_tablet_reader_params.return_columns.push_back(i);
209220
}
210221
for (auto index : _return_columns) {
211-
if (_tablet->tablet_schema().column(index).is_key()) {
222+
if (_tablet_schema.column(index).is_key()) {
212223
continue;
213224
} else {
214225
_tablet_reader_params.return_columns.push_back(index);
@@ -217,13 +228,12 @@ Status OlapScanner::_init_tablet_reader_params(
217228
}
218229

219230
// use _tablet_reader_params.return_columns, because reader use this to merge sort
220-
Status res =
221-
_read_row_cursor.init(_tablet->tablet_schema(), _tablet_reader_params.return_columns);
231+
Status res = _read_row_cursor.init(_tablet_schema, _tablet_reader_params.return_columns);
222232
if (!res.ok()) {
223233
LOG(WARNING) << "fail to init row cursor.res = " << res;
224234
return Status::InternalError("failed to initialize storage read row cursor");
225235
}
226-
_read_row_cursor.allocate_memory_for_string_type(_tablet->tablet_schema());
236+
_read_row_cursor.allocate_memory_for_string_type(_tablet_schema);
227237

228238
// If a agg node is this scan node direct parent
229239
// we will not call agg object finalize method in scan node,
@@ -242,15 +252,17 @@ Status OlapScanner::_init_return_columns(bool need_seq_col) {
242252
if (!slot->is_materialized()) {
243253
continue;
244254
}
245-
int32_t index = _tablet->field_index(slot->col_name());
255+
int32_t index = slot->col_unique_id() >= 0
256+
? _tablet_schema.field_index(slot->col_unique_id())
257+
: _tablet_schema.field_index(slot->col_name());
246258
if (index < 0) {
247259
std::stringstream ss;
248260
ss << "field name is invalid. field=" << slot->col_name();
249261
LOG(WARNING) << ss.str();
250262
return Status::InternalError(ss.str());
251263
}
252264
_return_columns.push_back(index);
253-
if (slot->is_nullable() && !_tablet->tablet_schema().column(index).is_nullable())
265+
if (slot->is_nullable() && !_tablet_schema.column(index).is_nullable())
254266
_tablet_columns_convert_to_null_set.emplace(index);
255267
_query_slots.push_back(slot);
256268
}
@@ -259,13 +271,13 @@ Status OlapScanner::_init_return_columns(bool need_seq_col) {
259271
if (_tablet->tablet_schema().has_sequence_col() && need_seq_col) {
260272
bool has_replace_col = false;
261273
for (auto col : _return_columns) {
262-
if (_tablet->tablet_schema().column(col).aggregation() ==
274+
if (_tablet_schema.column(col).aggregation() ==
263275
FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE) {
264276
has_replace_col = true;
265277
break;
266278
}
267279
}
268-
if (auto sequence_col_idx = _tablet->tablet_schema().sequence_col_idx();
280+
if (auto sequence_col_idx = _tablet_schema.sequence_col_idx();
269281
has_replace_col && std::find(_return_columns.begin(), _return_columns.end(),
270282
sequence_col_idx) == _return_columns.end()) {
271283
_return_columns.push_back(sequence_col_idx);

‎be/src/exec/olap_scanner.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,8 @@ class OlapScanner {
145145
MonotonicStopWatch _watcher;
146146

147147
std::shared_ptr<MemTracker> _mem_tracker;
148+
149+
TabletSchema _tablet_schema;
148150
};
149151

150152
} // namespace doris

‎be/src/exec/tablet_info.cpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ void OlapTableIndexSchema::to_protobuf(POlapTableIndexSchema* pindex) const {
3131
for (auto slot : slots) {
3232
pindex->add_columns(slot->col_name());
3333
}
34+
for (auto column : columns) {
35+
column->to_schema_pb(pindex->add_columns_desc());
36+
}
3437
}
3538

3639
Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) {
@@ -57,6 +60,11 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) {
5760
}
5861
index->slots.emplace_back(it->second);
5962
}
63+
for (auto& pcolumn_desc : p_index.columns_desc()) {
64+
TabletColumn* tc = _obj_pool.add(new TabletColumn());
65+
tc->init_from_pb(pcolumn_desc);
66+
index->columns.emplace_back(tc);
67+
}
6068
_indexes.emplace_back(index);
6169
}
6270

@@ -90,6 +98,11 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) {
9098
}
9199
index->slots.emplace_back(it->second);
92100
}
101+
for (auto& tcolumn_desc : t_index.columns_desc) {
102+
TabletColumn* tc = _obj_pool.add(new TabletColumn());
103+
tc->init_from_thrift(tcolumn_desc);
104+
index->columns.emplace_back(tc);
105+
}
93106
_indexes.emplace_back(index);
94107
}
95108

‎be/src/exec/tablet_info.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include "common/status.h"
2828
#include "gen_cpp/Descriptors_types.h"
2929
#include "gen_cpp/descriptors.pb.h"
30+
#include "olap/tablet_schema.h"
3031
#include "runtime/descriptors.h"
3132
#include "runtime/raw_value.h"
3233
#include "runtime/tuple.h"
@@ -41,6 +42,7 @@ struct OlapTableIndexSchema {
4142
int64_t index_id;
4243
std::vector<SlotDescriptor*> slots;
4344
int32_t schema_hash;
45+
std::vector<TabletColumn*> columns;
4446

4547
void to_protobuf(POlapTableIndexSchema* pindex) const;
4648
};

‎be/src/olap/base_tablet.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ class BaseTablet : public std::enable_shared_from_this<BaseTablet> {
6060
bool equal(int64_t tablet_id, int32_t schema_hash);
6161

6262
// properties encapsulated in TabletSchema
63-
const TabletSchema& tablet_schema() const;
63+
virtual const TabletSchema& tablet_schema() const;
6464

6565
protected:
6666
void _gen_tablet_path();

‎be/src/olap/collect_iterator.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,9 @@ Status CollectIterator::add_child(RowsetReaderSharedPtr rs_reader) {
5555
// then merged with the base rowset.
5656
void CollectIterator::build_heap(const std::vector<RowsetReaderSharedPtr>& rs_readers) {
5757
DCHECK(rs_readers.size() == _children.size());
58-
_reverse = _reader->_tablet->tablet_schema().keys_type() == KeysType::UNIQUE_KEYS;
59-
SortType sort_type = _reader->_tablet->tablet_schema().sort_type();
60-
int sort_col_num = _reader->_tablet->tablet_schema().sort_col_num();
58+
_reverse = _reader->_tablet_schema->keys_type() == KeysType::UNIQUE_KEYS;
59+
SortType sort_type = _reader->_tablet_schema->sort_type();
60+
int sort_col_num = _reader->_tablet_schema->sort_col_num();
6161
if (_children.empty()) {
6262
_inner_iter.reset(nullptr);
6363
return;
@@ -200,7 +200,7 @@ CollectIterator::Level0Iterator::Level0Iterator(RowsetReaderSharedPtr rs_reader,
200200
CollectIterator::Level0Iterator::~Level0Iterator() = default;
201201

202202
Status CollectIterator::Level0Iterator::init() {
203-
RETURN_NOT_OK_LOG(_row_cursor.init(_reader->_tablet->tablet_schema(), _reader->_seek_columns),
203+
RETURN_NOT_OK_LOG(_row_cursor.init(*_reader->_tablet_schema, _reader->_seek_columns),
204204
"failed to init row cursor");
205205
return (this->*_refresh_current_row)();
206206
}

‎be/src/olap/compaction.cpp‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -138,8 +138,21 @@ Status Compaction::do_compaction_impl(int64_t permits) {
138138

139139
LOG(INFO) << "start " << merge_type << compaction_name() << ". tablet=" << _tablet->full_name()
140140
<< ", output_version=" << _output_version << ", permits: " << permits;
141+
const TabletSchema* cur_tablet_schema = nullptr;
142+
if (_input_rowsets.front()->rowset_meta()->tablet_schema() == nullptr) {
143+
cur_tablet_schema = &(_tablet->tablet_schema());
144+
} else {
145+
// get cur schema if rowset schema exist, rowset schema must be newer than tablet schema
146+
auto max_version_rowset =
147+
std::max_element(_input_rowsets.begin(), _input_rowsets.end(),
148+
[](const RowsetSharedPtr& a, const RowsetSharedPtr& b) {
149+
return a->rowset_meta()->tablet_schema()->schema_version() <
150+
b->rowset_meta()->tablet_schema()->schema_version();
151+
});
152+
cur_tablet_schema = (*max_version_rowset)->rowset_meta()->tablet_schema();
153+
}
141154

142-
RETURN_NOT_OK(construct_output_rowset_writer());
155+
RETURN_NOT_OK(construct_output_rowset_writer(cur_tablet_schema));
143156
RETURN_NOT_OK(construct_input_rowset_readers());
144157
TRACE("prepare finished");
145158

@@ -149,11 +162,11 @@ Status Compaction::do_compaction_impl(int64_t permits) {
149162
Status res;
150163

151164
if (use_vectorized_compaction) {
152-
res = Merger::vmerge_rowsets(_tablet, compaction_type(), _input_rs_readers,
153-
_output_rs_writer.get(), &stats);
165+
res = Merger::vmerge_rowsets(_tablet, compaction_type(), cur_tablet_schema,
166+
_input_rs_readers, _output_rs_writer.get(), &stats);
154167
} else {
155-
res = Merger::merge_rowsets(_tablet, compaction_type(), _input_rs_readers,
156-
_output_rs_writer.get(), &stats);
168+
res = Merger::merge_rowsets(_tablet, compaction_type(), cur_tablet_schema,
169+
_input_rs_readers, _output_rs_writer.get(), &stats);
157170
}
158171

159172
if (!res.ok()) {
@@ -216,8 +229,8 @@ Status Compaction::do_compaction_impl(int64_t permits) {
216229
return Status::OK();
217230
}
218231

219-
Status Compaction::construct_output_rowset_writer() {
220-
return _tablet->create_rowset_writer(_output_version, VISIBLE, NONOVERLAPPING,
232+
Status Compaction::construct_output_rowset_writer(const TabletSchema* schema) {
233+
return _tablet->create_rowset_writer(_output_version, VISIBLE, NONOVERLAPPING, schema,
221234
&_output_rs_writer);
222235
}
223236

‎be/src/olap/compaction.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ class Compaction {
6767
Status modify_rowsets();
6868
void gc_output_rowset();
6969

70-
Status construct_output_rowset_writer();
70+
Status construct_output_rowset_writer(const TabletSchema* schama);
7171
Status construct_input_rowset_readers();
7272

7373
Status check_version_continuity(const std::vector<RowsetSharedPtr>& rowsets);

‎be/src/olap/convert_rowset.cpp‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "olap/convert_rowset.h"
19+
20+
namespace doris {
21+
22+
Status ConvertRowset::do_convert() {
23+
if (!_tablet->init_succeeded()) {
24+
return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR);
25+
}
26+
std::unique_lock<std::mutex> base_compaction_lock(_tablet->get_base_compaction_lock(),
27+
std::try_to_lock);
28+
std::unique_lock<std::mutex> cumulative_compaction_lock(
29+
_tablet->get_cumulative_compaction_lock(), std::try_to_lock);
30+
if (!base_compaction_lock.owns_lock() || !cumulative_compaction_lock.owns_lock()) {
31+
LOG(INFO) << "The tablet is under compaction. tablet=" << _tablet->full_name();
32+
return Status::OLAPInternalError(OLAP_ERR_CE_TRY_CE_LOCK_ERROR);
33+
}
34+
35+
std::vector<RowsetSharedPtr> alpah_rowsets;
36+
_tablet->find_alpha_rowsets(&alpah_rowsets);
37+
38+
Merger::Statistics stats;
39+
Status res;
40+
const size_t max_convert_row_count = 20000000;
41+
size_t row_count = 0;
42+
for (size_t i = 0; i < alpah_rowsets.size(); ++i) {
43+
Version output_version =
44+
Version(alpah_rowsets[i]->start_version(), alpah_rowsets[i]->end_version());
45+
46+
RowsetReaderSharedPtr input_rs_reader;
47+
RETURN_NOT_OK(alpah_rowsets[i]->create_reader(&input_rs_reader));
48+
49+
std::unique_ptr<RowsetWriter> output_rs_writer;
50+
RETURN_NOT_OK(_tablet->create_rowset_writer(output_version, VISIBLE, NONOVERLAPPING,
51+
&_tablet->tablet_schema(), &output_rs_writer));
52+
res = Merger::merge_rowsets(_tablet, ReaderType::READER_BASE_COMPACTION,
53+
&_tablet->tablet_schema(), {input_rs_reader},
54+
output_rs_writer.get(), &stats);
55+
56+
if (!res.ok()) {
57+
LOG(WARNING) << "fail to convert rowset. res=" << res
58+
<< ", tablet=" << _tablet->full_name();
59+
return res;
60+
} else {
61+
auto output_rowset = output_rs_writer->build();
62+
if (output_rowset == nullptr) {
63+
LOG(WARNING) << "rowset writer build failed"
64+
<< ", tablet=" << _tablet->full_name();
65+
return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR);
66+
}
67+
68+
RETURN_NOT_OK(check_correctness(alpah_rowsets[i], output_rowset, stats));
69+
70+
row_count += alpah_rowsets[i]->num_rows();
71+
72+
RETURN_NOT_OK(_modify_rowsets(alpah_rowsets[i], output_rowset));
73+
74+
LOG(INFO) << "succeed to convert rowset"
75+
<< ". tablet=" << _tablet->full_name()
76+
<< ", output_version=" << output_version
77+
<< ", disk=" << _tablet->data_dir()->path();
78+
79+
if (row_count >= max_convert_row_count) {
80+
break;
81+
}
82+
}
83+
}
84+
return Status::OK();
85+
}
86+
87+
Status ConvertRowset::check_correctness(RowsetSharedPtr input_rowset, RowsetSharedPtr output_rowset,
88+
const Merger::Statistics& stats) {
89+
// 1. check row number
90+
if (input_rowset->num_rows() !=
91+
output_rowset->num_rows() + stats.merged_rows + stats.filtered_rows) {
92+
LOG(WARNING) << "row_num does not match between input and output! "
93+
<< "input_row_num=" << input_rowset->num_rows()
94+
<< ", merged_row_num=" << stats.merged_rows
95+
<< ", filtered_row_num=" << stats.filtered_rows
96+
<< ", output_row_num=" << output_rowset->num_rows();
97+
98+
// ATTN(cmy): We found that the num_rows in some rowset meta may be set to the wrong value,
99+
// but it is not known which version of the code has the problem. So when the convert
100+
// result is inconsistent, we then try to verify by num_rows recorded in segment_groups.
101+
// If the check passes, ignore the error and set the correct value in the output rowset meta
102+
// to fix this problem.
103+
// Only handle alpha rowset because we only find this bug in alpha rowset
104+
int64_t num_rows = _get_input_num_rows_from_seg_grps(input_rowset);
105+
if (num_rows == -1) {
106+
return Status::OLAPInternalError(OLAP_ERR_CHECK_LINES_ERROR);
107+
}
108+
if (num_rows != output_rowset->num_rows() + stats.merged_rows + stats.filtered_rows) {
109+
// If it is still incorrect, it may be another problem
110+
LOG(WARNING) << "row_num got from seg groups does not match between cumulative input "
111+
"and output! "
112+
<< "input_row_num=" << num_rows << ", merged_row_num=" << stats.merged_rows
113+
<< ", filtered_row_num=" << stats.filtered_rows
114+
<< ", output_row_num=" << output_rowset->num_rows();
115+
116+
return Status::OLAPInternalError(OLAP_ERR_CHECK_LINES_ERROR);
117+
}
118+
}
119+
return Status::OK();
120+
}
121+
122+
int64_t ConvertRowset::_get_input_num_rows_from_seg_grps(RowsetSharedPtr rowset) {
123+
int64_t num_rows = 0;
124+
for (auto& seg_grp : rowset->rowset_meta()->alpha_rowset_extra_meta_pb().segment_groups()) {
125+
num_rows += seg_grp.num_rows();
126+
}
127+
return num_rows;
128+
}
129+
Status ConvertRowset::_modify_rowsets(RowsetSharedPtr input_rowset, RowsetSharedPtr output_rowset) {
130+
std::vector<RowsetSharedPtr> input_rowsets;
131+
input_rowsets.push_back(input_rowset);
132+
133+
std::vector<RowsetSharedPtr> output_rowsets;
134+
output_rowsets.push_back(output_rowset);
135+
136+
std::lock_guard<std::shared_mutex> wrlock(_tablet->get_header_lock());
137+
RETURN_NOT_OK(_tablet->modify_rowsets(output_rowsets, input_rowsets, true));
138+
_tablet->save_meta();
139+
return Status::OK();
140+
}
141+
} // namespace doris

0 commit comments

Comments
 (0)