Skip to content

Commit 3fe2a6b

Browse files
Lchangliangyixiuttswjtu-zhanglei
committed
[Schema Change] support fast add/drop column (apache#49)
* [feature](schema-change) support fast schema change. coauthor: yixiutt * [schema change] Using columns desc from fe to read data. coauthor: Lchangliang * [feature](schema change) schema change optimize for add/drop columns. 1.add uniqueId field for class column. 2.schema change for add/drop columns directly update schema meta Co-authored-by: yixiutt <[email protected]> Co-authored-by: SWJTU-ZhangLei <[email protected]>
1 parent 904e757 commit 3fe2a6b

60 files changed

Lines changed: 878 additions & 140 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/exec/olap_scanner.cpp‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,14 @@ Status OlapScanner::prepare(
8686
LOG(WARNING) << ss.str();
8787
return Status::InternalError(ss.str());
8888
}
89+
_tablet_schema = _tablet->tablet_schema();
90+
if (!_parent->_olap_scan_node.columns_desc.empty() &&
91+
_parent->_olap_scan_node.columns_desc[0].col_unique_id >= 0) {
92+
_tablet_schema.clear_columns();
93+
for (const auto& column_desc : _parent->_olap_scan_node.columns_desc) {
94+
_tablet_schema.append_column(TabletColumn(column_desc));
95+
}
96+
}
8997
{
9098
std::shared_lock rdlock(_tablet->get_header_lock());
9199
const RowsetSharedPtr rowset = _tablet->rowset_with_max_version();
@@ -168,6 +176,7 @@ Status OlapScanner::_init_tablet_reader_params(
168176
RETURN_IF_ERROR(_init_return_columns(!_tablet_reader_params.direct_mode));
169177

170178
_tablet_reader_params.tablet = _tablet;
179+
_tablet_reader_params.tablet_schema = &_tablet_schema;
171180
_tablet_reader_params.reader_type = READER_QUERY;
172181
_tablet_reader_params.aggregation = _aggregation;
173182
_tablet_reader_params.version = Version(0, _version);
@@ -208,7 +217,7 @@ Status OlapScanner::_init_tablet_reader_params(
208217
_tablet_reader_params.return_columns.push_back(i);
209218
}
210219
for (auto index : _return_columns) {
211-
if (_tablet->tablet_schema().column(index).is_key()) {
220+
if (_tablet_schema.column(index).is_key()) {
212221
continue;
213222
} else {
214223
_tablet_reader_params.return_columns.push_back(index);
@@ -217,13 +226,12 @@ Status OlapScanner::_init_tablet_reader_params(
217226
}
218227

219228
// use _tablet_reader_params.return_columns, because reader use this to merge sort
220-
Status res =
221-
_read_row_cursor.init(_tablet->tablet_schema(), _tablet_reader_params.return_columns);
229+
Status res = _read_row_cursor.init(_tablet_schema, _tablet_reader_params.return_columns);
222230
if (!res.ok()) {
223231
LOG(WARNING) << "fail to init row cursor.res = " << res;
224232
return Status::InternalError("failed to initialize storage read row cursor");
225233
}
226-
_read_row_cursor.allocate_memory_for_string_type(_tablet->tablet_schema());
234+
_read_row_cursor.allocate_memory_for_string_type(_tablet_schema);
227235

228236
// If a agg node is this scan node direct parent
229237
// we will not call agg object finalize method in scan node,
@@ -242,15 +250,16 @@ Status OlapScanner::_init_return_columns(bool need_seq_col) {
242250
if (!slot->is_materialized()) {
243251
continue;
244252
}
245-
int32_t index = _tablet->field_index(slot->col_name());
253+
int32_t index = slot->col_unique_id() >= 0 ? slot->col_unique_id()
254+
: _tablet_schema.field_index(slot->col_name());
246255
if (index < 0) {
247256
std::stringstream ss;
248257
ss << "field name is invalid. field=" << slot->col_name();
249258
LOG(WARNING) << ss.str();
250259
return Status::InternalError(ss.str());
251260
}
252261
_return_columns.push_back(index);
253-
if (slot->is_nullable() && !_tablet->tablet_schema().column(index).is_nullable())
262+
if (slot->is_nullable() && !_tablet_schema.column(index).is_nullable())
254263
_tablet_columns_convert_to_null_set.emplace(index);
255264
_query_slots.push_back(slot);
256265
}
@@ -259,13 +268,13 @@ Status OlapScanner::_init_return_columns(bool need_seq_col) {
259268
if (_tablet->tablet_schema().has_sequence_col() && need_seq_col) {
260269
bool has_replace_col = false;
261270
for (auto col : _return_columns) {
262-
if (_tablet->tablet_schema().column(col).aggregation() ==
271+
if (_tablet_schema.column(col).aggregation() ==
263272
FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE) {
264273
has_replace_col = true;
265274
break;
266275
}
267276
}
268-
if (auto sequence_col_idx = _tablet->tablet_schema().sequence_col_idx();
277+
if (auto sequence_col_idx = _tablet_schema.sequence_col_idx();
269278
has_replace_col && std::find(_return_columns.begin(), _return_columns.end(),
270279
sequence_col_idx) == _return_columns.end()) {
271280
_return_columns.push_back(sequence_col_idx);

‎be/src/exec/olap_scanner.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,8 @@ class OlapScanner {
145145
MonotonicStopWatch _watcher;
146146

147147
std::shared_ptr<MemTracker> _mem_tracker;
148+
149+
TabletSchema _tablet_schema;
148150
};
149151

150152
} // namespace doris

‎be/src/exec/tablet_info.cpp‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) {
6161
}
6262
_indexes.emplace_back(index);
6363
}
64+
for (auto& pcolumn : pschema.columns()) {
65+
TabletColumn* tc = _obj_pool.add(new TabletColumn());
66+
tc->init_from_pb(pcolumn);
67+
_columns.emplace_back(tc);
68+
}
6469

6570
std::sort(_indexes.begin(), _indexes.end(),
6671
[](const OlapTableIndexSchema* lhs, const OlapTableIndexSchema* rhs) {
@@ -96,6 +101,11 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) {
96101
}
97102
_indexes.emplace_back(index);
98103
}
104+
for (auto& tcolumn : tschema.columns) {
105+
TabletColumn* tc = _obj_pool.add(new TabletColumn());
106+
tc->init_from_thrift(tcolumn);
107+
_columns.emplace_back(tc);
108+
}
99109

100110
std::sort(_indexes.begin(), _indexes.end(),
101111
[](const OlapTableIndexSchema* lhs, const OlapTableIndexSchema* rhs) {
@@ -115,11 +125,18 @@ void OlapTableSchemaParam::to_protobuf(POlapTableSchemaParam* pschema) const {
115125
for (auto index : _indexes) {
116126
index->to_protobuf(pschema->add_indexes());
117127
}
128+
for (auto column : _columns) {
129+
column->to_schema_pb(pschema->add_columns());
130+
}
118131
}
119132

120133
std::string OlapTableSchemaParam::debug_string() const {
121134
std::stringstream ss;
122135
ss << "tuple_desc=" << _tuple_desc->debug_string();
136+
ss << "columns: ";
137+
for (auto column : _columns) {
138+
ss << column->unique_id() << ":" << column->name() << " ";
139+
}
123140
return ss.str();
124141
}
125142

‎be/src/exec/tablet_info.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ class OlapTableSchemaParam {
5858
int64_t version() const { return _version; }
5959

6060
TupleDescriptor* tuple_desc() const { return _tuple_desc; }
61+
const std::vector<TabletColumn*>& columns() const {return _columns;}
6162
const std::vector<OlapTableIndexSchema*>& indexes() const { return _indexes; }
6263

6364
void to_protobuf(POlapTableSchemaParam* pschema) const;
@@ -81,6 +82,7 @@ class OlapTableSchemaParam {
8182
TupleDescriptor* _tuple_desc = nullptr;
8283
mutable POlapTableSchemaParam* _proto_schema = nullptr;
8384
std::vector<OlapTableIndexSchema*> _indexes;
85+
std::vector<TabletColumn*> _columns;
8486
mutable ObjectPool _obj_pool;
8587
};
8688

‎be/src/exec/tablet_sink.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ void NodeChannel::open() {
125125
request.set_index_id(_index_channel->_index_id);
126126
request.set_txn_id(_parent->_txn_id);
127127
request.set_allocated_schema(_parent->_schema->to_protobuf());
128+
LOG(INFO) << "yixiu proto: " << _parent->_schema->to_protobuf()->DebugString();
128129
for (auto& tablet : _all_tablets) {
129130
auto ptablet = request.add_tablets();
130131
ptablet->set_partition_id(tablet.partition_id);
@@ -682,6 +683,7 @@ Status OlapTableSink::init(const TDataSink& t_sink) {
682683
_tuple_desc_id = table_sink.tuple_id;
683684
_schema.reset(new OlapTableSchemaParam());
684685
RETURN_IF_ERROR(_schema->init(table_sink.schema));
686+
LOG(INFO) << "yixiu thrift:" << apache::thrift::ThriftDebugString(table_sink.schema);
685687
_partition = _pool->add(new OlapTablePartitionParam(_schema, table_sink.partition));
686688
RETURN_IF_ERROR(_partition->init());
687689
_location = _pool->add(new OlapTableLocationParam(table_sink.location));

‎be/src/olap/base_tablet.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ class BaseTablet : public std::enable_shared_from_this<BaseTablet> {
6060
bool equal(int64_t tablet_id, int32_t schema_hash);
6161

6262
// properties encapsulated in TabletSchema
63-
const TabletSchema& tablet_schema() const;
63+
virtual const TabletSchema& tablet_schema() const;
6464

6565
protected:
6666
void _gen_tablet_path();

‎be/src/olap/collect_iterator.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,9 @@ Status CollectIterator::add_child(RowsetReaderSharedPtr rs_reader) {
5555
// then merged with the base rowset.
5656
void CollectIterator::build_heap(const std::vector<RowsetReaderSharedPtr>& rs_readers) {
5757
DCHECK(rs_readers.size() == _children.size());
58-
_reverse = _reader->_tablet->tablet_schema().keys_type() == KeysType::UNIQUE_KEYS;
59-
SortType sort_type = _reader->_tablet->tablet_schema().sort_type();
60-
int sort_col_num = _reader->_tablet->tablet_schema().sort_col_num();
58+
_reverse = _reader->_tablet_schema->keys_type() == KeysType::UNIQUE_KEYS;
59+
SortType sort_type = _reader->_tablet_schema->sort_type();
60+
int sort_col_num = _reader->_tablet_schema->sort_col_num();
6161
if (_children.empty()) {
6262
_inner_iter.reset(nullptr);
6363
return;
@@ -200,7 +200,7 @@ CollectIterator::Level0Iterator::Level0Iterator(RowsetReaderSharedPtr rs_reader,
200200
CollectIterator::Level0Iterator::~Level0Iterator() = default;
201201

202202
Status CollectIterator::Level0Iterator::init() {
203-
RETURN_NOT_OK_LOG(_row_cursor.init(_reader->_tablet->tablet_schema(), _reader->_seek_columns),
203+
RETURN_NOT_OK_LOG(_row_cursor.init(*_reader->_tablet_schema, _reader->_seek_columns),
204204
"failed to init row cursor");
205205
return (this->*_refresh_current_row)();
206206
}

‎be/src/olap/compaction.cpp‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
#include "olap/compaction.h"
1919

20+
#include "delta_writer.h"
2021
#include "gutil/strings/substitute.h"
2122
#include "util/time.h"
2223
#include "util/trace.h"
@@ -139,7 +140,14 @@ Status Compaction::do_compaction_impl(int64_t permits) {
139140
LOG(INFO) << "start " << merge_type << compaction_name() << ". tablet=" << _tablet->full_name()
140141
<< ", output_version=" << _output_version << ", permits: " << permits;
141142

142-
RETURN_NOT_OK(construct_output_rowset_writer());
143+
// get cur schema if rowset schema exist, rowset schema must be newer than tablet schema
144+
auto max_version_rowset = _input_rowsets.back();
145+
const TabletSchema* cur_tablet_schema = max_version_rowset->rowset_meta()->tablet_schema();
146+
if (cur_tablet_schema == nullptr) {
147+
cur_tablet_schema = &(_tablet->tablet_schema());
148+
}
149+
150+
RETURN_NOT_OK(construct_output_rowset_writer(cur_tablet_schema));
143151
RETURN_NOT_OK(construct_input_rowset_readers());
144152
TRACE("prepare finished");
145153

@@ -149,10 +157,10 @@ Status Compaction::do_compaction_impl(int64_t permits) {
149157
Status res;
150158

151159
if (use_vectorized_compaction) {
152-
res = Merger::vmerge_rowsets(_tablet, compaction_type(), _input_rs_readers,
160+
res = Merger::vmerge_rowsets(_tablet, compaction_type(), cur_tablet_schema, _input_rs_readers,
153161
_output_rs_writer.get(), &stats);
154162
} else {
155-
res = Merger::merge_rowsets(_tablet, compaction_type(), _input_rs_readers,
163+
res = Merger::merge_rowsets(_tablet, compaction_type(), cur_tablet_schema, _input_rs_readers,
156164
_output_rs_writer.get(), &stats);
157165
}
158166

@@ -216,8 +224,8 @@ Status Compaction::do_compaction_impl(int64_t permits) {
216224
return Status::OK();
217225
}
218226

219-
Status Compaction::construct_output_rowset_writer() {
220-
return _tablet->create_rowset_writer(_output_version, VISIBLE, NONOVERLAPPING,
227+
Status Compaction::construct_output_rowset_writer(const TabletSchema* schema) {
228+
return _tablet->create_rowset_writer(_output_version, VISIBLE, NONOVERLAPPING, schema,
221229
&_output_rs_writer);
222230
}
223231

‎be/src/olap/compaction.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ class Compaction {
6767
Status modify_rowsets();
6868
void gc_output_rowset();
6969

70-
Status construct_output_rowset_writer();
70+
Status construct_output_rowset_writer(const TabletSchema* schama);
7171
Status construct_input_rowset_readers();
7272

7373
Status check_version_continuity(const std::vector<RowsetSharedPtr>& rowsets);

‎be/src/olap/delta_writer.cpp‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ DeltaWriter::DeltaWriter(WriteRequest* req, StorageEngine* storage_engine, bool
4040
_tablet(nullptr),
4141
_cur_rowset(nullptr),
4242
_rowset_writer(nullptr),
43-
_tablet_schema(nullptr),
43+
_tablet_schema(new TabletSchema),
4444
_delta_written_success(false),
4545
_storage_engine(storage_engine),
4646
_is_vec(is_vec) {}
@@ -121,9 +121,11 @@ Status DeltaWriter::init() {
121121
RETURN_NOT_OK(_storage_engine->txn_manager()->prepare_txn(_req.partition_id, _tablet,
122122
_req.txn_id, _req.load_id));
123123
}
124+
// build tablet schema in request level
125+
_build_current_tablet_schema(_req.ptable_schema_param, _tablet->tablet_schema());
124126

125127
RETURN_NOT_OK(_tablet->create_rowset_writer(_req.txn_id, _req.load_id, PREPARED, OVERLAPPING,
126-
&_rowset_writer));
128+
&_tablet->tablet_schema(), &_rowset_writer));
127129
_tablet_schema = &(_tablet->tablet_schema());
128130
_schema.reset(new Schema(*_tablet_schema));
129131
_reset_mem_table();
@@ -266,9 +268,9 @@ Status DeltaWriter::wait_flush() {
266268
}
267269

268270
void DeltaWriter::_reset_mem_table() {
269-
_mem_table.reset(new MemTable(_tablet->tablet_id(), _schema.get(), _tablet_schema, _req.slots,
270-
_req.tuple_desc, _tablet->keys_type(), _rowset_writer.get(),
271-
_mem_tracker, _is_vec));
271+
_mem_table.reset(new MemTable(_tablet->tablet_id(), _schema.get(), _tablet_schema.get(),
272+
_req.slots, _req.tuple_desc, _tablet->keys_type(),
273+
_rowset_writer.get(), _mem_tracker, _is_vec));
272274
}
273275

274276
Status DeltaWriter::close() {
@@ -367,4 +369,16 @@ int64_t DeltaWriter::partition_id() const {
367369
return _req.partition_id;
368370
}
369371

372+
void DeltaWriter::_build_current_tablet_schema(const POlapTableSchemaParam& ptable_schema_param,
373+
const TabletSchema& ori_tablet_schema) {
374+
*_tablet_schema = ori_tablet_schema;
375+
LOG(INFO) << "build current tablet schema";
376+
LOG(INFO) << "yixiu: " << ptable_schema_param.DebugString();
377+
//new tablet schame if new table
378+
if (ptable_schema_param.columns_size() != 0 && ptable_schema_param.columns(0).unique_id() > 0) {
379+
LOG(INFO) << "yixiu new table";
380+
_tablet_schema->build_current_tablet_schema(ptable_schema_param, ori_tablet_schema);
381+
}
382+
}
383+
370384
} // namespace doris

0 commit comments

Comments
 (0)