Skip to content

Commit 8a66e96

Browse files
Lchangliangyixiuttswjtu-zhanglei
committed
[Schema Change] support fast add/drop column (apache#49)
* [feature](schema-change) support fast schema change. coauthor: yixiutt * [schema change] Using columns desc from fe to read data. coauthor: Lchangliang * [feature](schema change) schema change optimize for add/drop columns. 1.add uniqueId field for class column. 2.schema change for add/drop columns directly update schema meta Co-authored-by: yixiutt <[email protected]> Co-authored-by: SWJTU-ZhangLei <[email protected]>
1 parent 050cbba commit 8a66e96

60 files changed

Lines changed: 875 additions & 143 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/exec/olap_scanner.cpp‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,14 @@ Status OlapScanner::prepare(
8686
LOG(WARNING) << ss.str();
8787
return Status::InternalError(ss.str());
8888
}
89+
_tablet_schema = _tablet->tablet_schema();
90+
if (!_parent->_olap_scan_node.columns_desc.empty() &&
91+
_parent->_olap_scan_node.columns_desc[0].col_unique_id >= 0) {
92+
_tablet_schema.clear_columns();
93+
for (const auto& column_desc : _parent->_olap_scan_node.columns_desc) {
94+
_tablet_schema.append_column(TabletColumn(column_desc));
95+
}
96+
}
8997
{
9098
std::shared_lock rdlock(_tablet->get_header_lock());
9199
const RowsetSharedPtr rowset = _tablet->rowset_with_max_version();
@@ -150,6 +158,7 @@ Status OlapScanner::_init_tablet_reader_params(
150158
RETURN_IF_ERROR(_init_return_columns());
151159

152160
_tablet_reader_params.tablet = _tablet;
161+
_tablet_reader_params.tablet_schema = &_tablet_schema;
153162
_tablet_reader_params.reader_type = READER_QUERY;
154163
_tablet_reader_params.aggregation = _aggregation;
155164
_tablet_reader_params.version = Version(0, _version);
@@ -207,7 +216,7 @@ Status OlapScanner::_init_tablet_reader_params(
207216
_tablet_reader_params.return_columns.push_back(i);
208217
}
209218
for (auto index : _return_columns) {
210-
if (_tablet->tablet_schema().column(index).is_key()) {
219+
if (_tablet_schema.column(index).is_key()) {
211220
continue;
212221
} else {
213222
_tablet_reader_params.return_columns.push_back(index);
@@ -216,13 +225,12 @@ Status OlapScanner::_init_tablet_reader_params(
216225
}
217226

218227
// use _tablet_reader_params.return_columns, because reader use this to merge sort
219-
Status res =
220-
_read_row_cursor.init(_tablet->tablet_schema(), _tablet_reader_params.return_columns);
228+
Status res = _read_row_cursor.init(_tablet_schema, _tablet_reader_params.return_columns);
221229
if (!res.ok()) {
222230
LOG(WARNING) << "fail to init row cursor.res = " << res;
223231
return Status::InternalError("failed to initialize storage read row cursor");
224232
}
225-
_read_row_cursor.allocate_memory_for_string_type(_tablet->tablet_schema());
233+
_read_row_cursor.allocate_memory_for_string_type(_tablet_schema);
226234

227235
// If a agg node is this scan node direct parent
228236
// we will not call agg object finalize method in scan node,
@@ -241,30 +249,31 @@ Status OlapScanner::_init_return_columns() {
241249
if (!slot->is_materialized()) {
242250
continue;
243251
}
244-
int32_t index = _tablet->field_index(slot->col_name());
252+
int32_t index = slot->col_unique_id() >= 0 ? slot->col_unique_id()
253+
: _tablet_schema.field_index(slot->col_name());
245254
if (index < 0) {
246255
std::stringstream ss;
247256
ss << "field name is invalid. field=" << slot->col_name();
248257
LOG(WARNING) << ss.str();
249258
return Status::InternalError(ss.str());
250259
}
251260
_return_columns.push_back(index);
252-
if (slot->is_nullable() && !_tablet->tablet_schema().column(index).is_nullable())
261+
if (slot->is_nullable() && !_tablet_schema.column(index).is_nullable())
253262
_tablet_columns_convert_to_null_set.emplace(index);
254263
_query_slots.push_back(slot);
255264
}
256265

257266
// expand the sequence column
258-
if (_tablet->tablet_schema().has_sequence_col()) {
267+
if (_tablet_schema.has_sequence_col()) {
259268
bool has_replace_col = false;
260269
for (auto col : _return_columns) {
261-
if (_tablet->tablet_schema().column(col).aggregation() ==
270+
if (_tablet_schema.column(col).aggregation() ==
262271
FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE) {
263272
has_replace_col = true;
264273
break;
265274
}
266275
}
267-
if (auto sequence_col_idx = _tablet->tablet_schema().sequence_col_idx();
276+
if (auto sequence_col_idx = _tablet_schema.sequence_col_idx();
268277
has_replace_col && std::find(_return_columns.begin(), _return_columns.end(),
269278
sequence_col_idx) == _return_columns.end()) {
270279
_return_columns.push_back(sequence_col_idx);

‎be/src/exec/olap_scanner.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,8 @@ class OlapScanner {
151151
MonotonicStopWatch _watcher;
152152

153153
std::shared_ptr<MemTracker> _mem_tracker;
154+
155+
TabletSchema _tablet_schema;
154156
};
155157

156158
} // namespace doris

‎be/src/exec/tablet_info.cpp‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) {
6161
}
6262
_indexes.emplace_back(index);
6363
}
64+
for (auto& pcolumn : pschema.columns()) {
65+
TabletColumn* tc = _obj_pool.add(new TabletColumn());
66+
tc->init_from_pb(pcolumn);
67+
_columns.emplace_back(tc);
68+
}
6469

6570
std::sort(_indexes.begin(), _indexes.end(),
6671
[](const OlapTableIndexSchema* lhs, const OlapTableIndexSchema* rhs) {
@@ -96,6 +101,11 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) {
96101
}
97102
_indexes.emplace_back(index);
98103
}
104+
for (auto& tcolumn : tschema.columns) {
105+
TabletColumn* tc = _obj_pool.add(new TabletColumn());
106+
tc->init_from_thrift(tcolumn);
107+
_columns.emplace_back(tc);
108+
}
99109

100110
std::sort(_indexes.begin(), _indexes.end(),
101111
[](const OlapTableIndexSchema* lhs, const OlapTableIndexSchema* rhs) {
@@ -115,11 +125,18 @@ void OlapTableSchemaParam::to_protobuf(POlapTableSchemaParam* pschema) const {
115125
for (auto index : _indexes) {
116126
index->to_protobuf(pschema->add_indexes());
117127
}
128+
for (auto column : _columns) {
129+
column->to_schema_pb(pschema->add_columns());
130+
}
118131
}
119132

120133
std::string OlapTableSchemaParam::debug_string() const {
121134
std::stringstream ss;
122135
ss << "tuple_desc=" << _tuple_desc->debug_string();
136+
ss << "columns: ";
137+
for (auto column : _columns) {
138+
ss << column->unique_id() << ":" << column->name() << " ";
139+
}
123140
return ss.str();
124141
}
125142

‎be/src/exec/tablet_info.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ class OlapTableSchemaParam {
5858
int64_t version() const { return _version; }
5959

6060
TupleDescriptor* tuple_desc() const { return _tuple_desc; }
61+
const std::vector<TabletColumn*>& columns() const {return _columns;}
6162
const std::vector<OlapTableIndexSchema*>& indexes() const { return _indexes; }
6263

6364
void to_protobuf(POlapTableSchemaParam* pschema) const;
@@ -81,6 +82,7 @@ class OlapTableSchemaParam {
8182
TupleDescriptor* _tuple_desc = nullptr;
8283
mutable POlapTableSchemaParam* _proto_schema = nullptr;
8384
std::vector<OlapTableIndexSchema*> _indexes;
85+
std::vector<TabletColumn*> _columns;
8486
mutable ObjectPool _obj_pool;
8587
};
8688

‎be/src/exec/tablet_sink.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ void NodeChannel::open() {
130130
request.set_index_id(_index_channel->_index_id);
131131
request.set_txn_id(_parent->_txn_id);
132132
request.set_allocated_schema(_parent->_schema->to_protobuf());
133+
LOG(INFO) << "yixiu proto: " << _parent->_schema->to_protobuf()->DebugString();
133134
for (auto& tablet : _all_tablets) {
134135
auto ptablet = request.add_tablets();
135136
ptablet->set_partition_id(tablet.partition_id);
@@ -671,6 +672,7 @@ Status OlapTableSink::init(const TDataSink& t_sink) {
671672
_tuple_desc_id = table_sink.tuple_id;
672673
_schema.reset(new OlapTableSchemaParam());
673674
RETURN_IF_ERROR(_schema->init(table_sink.schema));
675+
LOG(INFO) << "yixiu thrift:" << apache::thrift::ThriftDebugString(table_sink.schema);
674676
_partition = _pool->add(new OlapTablePartitionParam(_schema, table_sink.partition));
675677
RETURN_IF_ERROR(_partition->init());
676678
_location = _pool->add(new OlapTableLocationParam(table_sink.location));

‎be/src/olap/base_tablet.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ class BaseTablet : public std::enable_shared_from_this<BaseTablet> {
5959
bool equal(int64_t tablet_id, int32_t schema_hash);
6060

6161
// properties encapsulated in TabletSchema
62-
const TabletSchema& tablet_schema() const;
62+
virtual const TabletSchema& tablet_schema() const;
6363

6464
protected:
6565
void _gen_tablet_path();

‎be/src/olap/collect_iterator.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,9 @@ Status CollectIterator::add_child(RowsetReaderSharedPtr rs_reader) {
5555
// then merged with the base rowset.
5656
void CollectIterator::build_heap(const std::vector<RowsetReaderSharedPtr>& rs_readers) {
5757
DCHECK(rs_readers.size() == _children.size());
58-
_reverse = _reader->_tablet->tablet_schema().keys_type() == KeysType::UNIQUE_KEYS;
59-
SortType sort_type = _reader->_tablet->tablet_schema().sort_type();
60-
int sort_col_num = _reader->_tablet->tablet_schema().sort_col_num();
58+
_reverse = _reader->_tablet_schema->keys_type() == KeysType::UNIQUE_KEYS;
59+
SortType sort_type = _reader->_tablet_schema->sort_type();
60+
int sort_col_num = _reader->_tablet_schema->sort_col_num();
6161
if (_children.empty()) {
6262
_inner_iter.reset(nullptr);
6363
return;
@@ -200,7 +200,7 @@ CollectIterator::Level0Iterator::Level0Iterator(RowsetReaderSharedPtr rs_reader,
200200
CollectIterator::Level0Iterator::~Level0Iterator() = default;
201201

202202
Status CollectIterator::Level0Iterator::init() {
203-
RETURN_NOT_OK_LOG(_row_cursor.init(_reader->_tablet->tablet_schema(), _reader->_seek_columns),
203+
RETURN_NOT_OK_LOG(_row_cursor.init(*_reader->_tablet_schema, _reader->_seek_columns),
204204
"failed to init row cursor");
205205
return (this->*_refresh_current_row)();
206206
}

‎be/src/olap/compaction.cpp‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
#include "olap/compaction.h"
1919

20+
#include "delta_writer.h"
2021
#include "gutil/strings/substitute.h"
2122
#include "util/time.h"
2223
#include "util/trace.h"
@@ -88,7 +89,14 @@ Status Compaction::do_compaction_impl(int64_t permits) {
8889
LOG(INFO) << "start " << merge_type << compaction_name() << ". tablet=" << _tablet->full_name()
8990
<< ", output_version=" << _output_version << ", permits: " << permits;
9091

91-
RETURN_NOT_OK(construct_output_rowset_writer());
92+
// get cur schema if rowset schema exist, rowset schema must be newer than tablet schema
93+
auto max_version_rowset = _input_rowsets.back();
94+
const TabletSchema* cur_tablet_schema = max_version_rowset->rowset_meta()->tablet_schema();
95+
if (cur_tablet_schema == nullptr) {
96+
cur_tablet_schema = &(_tablet->tablet_schema());
97+
}
98+
99+
RETURN_NOT_OK(construct_output_rowset_writer(cur_tablet_schema));
92100
RETURN_NOT_OK(construct_input_rowset_readers());
93101
TRACE("prepare finished");
94102

@@ -98,10 +106,10 @@ Status Compaction::do_compaction_impl(int64_t permits) {
98106
Status res;
99107

100108
if (use_vectorized_compaction) {
101-
res = Merger::vmerge_rowsets(_tablet, compaction_type(), _input_rs_readers,
109+
res = Merger::vmerge_rowsets(_tablet, compaction_type(), cur_tablet_schema, _input_rs_readers,
102110
_output_rs_writer.get(), &stats);
103111
} else {
104-
res = Merger::merge_rowsets(_tablet, compaction_type(), _input_rs_readers,
112+
res = Merger::merge_rowsets(_tablet, compaction_type(), cur_tablet_schema, _input_rs_readers,
105113
_output_rs_writer.get(), &stats);
106114
}
107115

@@ -165,8 +173,8 @@ Status Compaction::do_compaction_impl(int64_t permits) {
165173
return Status::OK();
166174
}
167175

168-
Status Compaction::construct_output_rowset_writer() {
169-
return _tablet->create_rowset_writer(_output_version, VISIBLE, NONOVERLAPPING,
176+
Status Compaction::construct_output_rowset_writer(const TabletSchema* schema) {
177+
return _tablet->create_rowset_writer(_output_version, VISIBLE, NONOVERLAPPING, schema,
170178
&_output_rs_writer);
171179
}
172180

‎be/src/olap/compaction.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ class Compaction {
6666
Status modify_rowsets();
6767
void gc_output_rowset();
6868

69-
Status construct_output_rowset_writer();
69+
Status construct_output_rowset_writer(const TabletSchema* schama);
7070
Status construct_input_rowset_readers();
7171

7272
Status check_version_continuity(const std::vector<RowsetSharedPtr>& rowsets);

‎be/src/olap/delta_writer.cpp‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ DeltaWriter::DeltaWriter(WriteRequest* req, StorageEngine* storage_engine, bool
3838
_tablet(nullptr),
3939
_cur_rowset(nullptr),
4040
_rowset_writer(nullptr),
41-
_tablet_schema(nullptr),
41+
_tablet_schema(new TabletSchema),
4242
_delta_written_success(false),
4343
_storage_engine(storage_engine),
4444
_is_vec(is_vec) {}
@@ -113,9 +113,11 @@ Status DeltaWriter::init() {
113113
RETURN_NOT_OK(_storage_engine->txn_manager()->prepare_txn(_req.partition_id, _tablet,
114114
_req.txn_id, _req.load_id));
115115
}
116+
// build tablet schema in request level
117+
_build_current_tablet_schema(_req.ptable_schema_param, _tablet->tablet_schema());
116118

117119
RETURN_NOT_OK(_tablet->create_rowset_writer(_req.txn_id, _req.load_id, PREPARED, OVERLAPPING,
118-
&_rowset_writer));
120+
&_tablet->tablet_schema(), &_rowset_writer));
119121
_tablet_schema = &(_tablet->tablet_schema());
120122
_schema.reset(new Schema(*_tablet_schema));
121123
_reset_mem_table();
@@ -258,9 +260,9 @@ Status DeltaWriter::wait_flush() {
258260
}
259261

260262
void DeltaWriter::_reset_mem_table() {
261-
_mem_table.reset(new MemTable(_tablet->tablet_id(), _schema.get(), _tablet_schema, _req.slots,
262-
_req.tuple_desc, _tablet->keys_type(), _rowset_writer.get(),
263-
_mem_tracker, _is_vec));
263+
_mem_table.reset(new MemTable(_tablet->tablet_id(), _schema.get(), _tablet_schema.get(),
264+
_req.slots, _req.tuple_desc, _tablet->keys_type(),
265+
_rowset_writer.get(), _mem_tracker, _is_vec));
264266
}
265267

266268
Status DeltaWriter::close() {
@@ -353,4 +355,16 @@ int64_t DeltaWriter::partition_id() const {
353355
return _req.partition_id;
354356
}
355357

358+
void DeltaWriter::_build_current_tablet_schema(const POlapTableSchemaParam& ptable_schema_param,
359+
const TabletSchema& ori_tablet_schema) {
360+
*_tablet_schema = ori_tablet_schema;
361+
LOG(INFO) << "build current tablet schema";
362+
LOG(INFO) << "yixiu: " << ptable_schema_param.DebugString();
363+
//new tablet schame if new table
364+
if (ptable_schema_param.columns_size() != 0 && ptable_schema_param.columns(0).unique_id() > 0) {
365+
LOG(INFO) << "yixiu new table";
366+
_tablet_schema->build_current_tablet_schema(ptable_schema_param, ori_tablet_schema);
367+
}
368+
}
369+
356370
} // namespace doris

0 commit comments

Comments
 (0)