Skip to content

Commit bdea689

Browse files
Lchangliangyixiuttswjtu-zhanglei
committed
[Schema Change] support fast add/drop column (apache#49)
* [feature](schema-change) support fast schema change. coauthor: yixiutt * [schema change] Using columns desc from fe to read data. coauthor: Lchangliang * [feature](schema change) schema change optimize for add/drop columns. 1.add uniqueId field for class column. 2.schema change for add/drop columns directly update schema meta Co-authored-by: yixiutt <[email protected]> Co-authored-by: SWJTU-ZhangLei <[email protected]>
1 parent de86c0d commit bdea689

60 files changed

Lines changed: 879 additions & 142 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/exec/olap_scanner.cpp‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,14 @@ Status OlapScanner::prepare(
8686
LOG(WARNING) << ss.str();
8787
return Status::InternalError(ss.str());
8888
}
89+
_tablet_schema = _tablet->tablet_schema();
90+
if (!_parent->_olap_scan_node.columns_desc.empty() &&
91+
_parent->_olap_scan_node.columns_desc[0].col_unique_id >= 0) {
92+
_tablet_schema.clear_columns();
93+
for (const auto& column_desc : _parent->_olap_scan_node.columns_desc) {
94+
_tablet_schema.append_column(TabletColumn(column_desc));
95+
}
96+
}
8997
{
9098
std::shared_lock rdlock(_tablet->get_header_lock());
9199
const RowsetSharedPtr rowset = _tablet->rowset_with_max_version();
@@ -150,6 +158,7 @@ Status OlapScanner::_init_tablet_reader_params(
150158
RETURN_IF_ERROR(_init_return_columns());
151159

152160
_tablet_reader_params.tablet = _tablet;
161+
_tablet_reader_params.tablet_schema = &_tablet_schema;
153162
_tablet_reader_params.reader_type = READER_QUERY;
154163
_tablet_reader_params.aggregation = _aggregation;
155164
_tablet_reader_params.version = Version(0, _version);
@@ -207,7 +216,7 @@ Status OlapScanner::_init_tablet_reader_params(
207216
_tablet_reader_params.return_columns.push_back(i);
208217
}
209218
for (auto index : _return_columns) {
210-
if (_tablet->tablet_schema().column(index).is_key()) {
219+
if (_tablet_schema.column(index).is_key()) {
211220
continue;
212221
} else {
213222
_tablet_reader_params.return_columns.push_back(index);
@@ -216,13 +225,12 @@ Status OlapScanner::_init_tablet_reader_params(
216225
}
217226

218227
// use _tablet_reader_params.return_columns, because reader use this to merge sort
219-
Status res =
220-
_read_row_cursor.init(_tablet->tablet_schema(), _tablet_reader_params.return_columns);
228+
Status res = _read_row_cursor.init(_tablet_schema, _tablet_reader_params.return_columns);
221229
if (!res.ok()) {
222230
LOG(WARNING) << "fail to init row cursor.res = " << res;
223231
return Status::InternalError("failed to initialize storage read row cursor");
224232
}
225-
_read_row_cursor.allocate_memory_for_string_type(_tablet->tablet_schema());
233+
_read_row_cursor.allocate_memory_for_string_type(_tablet_schema);
226234

227235
// If a agg node is this scan node direct parent
228236
// we will not call agg object finalize method in scan node,
@@ -241,30 +249,31 @@ Status OlapScanner::_init_return_columns() {
241249
if (!slot->is_materialized()) {
242250
continue;
243251
}
244-
int32_t index = _tablet->field_index(slot->col_name());
252+
int32_t index = slot->col_unique_id() >= 0 ? slot->col_unique_id()
253+
: _tablet_schema.field_index(slot->col_name());
245254
if (index < 0) {
246255
std::stringstream ss;
247256
ss << "field name is invalid. field=" << slot->col_name();
248257
LOG(WARNING) << ss.str();
249258
return Status::InternalError(ss.str());
250259
}
251260
_return_columns.push_back(index);
252-
if (slot->is_nullable() && !_tablet->tablet_schema().column(index).is_nullable())
261+
if (slot->is_nullable() && !_tablet_schema.column(index).is_nullable())
253262
_tablet_columns_convert_to_null_set.emplace(index);
254263
_query_slots.push_back(slot);
255264
}
256265

257266
// expand the sequence column
258-
if (_tablet->tablet_schema().has_sequence_col()) {
267+
if (_tablet_schema.has_sequence_col()) {
259268
bool has_replace_col = false;
260269
for (auto col : _return_columns) {
261-
if (_tablet->tablet_schema().column(col).aggregation() ==
270+
if (_tablet_schema.column(col).aggregation() ==
262271
FieldAggregationMethod::OLAP_FIELD_AGGREGATION_REPLACE) {
263272
has_replace_col = true;
264273
break;
265274
}
266275
}
267-
if (auto sequence_col_idx = _tablet->tablet_schema().sequence_col_idx();
276+
if (auto sequence_col_idx = _tablet_schema.sequence_col_idx();
268277
has_replace_col && std::find(_return_columns.begin(), _return_columns.end(),
269278
sequence_col_idx) == _return_columns.end()) {
270279
_return_columns.push_back(sequence_col_idx);

‎be/src/exec/olap_scanner.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,8 @@ class OlapScanner {
151151
MonotonicStopWatch _watcher;
152152

153153
std::shared_ptr<MemTracker> _mem_tracker;
154+
155+
TabletSchema _tablet_schema;
154156
};
155157

156158
} // namespace doris

‎be/src/exec/tablet_info.cpp‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) {
6161
}
6262
_indexes.emplace_back(index);
6363
}
64+
for (auto& pcolumn : pschema.columns()) {
65+
TabletColumn* tc = _obj_pool.add(new TabletColumn());
66+
tc->init_from_pb(pcolumn);
67+
_columns.emplace_back(tc);
68+
}
6469

6570
std::sort(_indexes.begin(), _indexes.end(),
6671
[](const OlapTableIndexSchema* lhs, const OlapTableIndexSchema* rhs) {
@@ -96,6 +101,11 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) {
96101
}
97102
_indexes.emplace_back(index);
98103
}
104+
for (auto& tcolumn : tschema.columns) {
105+
TabletColumn* tc = _obj_pool.add(new TabletColumn());
106+
tc->init_from_thrift(tcolumn);
107+
_columns.emplace_back(tc);
108+
}
99109

100110
std::sort(_indexes.begin(), _indexes.end(),
101111
[](const OlapTableIndexSchema* lhs, const OlapTableIndexSchema* rhs) {
@@ -115,11 +125,18 @@ void OlapTableSchemaParam::to_protobuf(POlapTableSchemaParam* pschema) const {
115125
for (auto index : _indexes) {
116126
index->to_protobuf(pschema->add_indexes());
117127
}
128+
for (auto column : _columns) {
129+
column->to_schema_pb(pschema->add_columns());
130+
}
118131
}
119132

120133
std::string OlapTableSchemaParam::debug_string() const {
121134
std::stringstream ss;
122135
ss << "tuple_desc=" << _tuple_desc->debug_string();
136+
ss << "columns: ";
137+
for (auto column : _columns) {
138+
ss << column->unique_id() << ":" << column->name() << " ";
139+
}
123140
return ss.str();
124141
}
125142

‎be/src/exec/tablet_info.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ class OlapTableSchemaParam {
5858
int64_t version() const { return _version; }
5959

6060
TupleDescriptor* tuple_desc() const { return _tuple_desc; }
61+
const std::vector<TabletColumn*>& columns() const {return _columns;}
6162
const std::vector<OlapTableIndexSchema*>& indexes() const { return _indexes; }
6263

6364
void to_protobuf(POlapTableSchemaParam* pschema) const;
@@ -81,6 +82,7 @@ class OlapTableSchemaParam {
8182
TupleDescriptor* _tuple_desc = nullptr;
8283
mutable POlapTableSchemaParam* _proto_schema = nullptr;
8384
std::vector<OlapTableIndexSchema*> _indexes;
85+
std::vector<TabletColumn*> _columns;
8486
mutable ObjectPool _obj_pool;
8587
};
8688

‎be/src/exec/tablet_sink.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ void NodeChannel::open() {
128128
request.set_index_id(_index_channel->_index_id);
129129
request.set_txn_id(_parent->_txn_id);
130130
request.set_allocated_schema(_parent->_schema->to_protobuf());
131+
LOG(INFO) << "yixiu proto: " << _parent->_schema->to_protobuf()->DebugString();
131132
for (auto& tablet : _all_tablets) {
132133
auto ptablet = request.add_tablets();
133134
ptablet->set_partition_id(tablet.partition_id);
@@ -685,6 +686,7 @@ Status OlapTableSink::init(const TDataSink& t_sink) {
685686
_tuple_desc_id = table_sink.tuple_id;
686687
_schema.reset(new OlapTableSchemaParam());
687688
RETURN_IF_ERROR(_schema->init(table_sink.schema));
689+
LOG(INFO) << "yixiu thrift:" << apache::thrift::ThriftDebugString(table_sink.schema);
688690
_partition = _pool->add(new OlapTablePartitionParam(_schema, table_sink.partition));
689691
RETURN_IF_ERROR(_partition->init());
690692
_location = _pool->add(new OlapTableLocationParam(table_sink.location));

‎be/src/olap/base_tablet.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ class BaseTablet : public std::enable_shared_from_this<BaseTablet> {
6060
bool equal(int64_t tablet_id, int32_t schema_hash);
6161

6262
// properties encapsulated in TabletSchema
63-
const TabletSchema& tablet_schema() const;
63+
virtual const TabletSchema& tablet_schema() const;
6464

6565
protected:
6666
void _gen_tablet_path();

‎be/src/olap/collect_iterator.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,9 @@ Status CollectIterator::add_child(RowsetReaderSharedPtr rs_reader) {
5555
// then merged with the base rowset.
5656
void CollectIterator::build_heap(const std::vector<RowsetReaderSharedPtr>& rs_readers) {
5757
DCHECK(rs_readers.size() == _children.size());
58-
_reverse = _reader->_tablet->tablet_schema().keys_type() == KeysType::UNIQUE_KEYS;
59-
SortType sort_type = _reader->_tablet->tablet_schema().sort_type();
60-
int sort_col_num = _reader->_tablet->tablet_schema().sort_col_num();
58+
_reverse = _reader->_tablet_schema->keys_type() == KeysType::UNIQUE_KEYS;
59+
SortType sort_type = _reader->_tablet_schema->sort_type();
60+
int sort_col_num = _reader->_tablet_schema->sort_col_num();
6161
if (_children.empty()) {
6262
_inner_iter.reset(nullptr);
6363
return;
@@ -200,7 +200,7 @@ CollectIterator::Level0Iterator::Level0Iterator(RowsetReaderSharedPtr rs_reader,
200200
CollectIterator::Level0Iterator::~Level0Iterator() = default;
201201

202202
Status CollectIterator::Level0Iterator::init() {
203-
RETURN_NOT_OK_LOG(_row_cursor.init(_reader->_tablet->tablet_schema(), _reader->_seek_columns),
203+
RETURN_NOT_OK_LOG(_row_cursor.init(*_reader->_tablet_schema, _reader->_seek_columns),
204204
"failed to init row cursor");
205205
return (this->*_refresh_current_row)();
206206
}

‎be/src/olap/compaction.cpp‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
#include "olap/compaction.h"
1919

20+
#include "delta_writer.h"
2021
#include "gutil/strings/substitute.h"
2122
#include "util/time.h"
2223
#include "util/trace.h"
@@ -139,7 +140,14 @@ Status Compaction::do_compaction_impl(int64_t permits) {
139140
LOG(INFO) << "start " << merge_type << compaction_name() << ". tablet=" << _tablet->full_name()
140141
<< ", output_version=" << _output_version << ", permits: " << permits;
141142

142-
RETURN_NOT_OK(construct_output_rowset_writer());
143+
// get cur schema if rowset schema exist, rowset schema must be newer than tablet schema
144+
auto max_version_rowset = _input_rowsets.back();
145+
const TabletSchema* cur_tablet_schema = max_version_rowset->rowset_meta()->tablet_schema();
146+
if (cur_tablet_schema == nullptr) {
147+
cur_tablet_schema = &(_tablet->tablet_schema());
148+
}
149+
150+
RETURN_NOT_OK(construct_output_rowset_writer(cur_tablet_schema));
143151
RETURN_NOT_OK(construct_input_rowset_readers());
144152
TRACE("prepare finished");
145153

@@ -149,10 +157,10 @@ Status Compaction::do_compaction_impl(int64_t permits) {
149157
Status res;
150158

151159
if (use_vectorized_compaction) {
152-
res = Merger::vmerge_rowsets(_tablet, compaction_type(), _input_rs_readers,
160+
res = Merger::vmerge_rowsets(_tablet, compaction_type(), cur_tablet_schema, _input_rs_readers,
153161
_output_rs_writer.get(), &stats);
154162
} else {
155-
res = Merger::merge_rowsets(_tablet, compaction_type(), _input_rs_readers,
163+
res = Merger::merge_rowsets(_tablet, compaction_type(), cur_tablet_schema, _input_rs_readers,
156164
_output_rs_writer.get(), &stats);
157165
}
158166

@@ -216,8 +224,8 @@ Status Compaction::do_compaction_impl(int64_t permits) {
216224
return Status::OK();
217225
}
218226

219-
Status Compaction::construct_output_rowset_writer() {
220-
return _tablet->create_rowset_writer(_output_version, VISIBLE, NONOVERLAPPING,
227+
Status Compaction::construct_output_rowset_writer(const TabletSchema* schema) {
228+
return _tablet->create_rowset_writer(_output_version, VISIBLE, NONOVERLAPPING, schema,
221229
&_output_rs_writer);
222230
}
223231

‎be/src/olap/compaction.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ class Compaction {
6767
Status modify_rowsets();
6868
void gc_output_rowset();
6969

70-
Status construct_output_rowset_writer();
70+
Status construct_output_rowset_writer(const TabletSchema* schama);
7171
Status construct_input_rowset_readers();
7272

7373
Status check_version_continuity(const std::vector<RowsetSharedPtr>& rowsets);

‎be/src/olap/delta_writer.cpp‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ DeltaWriter::DeltaWriter(WriteRequest* req, StorageEngine* storage_engine, bool
4040
_tablet(nullptr),
4141
_cur_rowset(nullptr),
4242
_rowset_writer(nullptr),
43-
_tablet_schema(nullptr),
43+
_tablet_schema(new TabletSchema),
4444
_delta_written_success(false),
4545
_storage_engine(storage_engine),
4646
_is_vec(is_vec) {}
@@ -119,9 +119,11 @@ Status DeltaWriter::init() {
119119
RETURN_NOT_OK(_storage_engine->txn_manager()->prepare_txn(_req.partition_id, _tablet,
120120
_req.txn_id, _req.load_id));
121121
}
122+
// build tablet schema in request level
123+
_build_current_tablet_schema(_req.ptable_schema_param, _tablet->tablet_schema());
122124

123125
RETURN_NOT_OK(_tablet->create_rowset_writer(_req.txn_id, _req.load_id, PREPARED, OVERLAPPING,
124-
&_rowset_writer));
126+
&_tablet->tablet_schema(), &_rowset_writer));
125127
_tablet_schema = &(_tablet->tablet_schema());
126128
_schema.reset(new Schema(*_tablet_schema));
127129
_reset_mem_table();
@@ -264,9 +266,9 @@ Status DeltaWriter::wait_flush() {
264266
}
265267

266268
void DeltaWriter::_reset_mem_table() {
267-
_mem_table.reset(new MemTable(_tablet->tablet_id(), _schema.get(), _tablet_schema, _req.slots,
268-
_req.tuple_desc, _tablet->keys_type(), _rowset_writer.get(),
269-
_mem_tracker, _is_vec));
269+
_mem_table.reset(new MemTable(_tablet->tablet_id(), _schema.get(), _tablet_schema.get(),
270+
_req.slots, _req.tuple_desc, _tablet->keys_type(),
271+
_rowset_writer.get(), _mem_tracker, _is_vec));
270272
}
271273

272274
Status DeltaWriter::close() {
@@ -359,4 +361,16 @@ int64_t DeltaWriter::partition_id() const {
359361
return _req.partition_id;
360362
}
361363

364+
void DeltaWriter::_build_current_tablet_schema(const POlapTableSchemaParam& ptable_schema_param,
365+
const TabletSchema& ori_tablet_schema) {
366+
*_tablet_schema = ori_tablet_schema;
367+
LOG(INFO) << "build current tablet schema";
368+
LOG(INFO) << "yixiu: " << ptable_schema_param.DebugString();
369+
//new tablet schame if new table
370+
if (ptable_schema_param.columns_size() != 0 && ptable_schema_param.columns(0).unique_id() > 0) {
371+
LOG(INFO) << "yixiu new table";
372+
_tablet_schema->build_current_tablet_schema(ptable_schema_param, ori_tablet_schema);
373+
}
374+
}
375+
362376
} // namespace doris

0 commit comments

Comments
 (0)