Skip to content

Commit 1072181

Browse files
zhannngchenpull[bot]
authored andcommitted
[Enhancement](storage) add a new hidden column __DORIS_VERSION_COL__ for unique key table (#16509)
1 parent 2f3de12 commit 1072181

23 files changed

Lines changed: 169 additions & 58 deletions

File tree

‎be/src/olap/iterators.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ class StorageReadOptions {
123123
// runtime state
124124
RuntimeState* runtime_state = nullptr;
125125
RowsetId rowset_id;
126+
Version version;
126127
int32_t tablet_id = 0;
127128
};
128129

‎be/src/olap/rowset/beta_rowset_reader.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
6767
_read_options.push_down_agg_type_opt = _context->push_down_agg_type_opt;
6868
_read_options.remaining_vconjunct_root = _context->remaining_vconjunct_root;
6969
_read_options.rowset_id = _rowset->rowset_id();
70+
_read_options.version = _rowset->version();
7071
_read_options.tablet_id = _rowset->rowset_meta()->tablet_id();
7172
if (read_context->lower_bound_keys != nullptr) {
7273
for (int i = 0; i < read_context->lower_bound_keys->size(); ++i) {

‎be/src/olap/rowset/segment_v2/segment_iterator.cpp‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1394,6 +1394,33 @@ Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint32
13941394
return Status::OK();
13951395
}
13961396

1397+
void SegmentIterator::_replace_version_col(size_t num_rows) {
1398+
// Only the rowset with single version need to replace the version column.
1399+
// Doris can't determine the version before publish_version finished, so
1400+
// we can't write data to __DORIS_VERSION_COL__ in segment writer, the value
1401+
// is 0 by default.
1402+
// So we need to replace the value to real version while reading.
1403+
if (_opts.version.first != _opts.version.second) {
1404+
return;
1405+
}
1406+
auto cids = _schema.column_ids();
1407+
int32_t version_idx = _schema.version_col_idx();
1408+
auto iter = std::find(cids.begin(), cids.end(), version_idx);
1409+
if (iter == cids.end()) {
1410+
return;
1411+
}
1412+
1413+
auto column_desc = _schema.column(version_idx);
1414+
auto column = Schema::get_data_type_ptr(*column_desc)->create_column();
1415+
DCHECK(_schema.column(version_idx)->type() == FieldType::OLAP_FIELD_TYPE_BIGINT);
1416+
auto col_ptr = reinterpret_cast<vectorized::ColumnVector<vectorized::Int64>*>(column.get());
1417+
for (size_t j = 0; j < num_rows; j++) {
1418+
col_ptr->insert_value(_opts.version.second);
1419+
}
1420+
_current_return_columns[version_idx] = std::move(column);
1421+
VLOG_DEBUG << "replaced version column in segment iterator, version_col_idx:" << version_idx;
1422+
}
1423+
13971424
uint16_t SegmentIterator::_evaluate_vectorization_predicate(uint16_t* sel_rowid_idx,
13981425
uint16_t selected_size) {
13991426
SCOPED_RAW_TIMER(&_opts.stats->vec_cond_ns);
@@ -1559,6 +1586,7 @@ Status SegmentIterator::next_batch(vectorized::Block* block) {
15591586
}
15601587

15611588
if (!_is_need_vec_eval && !_is_need_short_eval) {
1589+
_replace_version_col(_current_batch_rows_read);
15621590
_output_non_pred_columns(block);
15631591
_output_index_result_column(nullptr, 0, block);
15641592
} else {
@@ -1586,6 +1614,7 @@ Status SegmentIterator::next_batch(vectorized::Block* block) {
15861614
if (!_lazy_materialization_read) {
15871615
Status ret = Status::OK();
15881616
if (selected_size > 0) {
1617+
_replace_version_col(selected_size);
15891618
ret = _output_column_by_sel_idx(block, _first_read_column_ids, sel_rowid_idx,
15901619
selected_size);
15911620
}
@@ -1606,6 +1635,8 @@ Status SegmentIterator::next_batch(vectorized::Block* block) {
16061635
sel_rowid_idx, selected_size,
16071636
&_current_return_columns));
16081637

1638+
_replace_version_col(selected_size);
1639+
16091640
// step4: output columns
16101641
// 4.1 output non-predicate column
16111642
_output_non_pred_columns(block);

‎be/src/olap/rowset/segment_v2/segment_iterator.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ class SegmentIterator : public RowwiseIterator {
187187
vectorized::MutableColumns& column_block, size_t nrows);
188188
Status _read_columns_by_index(uint32_t nrows_read_limit, uint32_t& nrows_read,
189189
bool set_block_rowid);
190+
void _replace_version_col(size_t num_rows);
190191
void _init_current_block(vectorized::Block* block,
191192
std::vector<vectorized::MutableColumnPtr>& non_pred_vector);
192193
uint16_t _evaluate_vectorization_predicate(uint16_t* sel_rowid_idx, uint16_t selected_size);

‎be/src/olap/schema.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ class Schema {
5959
if (column.name() == BeConsts::ROWID_COL) {
6060
_rowid_col_idx = cid;
6161
}
62+
if (column.name() == VERSION_COL) {
63+
_version_col_idx = cid;
64+
}
6265
columns.push_back(column);
6366
}
6467
_delete_sign_idx = tablet_schema->delete_sign_idx();
@@ -82,6 +85,9 @@ class Schema {
8285
if (columns[i].name() == BeConsts::ROWID_COL) {
8386
_rowid_col_idx = i;
8487
}
88+
if (columns[i].name() == VERSION_COL) {
89+
_version_col_idx = i;
90+
}
8591
_unique_ids[i] = columns[i].unique_id();
8692
}
8793
_init(columns, col_ids, num_key_columns);
@@ -107,6 +113,9 @@ class Schema {
107113
if (cols.at(cid)->name() == DELETE_SIGN) {
108114
_delete_sign_idx = cid;
109115
}
116+
if (cols.at(cid)->name() == VERSION_COL) {
117+
_version_col_idx = cid;
118+
}
110119
_unique_ids[cid] = cols[cid]->unique_id();
111120
}
112121

@@ -154,6 +163,7 @@ class Schema {
154163
int32_t delete_sign_idx() const { return _delete_sign_idx; }
155164
bool has_sequence_col() const { return _has_sequence_col; }
156165
int32_t rowid_col_idx() const { return _rowid_col_idx; }
166+
int32_t version_col_idx() const { return _version_col_idx; }
157167

158168
private:
159169
void _init(const std::vector<TabletColumn>& cols, const std::vector<ColumnId>& col_ids,
@@ -179,6 +189,7 @@ class Schema {
179189
int32_t _delete_sign_idx = -1;
180190
bool _has_sequence_col = false;
181191
int32_t _rowid_col_idx = -1;
192+
int32_t _version_col_idx = -1;
182193
};
183194

184195
} // namespace doris

‎be/src/olap/tablet_schema.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -592,6 +592,8 @@ void TabletSchema::append_column(TabletColumn column, bool is_dropped_column) {
592592
_delete_sign_idx = _num_columns;
593593
} else if (UNLIKELY(column.name() == SEQUENCE_COL)) {
594594
_sequence_col_idx = _num_columns;
595+
} else if (UNLIKELY(column.name() == VERSION_COL)) {
596+
_version_col_idx = _num_columns;
595597
}
596598
// The dropped column may have same name with exsiting column, so that
597599
// not add to name to index map, only for uid to index map
@@ -658,6 +660,7 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema) {
658660
_is_dynamic_schema = schema.is_dynamic_schema();
659661
_delete_sign_idx = schema.delete_sign_idx();
660662
_sequence_col_idx = schema.sequence_col_idx();
663+
_version_col_idx = schema.version_col_idx();
661664
_sort_type = schema.sort_type();
662665
_sort_col_num = schema.sort_col_num();
663666
_compression_type = schema.compression_type();
@@ -718,6 +721,8 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version
718721
_delete_sign_idx = _num_columns;
719722
} else if (UNLIKELY(column->name() == SEQUENCE_COL)) {
720723
_sequence_col_idx = _num_columns;
724+
} else if (UNLIKELY(column->name() == VERSION_COL)) {
725+
_version_col_idx = _num_columns;
721726
}
722727
_field_name_to_index[column->name()] = _num_columns;
723728
_field_id_to_index[column->unique_id()] = _num_columns;
@@ -793,6 +798,7 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const {
793798
tablet_schema_pb->set_schema_version(_schema_version);
794799
tablet_schema_pb->set_compression_type(_compression_type);
795800
tablet_schema_pb->set_is_dynamic_schema(_is_dynamic_schema);
801+
tablet_schema_pb->set_version_col_idx(_version_col_idx);
796802
}
797803

798804
size_t TabletSchema::row_size() const {

‎be/src/olap/tablet_schema.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,8 @@ class TabletSchema {
213213
void set_delete_sign_idx(int32_t delete_sign_idx) { _delete_sign_idx = delete_sign_idx; }
214214
bool has_sequence_col() const { return _sequence_col_idx != -1; }
215215
int32_t sequence_col_idx() const { return _sequence_col_idx; }
216+
void set_version_col_idx(int32_t version_col_idx) { _version_col_idx = version_col_idx; }
217+
int32_t version_col_idx() const { return _version_col_idx; }
216218
segment_v2::CompressionTypePB compression_type() const { return _compression_type; }
217219

218220
const std::vector<TabletIndex>& indexes() const { return _indexes; }
@@ -291,6 +293,7 @@ class TabletSchema {
291293
bool _is_dynamic_schema = false;
292294
int32_t _delete_sign_idx = -1;
293295
int32_t _sequence_col_idx = -1;
296+
int32_t _version_col_idx = -1;
294297
int32_t _schema_version = -1;
295298
int32_t _table_id = -1;
296299
bool _disable_auto_compaction = false;

‎be/src/olap/utils.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
namespace doris {
4444
void write_log_info(char* buf, size_t buf_len, const char* fmt, ...);
4545
static const std::string DELETE_SIGN = "__DORIS_DELETE_SIGN__";
46+
static const std::string VERSION_COL = "__DORIS_VERSION_COL__";
4647

4748
// 用来加速运算
4849
const static int32_t g_power_table[] = {1, 10, 100, 1000, 10000,

‎be/test/olap/test_data/header_without_inc_rs.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@
5858
"schema_version": 0,
5959
"disable_auto_compaction": false,
6060
"store_row_column": false,
61-
"is_dynamic_schema": false
61+
"is_dynamic_schema": false,
62+
"version_col_idx": -1
6263
},
6364
"rs_metas": [
6465
{

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1410,6 +1410,12 @@ public class Config extends ConfigBase {
14101410
@ConfField(mutable = true, masterOnly = true)
14111411
public static boolean enable_batch_delete_by_default = true;
14121412

1413+
/**
1414+
* Whether to add a version column when create unique table
1415+
*/
1416+
@ConfField(mutable = true, masterOnly = true)
1417+
public static boolean enable_hidden_version_column_by_default = true;
1418+
14131419
/**
14141420
* Used to set default db data quota bytes.
14151421
*/

0 commit comments

Comments
 (0)