Skip to content

Commit 2f4b6ea

Browse files
committed
choose newest schema by schema version when compaction (apache#182)
1 parent 44f25f1 commit 2f4b6ea

9 files changed

Lines changed: 136 additions & 26 deletions

File tree

‎be/src/olap/compaction.cpp‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,8 +140,13 @@ Status Compaction::do_compaction_impl(int64_t permits) {
140140
<< ", output_version=" << _output_version << ", permits: " << permits;
141141

142142
// get cur schema if rowset schema exist, rowset schema must be newer than tablet schema
143-
auto max_version_rowset = _input_rowsets.back();
144-
const TabletSchema* cur_tablet_schema = max_version_rowset->rowset_meta()->tablet_schema();
143+
auto max_version_rowset =
144+
std::max_element(_input_rowsets.begin(), _input_rowsets.end(),
145+
[](const RowsetSharedPtr& a, const RowsetSharedPtr& b) {
146+
return a->rowset_meta()->tablet_schema()->schema_version() <
147+
b->rowset_meta()->tablet_schema()->schema_version();
148+
});
149+
const TabletSchema* cur_tablet_schema = (*max_version_rowset)->rowset_meta()->tablet_schema();
145150
if (cur_tablet_schema == nullptr) {
146151
cur_tablet_schema = &(_tablet->tablet_schema());
147152
}

‎be/src/olap/tablet_schema.cpp‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -499,6 +499,7 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema) {
499499
_sort_type = schema.sort_type();
500500
_sort_col_num = schema.sort_col_num();
501501
_compression_type = schema.compression_type();
502+
_schema_version = schema.schema_version();
502503
}
503504

504505
void TabletSchema::build_current_tablet_schema(int64_t index_id,
@@ -569,30 +570,32 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id,
569570
_has_bf_fpp = false;
570571
_bf_fpp = BLOOM_FILTER_DEFAULT_FPP;
571572
}
573+
_schema_version = ptable_schema_param.version();
572574
}
573575

574-
void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_meta_pb) const {
575-
tablet_meta_pb->set_keys_type(_keys_type);
576+
void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const {
577+
tablet_schema_pb->set_keys_type(_keys_type);
576578
for (auto& col : _cols) {
577-
ColumnPB* column = tablet_meta_pb->add_column();
579+
ColumnPB* column = tablet_schema_pb->add_column();
578580
col.to_schema_pb(column);
579581
}
580-
tablet_meta_pb->set_num_short_key_columns(_num_short_key_columns);
581-
tablet_meta_pb->set_num_rows_per_row_block(_num_rows_per_row_block);
582-
tablet_meta_pb->set_compress_kind(_compress_kind);
582+
tablet_schema_pb->set_num_short_key_columns(_num_short_key_columns);
583+
tablet_schema_pb->set_num_rows_per_row_block(_num_rows_per_row_block);
584+
tablet_schema_pb->set_compress_kind(_compress_kind);
583585
if (_has_bf_fpp) {
584-
tablet_meta_pb->set_bf_fpp(_bf_fpp);
586+
tablet_schema_pb->set_bf_fpp(_bf_fpp);
585587
}
586-
tablet_meta_pb->set_next_column_unique_id(_next_column_unique_id);
587-
tablet_meta_pb->set_is_in_memory(_is_in_memory);
588-
tablet_meta_pb->set_delete_sign_idx(_delete_sign_idx);
589-
tablet_meta_pb->set_sequence_col_idx(_sequence_col_idx);
590-
tablet_meta_pb->set_sort_type(_sort_type);
591-
tablet_meta_pb->set_sort_col_num(_sort_col_num);
592-
tablet_meta_pb->set_compression_type(_compression_type);
588+
tablet_schema_pb->set_next_column_unique_id(_next_column_unique_id);
589+
tablet_schema_pb->set_is_in_memory(_is_in_memory);
590+
tablet_schema_pb->set_delete_sign_idx(_delete_sign_idx);
591+
tablet_schema_pb->set_sequence_col_idx(_sequence_col_idx);
592+
tablet_schema_pb->set_sort_type(_sort_type);
593+
tablet_schema_pb->set_sort_col_num(_sort_col_num);
594+
tablet_schema_pb->set_schema_version(_schema_version);
595+
tablet_schema_pb->set_compression_type(_compression_type);
593596
}
594597

595-
uint32_t TabletSchema::mem_size() const {
598+
uint32_t Tatablet_schema_pbbletSchema::mem_size() const {
596599
auto size = sizeof(TabletSchema);
597600
for (auto& col : _cols) {
598601
size += col.mem_size();

‎be/src/olap/tablet_schema.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ class TabletSchema {
162162
int32_t sequence_col_idx() const { return _sequence_col_idx; }
163163
segment_v2::CompressionTypePB compression_type() const { return _compression_type; }
164164

165+
int32_t schema_version() const { return _schema_version; }
165166
void clear_columns();
166167
vectorized::Block create_block(
167168
const std::vector<uint32_t>& return_columns,
@@ -200,6 +201,7 @@ class TabletSchema {
200201
bool _is_in_memory = false;
201202
int32_t _delete_sign_idx = -1;
202203
int32_t _sequence_col_idx = -1;
204+
int32_t _schema_version = -1;
203205
};
204206

205207
bool operator==(const TabletSchema& a, const TabletSchema& b);

‎fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ private TOlapTableSchemaParam createSchema(long dbId, OlapTable table) {
178178
TOlapTableSchemaParam schemaParam = new TOlapTableSchemaParam();
179179
schemaParam.setDbId(dbId);
180180
schemaParam.setTableId(table.getId());
181-
schemaParam.setVersion(0);
181+
schemaParam.setVersion(table.getIndexMetaByIndexId(table.getBaseIndexId()).getSchemaVersion());
182182

183183
schemaParam.tuple_desc = tupleDescriptor.toThrift();
184184
for (SlotDescriptor slotDesc : tupleDescriptor.getSlots()) {

‎gensrc/proto/olap_file.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,7 @@ message TabletSchemaPB {
200200
optional SortType sort_type = 11;
201201
optional int32 sort_col_num = 12;
202202
optional segment_v2.CompressionTypePB compression_type = 13 [default=LZ4F];
203+
optional int32 schema_version = 14;
203204
}
204205

205206
enum TabletStatePB {
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !compaction --
3+
100
4+
5+
-- !compaction --
6+
100
7+

‎regression-test/data/schema_change/test_update_schema_change.out‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@
2626
1
2727

2828
-- !test_update_schema_change_9 --
29-
2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 1
3029
1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20 2
30+
2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 1
3131

3232
-- !test_update_schema_change_10 --
3333
1
@@ -40,25 +40,25 @@
4040

4141
-- !test_update_schema_change_13 --
4242
5 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 20
43-
2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 1
4443
3 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20 20
44+
2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21 1
4545
1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20 20
4646

4747
-- !test_update_schema_change_14 --
4848
0
4949

5050
-- !test_update_schema_change_15 --
5151
5 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21
52-
2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21
5352
3 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20
53+
2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21
5454
1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20
5555

5656
-- !test_update_schema_change_16 --
5757
1
5858

5959
-- !test_update_schema_change_17 --
60-
2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21
61-
3 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20
6260
5 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 20 31 21
61+
3 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20
62+
2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 31 21
6363
1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 30 20
6464

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import org.codehaus.groovy.runtime.IOGroovyMethods
2+
3+
suite ("test_compaction_schema_change") {
4+
def tableName = "lineorder"
5+
def columns = """lo_orderkey,lo_linenumber,lo_custkey,lo_partkey,lo_suppkey,lo_orderdate,lo_orderpriority,
6+
lo_shippriority,lo_quantity,lo_extendedprice,lo_ordtotalprice,lo_discount,
7+
lo_revenue,lo_supplycost,lo_tax,lo_commitdate,lo_shipmode,lo_dummy"""
8+
9+
sql """ DROP TABLE IF EXISTS ${tableName} """
10+
11+
sql """ CREATE TABLE IF NOT EXISTS ${tableName} (
12+
`lo_orderkey` bigint(20) NOT NULL COMMENT "",
13+
`lo_linenumber` bigint(20) NOT NULL COMMENT "",
14+
`lo_custkey` int(11) NOT NULL COMMENT "",
15+
`lo_partkey` int(11) NOT NULL COMMENT "",
16+
`lo_suppkey` int(11) NOT NULL COMMENT "",
17+
`lo_orderdate` int(11) NOT NULL COMMENT "",
18+
`lo_orderpriority` varchar(16) NOT NULL COMMENT "",
19+
`lo_shippriority` int(11) NOT NULL COMMENT "",
20+
`lo_quantity` bigint(20) NOT NULL COMMENT "",
21+
`lo_extendedprice` bigint(20) NOT NULL COMMENT "",
22+
`lo_ordtotalprice` bigint(20) NOT NULL COMMENT "",
23+
`lo_discount` bigint(20) NOT NULL COMMENT "",
24+
`lo_revenue` bigint(20) NOT NULL COMMENT "",
25+
`lo_supplycost` bigint(20) NOT NULL COMMENT "",
26+
`lo_tax` bigint(20) NOT NULL COMMENT "",
27+
`lo_commitdate` bigint(20) NOT NULL COMMENT "",
28+
`lo_shipmode` varchar(11) NOT NULL COMMENT ""
29+
)
30+
PARTITION BY RANGE(`lo_orderdate`)
31+
(PARTITION p1992 VALUES [("-2147483648"), ("19930101")),
32+
PARTITION p1993 VALUES [("19930101"), ("19940101")),
33+
PARTITION p1994 VALUES [("19940101"), ("19950101")),
34+
PARTITION p1995 VALUES [("19950101"), ("19960101")),
35+
PARTITION p1996 VALUES [("19960101"), ("19970101")),
36+
PARTITION p1997 VALUES [("19970101"), ("19980101")),
37+
PARTITION p1998 VALUES [("19980101"), ("19990101")))
38+
DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 1
39+
PROPERTIES (
40+
"replication_num" = "1"
41+
);"""
42+
43+
Thread thread = new Thread(() -> {
44+
streamLoad {
45+
// a default db 'regression_test' is specified in
46+
// ${DORIS_HOME}/conf/regression-conf.groovy
47+
table tableName
48+
49+
// default label is UUID:
50+
// set 'label' UUID.randomUUID().toString()
51+
52+
// default column_separator is specify in doris fe config, usually is '\t'.
53+
// this line change to ','
54+
set 'column_separator', '|'
55+
set 'compress_type', 'GZ'
56+
set 'columns', columns
57+
// relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
58+
// also, you can stream load a http stream, e.g. http://xxx/some.csv
59+
file """${getS3Url() + '/regression/ssb/sf1/' + tableName}.tbl.gz"""
60+
61+
time 10000 // limit inflight 10s
62+
63+
// stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
64+
65+
// if declared a check callback, the default check condition will ignore.
66+
// So you must check all condition
67+
check { result, exception, startTime, endTime ->
68+
if (exception != null) {
69+
throw exception
70+
}
71+
log.info("Stream load result: ${result}".toString())
72+
def json = parseJson(result)
73+
assertEquals("success", json.Status.toLowerCase())
74+
assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
75+
assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
76+
}
77+
}
78+
})
79+
80+
Thread.sleep(60000)
81+
82+
sql """ alter table ${tableName} add column test_column int default '1'; """
83+
sql """ insert into ${tableName} values (-1, 1, 1, 1, 1, 1, '1', 1, 1, 1, 1, 1, 1, 1, 1, 1, '1', 100); """
84+
qt_compaction """ select test_column from ${tableName} where lo_orderkey = -1; """
85+
86+
thread.join()
87+
88+
// wait to auto compaction
89+
Thread.sleep(60000)
90+
91+
qt_compaction """ select test_column from ${tableName} where lo_orderkey = -1; """
92+
}

‎regression-test/suites/schema_change/test_update_schema_change.sql‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,12 @@ INSERT INTO schema_change_update_regression_test VALUES
3939

4040
UPDATE schema_change_update_regression_test set new_column = 20 where new_column = 2;
4141

42-
SELECT * FROM schema_change_update_regression_test;
42+
SELECT * FROM schema_change_update_regression_test order by user_id DESC, last_visit_date;
4343

4444
ALTER TABLE schema_change_update_regression_test DROP COLUMN new_column;
4545

46-
SELECT * FROM schema_change_update_regression_test;
46+
SELECT * FROM schema_change_update_regression_test order by user_id DESC, last_visit_date;
4747

4848
UPDATE schema_change_update_regression_test set cost = 20 where user_id = 5;
4949

50-
SELECT * FROM schema_change_update_regression_test;
50+
SELECT * FROM schema_change_update_regression_test order by user_id DESC, last_visit_date;

0 commit comments

Comments
 (0)