Skip to content

Commit a14c78e

Browse files
authored
[Fix](flexible partial update) Fix flexible partial update incorrect read index when resolving conflict in publish phase (apache#182)
* add reproduce problem case * fix flexible partial update publish read from old
1 parent 8cac0a6 commit a14c78e

4 files changed

Lines changed: 81 additions & 19 deletions

File tree

‎be/src/olap/tablet.cpp‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3480,12 +3480,13 @@ Status Tablet::generate_new_block_for_flexible_partial_update(
34803480
}
34813481
}
34823482

3483-
auto fill_one_cell = [&read_index_old](const TabletColumn& tablet_column, std::size_t idx,
3484-
vectorized::MutableColumnPtr& new_col,
3485-
const vectorized::IColumn& default_value_col,
3486-
const vectorized::IColumn& old_value_col,
3487-
const vectorized::IColumn& cur_col, bool skipped,
3488-
const signed char* delete_sign_column_data) {
3483+
auto fill_one_cell = [&read_index_old, &read_index_update](
3484+
const TabletColumn& tablet_column, std::size_t idx,
3485+
vectorized::MutableColumnPtr& new_col,
3486+
const vectorized::IColumn& default_value_col,
3487+
const vectorized::IColumn& old_value_col,
3488+
const vectorized::IColumn& cur_col, bool skipped,
3489+
const signed char* delete_sign_column_data) {
34893490
if (skipped) {
34903491
if (delete_sign_column_data != nullptr &&
34913492
delete_sign_column_data[read_index_old[idx]] != 0) {
@@ -3498,10 +3499,10 @@ Status Tablet::generate_new_block_for_flexible_partial_update(
34983499
new_col->insert_default();
34993500
}
35003501
} else {
3501-
new_col->insert_from(old_value_col, idx);
3502+
new_col->insert_from(old_value_col, read_index_old[idx]);
35023503
}
35033504
} else {
3504-
new_col->insert_from(cur_col, idx);
3505+
new_col->insert_from(cur_col, read_index_update[idx]);
35053506
}
35063507
};
35073508

@@ -3512,7 +3513,7 @@ Status Tablet::generate_new_block_for_flexible_partial_update(
35123513
auto col_uid = rs_column.unique_id();
35133514
for (auto idx = 0; idx < update_rows; ++idx) {
35143515
if (cid < rowset_schema->num_key_columns()) {
3515-
new_col->insert_from(cur_col, idx);
3516+
new_col->insert_from(cur_col, read_index_update[idx]);
35163517
} else {
35173518
const vectorized::IColumn& default_value_col =
35183519
*default_value_block.get_by_position(cid - rowset_schema->num_key_columns())
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
{"k":1, "v3":888, "v4":777}
2+
{"k":2, "v3":444, "v5":555}

‎regression-test/data/fault_injection_p0/flexible/test_flexible_partial_update_publish_conflict.out‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,11 @@
1717
6 1999 2777 3777 1234 \N 1,4,6
1818
7 \N \N 3777 1234 5777 1,4,6
1919

20+
-- !sql1 --
21+
1 99 99 99 1 1
22+
2 2 2 2 2 2
23+
24+
-- !sql2 --
25+
1 99 99 888 777 1
26+
2 2 2 444 2 555
27+

‎regression-test/suites/fault_injection_p0/flexible/test_flexible_partial_update_publish_conflict.groovy‎

Lines changed: 61 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,19 +40,27 @@ suite("test_flexible_partial_update_publish_conflict", "nonConcurrent") {
4040
sql """insert into ${tableName} select number, number, number, number, number, number from numbers("number" = "6"); """
4141
order_qt_sql "select k,v1,v2,v3,v4,v5,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};"
4242

43-
def enable_publish_spin_wait = {
43+
def enable_publish_spin_wait = { tokenName ->
4444
if (isCloudMode()) {
45-
GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.enable_spin_wait")
45+
GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.enable_spin_wait", [token: "${tokenName}"])
4646
} else {
47-
GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.enable_spin_wait")
47+
GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.enable_spin_wait", [token: "${tokenName}"])
4848
}
4949
}
5050

51-
def enable_block_in_publish = {
51+
def disable_publish_spin_wait = {
5252
if (isCloudMode()) {
53-
GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block")
53+
GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.enable_spin_wait")
5454
} else {
55-
GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.block")
55+
GetDebugPoint().disableDebugPointForAllBEs("EnginePublishVersionTask::execute.enable_spin_wait")
56+
}
57+
}
58+
59+
def enable_block_in_publish = { passToken ->
60+
if (isCloudMode()) {
61+
GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block", [pass_token: "${passToken}"])
62+
} else {
63+
GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.block", [pass_token: "${passToken}"])
5664
}
5765
}
5866

@@ -79,7 +87,7 @@ suite("test_flexible_partial_update_publish_conflict", "nonConcurrent") {
7987
set 'strict_mode', 'false'
8088
set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS'
8189
file "test1.json"
82-
time 20000
90+
time 40000
8391
}
8492
}
8593

@@ -93,25 +101,68 @@ suite("test_flexible_partial_update_publish_conflict", "nonConcurrent") {
93101
set 'strict_mode', 'false'
94102
set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS'
95103
file "test2.json"
96-
time 20000
104+
time 40000
97105
}
98106
}
99107

100108
Thread.sleep(500)
101109

110+
disable_publish_spin_wait()
102111
disable_block_in_publish()
103112
t1.join()
104113
t2.join()
105114

106115
order_qt_sql "select k,v1,v2,v3,v4,v5,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};"
107116

117+
118+
// ==================================================================================================
119+
sql "truncate table ${tableName}"
120+
enable_publish_spin_wait("token1")
121+
enable_block_in_publish("-1")
122+
def t3 = Thread.start {
123+
sql "set insert_visible_timeout_ms=60000;"
124+
sql "sync;"
125+
sql "insert into ${tableName} values(1,1,1,1,1,1),(2,2,2,2,2,2);"
126+
}
127+
Thread.sleep(700)
128+
def t4 = Thread.start {
129+
sql "set enable_unique_key_partial_update=true;"
130+
sql "set insert_visible_timeout_ms=60000;"
131+
sql "set enable_insert_strict=false;"
132+
sql "sync;"
133+
sql "insert into ${tableName}(k,v1,v2,v3) values(1,99,99,99);"
134+
}
135+
Thread.sleep(700)
136+
enable_publish_spin_wait("token2")
137+
def t5 = Thread.start {
138+
streamLoad {
139+
table "${tableName}"
140+
set 'format', 'json'
141+
set 'read_json_by_line', 'true'
142+
set 'strict_mode', 'false'
143+
set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS'
144+
file "test5.json"
145+
time 40000
146+
}
147+
}
148+
Thread.sleep(700)
149+
// let t3 and t4 publish
150+
enable_block_in_publish("token1")
151+
t3.join()
152+
t4.join()
153+
Thread.sleep(1000)
154+
qt_sql1 "select k,v1,v2,v3,v4,v5 from ${tableName} order by k;"
155+
156+
// let t5 publish
157+
enable_block_in_publish("token2")
158+
t5.join()
159+
qt_sql2 "select k,v1,v2,v3,v4,v5 from ${tableName} order by k;"
160+
108161
} catch(Exception e) {
109162
logger.info(e.getMessage())
110163
throw e
111164
} finally {
112165
GetDebugPoint().clearDebugPointsForAllFEs()
113166
GetDebugPoint().clearDebugPointsForAllBEs()
114167
}
115-
116-
sql "DROP TABLE IF EXISTS ${tableName};"
117168
}

0 commit comments

Comments
 (0)