Skip to content

Commit 8c8f322

Browse files
authored
[fix](partial update) Fix missing rowsets during doing alignment when flushing memtable due to compaction #28062 (#28468)
1 parent 7a061c5 commit 8c8f322

6 files changed

Lines changed: 155 additions & 4 deletions

File tree

‎be/src/olap/memtable.cpp‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
#include "runtime/load_channel_mgr.h"
4545
#include "runtime/thread_context.h"
4646
#include "tablet_meta.h"
47+
#include "util/debug_points.h"
4748
#include "util/doris_metrics.h"
4849
#include "util/runtime_profile.h"
4950
#include "util/stopwatch.hpp"
@@ -474,6 +475,31 @@ Status MemTable::_generate_delete_bitmap(int32_t segment_id) {
474475
{
475476
std::shared_lock meta_rlock(_tablet->get_header_lock());
476477
specified_rowsets = _tablet->get_rowset_by_ids(&_mow_context->rowset_ids);
478+
DBUG_EXECUTE_IF("_append_block_with_partial_content.clear_specified_rowsets",
479+
{ specified_rowsets.clear(); });
480+
if (specified_rowsets.size() != _mow_context->rowset_ids.size()) {
481+
// `get_rowset_by_ids` may fail to find some of the rowsets we request if cumulative compaction delete
482+
// rowsets from `_rs_version_map`(see `Tablet::modify_rowsets` for detials) before we get here.
483+
// Becasue we havn't begun calculation for merge-on-write table, we can safely reset the `_mow_context->rowset_ids`
484+
// to the latest value and re-request the correspoding rowsets.
485+
LOG(INFO) << fmt::format(
486+
"[Memtable Flush] some rowsets have been deleted due to "
487+
"compaction(specified_rowsets.size()={}, but rowset_ids.size()={}), reset "
488+
"rowset_ids to the latest value. tablet_id: {}, cur max_version: {}, "
489+
"transaction_id: {}",
490+
specified_rowsets.size(), _mow_context->rowset_ids.size(), _tablet->tablet_id(),
491+
_mow_context->max_version, _mow_context->txn_id);
492+
Status st {Status::OK()};
493+
_mow_context->update_rowset_ids_with_lock([&]() {
494+
_mow_context->rowset_ids.clear();
495+
st = _tablet->all_rs_id(_mow_context->max_version, &_mow_context->rowset_ids);
496+
});
497+
if (!st.ok()) {
498+
return st;
499+
}
500+
specified_rowsets = _tablet->get_rowset_by_ids(&_mow_context->rowset_ids);
501+
DCHECK(specified_rowsets.size() == _mow_context->rowset_ids.size());
502+
}
477503
}
478504
OlapStopWatch watch;
479505
RETURN_IF_ERROR(_tablet->calc_delete_bitmap(rowset, segments, specified_rowsets,

‎be/src/olap/olap_common.h‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -463,13 +463,18 @@ using RowsetIdUnorderedSet = std::unordered_set<RowsetId, HashOfRowsetId>;
463463
class DeleteBitmap;
464464
// merge on write context
465465
struct MowContext {
466-
MowContext(int64_t version, int64_t txnid, const RowsetIdUnorderedSet& ids,
466+
MowContext(int64_t version, int64_t txnid, RowsetIdUnorderedSet& ids,
467467
std::shared_ptr<DeleteBitmap> db)
468468
: max_version(version), txn_id(txnid), rowset_ids(ids), delete_bitmap(db) {}
469+
void update_rowset_ids_with_lock(std::function<void()> callback) {
470+
std::lock_guard<std::mutex> lock(m);
471+
callback();
472+
}
469473
int64_t max_version;
470474
int64_t txn_id;
471-
const RowsetIdUnorderedSet& rowset_ids;
475+
RowsetIdUnorderedSet& rowset_ids;
472476
std::shared_ptr<DeleteBitmap> delete_bitmap;
477+
std::mutex m; // protection for updating rowset_ids only
473478
};
474479

475480
// used in mow partial update

‎be/src/olap/rowset/segment_v2/segment_writer.cpp‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
#include "service/point_query_executor.h"
5252
#include "util/coding.h"
5353
#include "util/crc32c.h"
54+
#include "util/debug_points.h"
5455
#include "util/faststring.h"
5556
#include "util/key_util.h"
5657
#include "vec/columns/column_nullable.h"
@@ -401,6 +402,31 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block*
401402
{
402403
std::shared_lock rlock(_tablet->get_header_lock());
403404
specified_rowsets = _tablet->get_rowset_by_ids(&_mow_context->rowset_ids);
405+
DBUG_EXECUTE_IF("_append_block_with_partial_content.clear_specified_rowsets",
406+
{ specified_rowsets.clear(); });
407+
if (specified_rowsets.size() != _mow_context->rowset_ids.size()) {
408+
// `get_rowset_by_ids` may fail to find some of the rowsets we request if cumulative compaction delete
409+
// rowsets from `_rs_version_map`(see `Tablet::modify_rowsets` for detials) before we get here.
410+
// Becasue we havn't begun calculation for merge-on-write table, we can safely reset the `_mow_context->rowset_ids`
411+
// to the latest value and re-request the correspoding rowsets.
412+
LOG(INFO) << fmt::format(
413+
"[Memtable Flush] some rowsets have been deleted due to "
414+
"compaction(specified_rowsets.size()={}, but rowset_ids.size()={}), reset "
415+
"rowset_ids to the latest value. tablet_id: {}, cur max_version: {}, "
416+
"transaction_id: {}",
417+
specified_rowsets.size(), _mow_context->rowset_ids.size(), _tablet->tablet_id(),
418+
_mow_context->max_version, _mow_context->txn_id);
419+
Status st {Status::OK()};
420+
_mow_context->update_rowset_ids_with_lock([&]() {
421+
_mow_context->rowset_ids.clear();
422+
st = _tablet->all_rs_id(_mow_context->max_version, &_mow_context->rowset_ids);
423+
});
424+
if (!st.ok()) {
425+
return st;
426+
}
427+
specified_rowsets = _tablet->get_rowset_by_ids(&_mow_context->rowset_ids);
428+
DCHECK(specified_rowsets.size() == _mow_context->rowset_ids.size());
429+
}
404430
}
405431
std::vector<std::unique_ptr<SegmentCacheHandle>> segment_caches(specified_rowsets.size());
406432
// locate rows in base data
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !1 --
3+
1 doris 1000 123 1
4+
2 doris2 2000 223 1
5+
6+
-- !2 --
7+
1 doris 200 123 1
8+
2 doris2 400 223 1
9+
4 yixiu 400 \N 4321
10+
11+
-- !3 --
12+
1 doris 1000 123 1
13+
2 doris2 2000 223 1
14+
15+
-- !4 --
16+
1 doris333 6666 555 4
17+
2 doris666 9999 888 7
18+
3 doris222 1111 987 567
19+
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
import org.codehaus.groovy.runtime.IOGroovyMethods
19+
import org.apache.doris.regression.util.Http
20+
21+
suite("test_unique_key_mow_rowsets_deleted", "nonConcurrent"){
22+
23+
def tableName = "test_unique_key_mow_rowsets_deleted1"
24+
25+
// 1. requested rowsets have been deleted during partial update
26+
sql """ DROP TABLE IF EXISTS ${tableName} """
27+
sql """ CREATE TABLE ${tableName} (
28+
`id` int(11) NOT NULL COMMENT "用户 ID",
29+
`name` varchar(65533) NOT NULL DEFAULT "yixiu" COMMENT "用户姓名",
30+
`score` int(11) NOT NULL COMMENT "用户得分",
31+
`test` int(11) NULL COMMENT "null test",
32+
`dft` int(11) DEFAULT "4321")
33+
UNIQUE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1
34+
PROPERTIES("replication_num" = "1", "enable_unique_key_merge_on_write" = "true"); """
35+
36+
sql """insert into ${tableName} values(2, "doris2", 2000, 223, 1),(1, "doris", 1000, 123, 1)"""
37+
qt_1 """ select * from ${tableName} order by id; """
38+
try {
39+
GetDebugPoint().enableDebugPointForAllBEs("_append_block_with_partial_content.clear_specified_rowsets")
40+
sql "set enable_unique_key_partial_update=true;"
41+
sql "set enable_insert_strict = false;"
42+
sql "sync;"
43+
sql """insert into ${tableName}(id,score) values(2,400),(1,200),(4,400)"""
44+
qt_2 """ select * from ${tableName} order by id; """
45+
sql "set enable_unique_key_partial_update=false;"
46+
sql "set enable_insert_strict = true;"
47+
sql "sync;"
48+
} finally {
49+
GetDebugPoint().disableDebugPointForAllBEs("_append_block_with_partial_content.clear_specified_rowsets")
50+
}
51+
sql "DROP TABLE IF EXISTS ${tableName};"
52+
53+
54+
// 2. requested rowsets have been deleted during row update
55+
tableName = "test_unique_key_mow_rowsets_deleted2"
56+
sql """ DROP TABLE IF EXISTS ${tableName} """
57+
sql """ CREATE TABLE ${tableName} (
58+
`id` int(11) NOT NULL COMMENT "用户 ID",
59+
`name` varchar(65533) NOT NULL DEFAULT "yixiu" COMMENT "用户姓名",
60+
`score` int(11) NOT NULL COMMENT "用户得分",
61+
`test` int(11) NULL COMMENT "null test",
62+
`dft` int(11) DEFAULT "4321")
63+
UNIQUE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1
64+
PROPERTIES("replication_num" = "1", "enable_unique_key_merge_on_write" = "true"); """
65+
sql """insert into ${tableName} values(2, "doris2", 2000, 223, 1),(1, "doris", 1000, 123, 1)"""
66+
qt_3 """ select * from ${tableName} order by id; """
67+
try {
68+
GetDebugPoint().enableDebugPointForAllBEs("BetaRowsetWriter::_generate_delete_bitmap.clear_specified_rowsets")
69+
sql """insert into ${tableName} values(2, "doris666", 9999, 888, 7),(1, "doris333", 6666, 555, 4), (3, "doris222", 1111, 987, 567);"""
70+
qt_4 """ select * from ${tableName} order by id; """
71+
} finally {
72+
GetDebugPoint().disableDebugPointForAllBEs("BetaRowsetWriter::_generate_delete_bitmap.clear_specified_rowsets")
73+
}
74+
sql "DROP TABLE IF EXISTS ${tableName};"
75+
}

‎regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_publish.groovy‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ suite("test_primary_key_partial_update_publish", "p0") {
3737

3838
file '10000.csv'
3939
time 10000 // limit inflight 10s
40-
}
40+
}
4141
streamLoad {
4242
table "${tableName}"
4343

@@ -68,5 +68,5 @@ suite("test_primary_key_partial_update_publish", "p0") {
6868
"""
6969

7070
// drop drop
71-
// sql """ DROP TABLE IF EXISTS ${tableName} """
71+
sql """ DROP TABLE IF EXISTS ${tableName} """
7272
}

0 commit comments

Comments
 (0)