Skip to content

Commit 21160d7

Browse files
authored
[fix](binlog) Decouple row binlog compaction from CCR binlog config (#68057)
Problem Summary: Row Binlog compaction scheduling and execution were incorrectly gated by the CCR binlog feature switch. This prevented Cloud and non-Cloud Row Binlog compaction when CCR was disabled. Decouple Row Binlog compaction from the CCR switch while preserving automatic-compaction, memory, and tablet-role checks. CCR download, ingest, and enablement paths remain unchanged. ### Release note Fix Row Binlog compaction scheduling and execution when CCR binlog is disabled.
1 parent e1ba108 commit 21160d7

6 files changed

Lines changed: 119 additions & 4 deletions

File tree

‎be/src/cloud/cloud_storage_engine.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -681,7 +681,7 @@ void CloudStorageEngine::_binlog_compaction_tasks_producer_callback() {
681681
int64_t interval = config::generate_compaction_tasks_interval_ms;
682682
do {
683683
int64_t cur_time = UnixMillis();
684-
if (config::enable_feature_binlog && !config::disable_auto_compaction) {
684+
if (!config::disable_auto_compaction) {
685685
Status st = _adjust_compaction_thread_num();
686686
if (!st.ok()) {
687687
break;

‎be/src/storage/olap_server.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -776,7 +776,7 @@ void StorageEngine::_binlog_compaction_tasks_producer_callback() {
776776
int64_t interval = config::generate_compaction_tasks_interval_ms;
777777
do {
778778
int64_t cur_time = UnixMillis();
779-
if (config::enable_feature_binlog && !config::disable_auto_compaction &&
779+
if (!config::disable_auto_compaction &&
780780
(!config::enable_compaction_pause_on_high_memory ||
781781
!GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE))) {
782782
_adjust_compaction_thread_num();

‎be/src/storage/tablet/tablet.cpp‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1019,8 +1019,7 @@ bool Tablet::can_do_compaction(size_t path_hash, CompactionType compaction_type)
10191019
return false;
10201020
}
10211021

1022-
if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION &&
1023-
(!config::enable_feature_binlog || !is_row_binlog_tablet())) {
1022+
if (compaction_type == CompactionType::CUMU_BINLOG_COMPACTION && !is_row_binlog_tablet()) {
10241023
return false;
10251024
}
10261025

‎be/test/cloud/cloud_compaction_test.cpp‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,56 @@ TEST_F(CloudCompactionTest, generate_cloud_binlog_compaction_tasks_updates_only_
389389
ASSERT_EQ(metrics->tablet_time_series_max_compaction_score->value(), 103);
390390
}
391391

392+
TEST_F(CloudCompactionTest, binlog_compaction_producer_ignores_ccr_switch) {
393+
const bool old_enable_feature_binlog = config::enable_feature_binlog;
394+
const bool old_disable_auto_compaction = config::disable_auto_compaction;
395+
const int32_t old_binlog_compaction_task_num_per_disk =
396+
config::binlog_compaction_task_num_per_disk;
397+
Defer restore_config {[&]() {
398+
config::enable_feature_binlog = old_enable_feature_binlog;
399+
config::disable_auto_compaction = old_disable_auto_compaction;
400+
config::binlog_compaction_task_num_per_disk = old_binlog_compaction_task_num_per_disk;
401+
}};
402+
403+
ASSERT_TRUE(ThreadPoolBuilder("BaseCompactionTaskThreadPoolTest")
404+
.set_min_threads(1)
405+
.set_max_threads(1)
406+
.build(&_engine._base_compaction_thread_pool)
407+
.ok());
408+
ASSERT_TRUE(ThreadPoolBuilder("CumuCompactionTaskThreadPoolTest")
409+
.set_min_threads(1)
410+
.set_max_threads(1)
411+
.build(&_engine._cumu_compaction_thread_pool)
412+
.ok());
413+
ASSERT_TRUE(ThreadPoolBuilder("BinlogCompactionTaskThreadPoolTest")
414+
.set_min_threads(1)
415+
.set_max_threads(1)
416+
.build(&_engine._binlog_compaction_thread_pool)
417+
.ok());
418+
419+
auto binlog_meta = std::make_shared<TabletMeta>(*_tablet_meta);
420+
binlog_meta->_tablet_id = 11004;
421+
binlog_meta->set_tablet_role(TabletRolePB::TABLET_ROLE_ROW_BINLOG);
422+
auto binlog_tablet = std::make_shared<CloudTablet>(_engine, binlog_meta);
423+
binlog_tablet->tablet_meta()->tablet_schema()->set_disable_auto_compaction(false);
424+
binlog_tablet->_approximate_cumu_num_deltas = 7;
425+
_engine.tablet_mgr().put_tablet_for_UT(binlog_tablet);
426+
427+
config::enable_feature_binlog = false;
428+
config::disable_auto_compaction = false;
429+
config::binlog_compaction_task_num_per_disk = 0;
430+
auto* metric = DorisMetrics::instance()->tablet_binlog_max_compaction_score;
431+
metric->set_value(0);
432+
_engine._stop_background_threads_latch.count_down();
433+
_engine._binlog_compaction_tasks_producer_callback();
434+
EXPECT_EQ(metric->value(), 7);
435+
436+
config::disable_auto_compaction = true;
437+
metric->set_value(11);
438+
_engine._binlog_compaction_tasks_producer_callback();
439+
EXPECT_EQ(metric->value(), 11);
440+
}
441+
392442
TEST_F(CloudCompactionTest, generate_cloud_compaction_tasks_clears_metrics_without_tablets) {
393443
auto* metrics = DorisMetrics::instance();
394444
metrics->tablet_cumulative_max_compaction_score->set_value(101);

‎be/test/storage/compaction/compaction_task_test.cpp‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,46 @@ TEST_F(CompactionTaskTest, TestSubmitCompactionTask) {
130130
EXPECT_EQ(executing_task_num, 2);
131131
}
132132

133+
TEST_F(CompactionTaskTest, BinlogCompactionProducerIgnoresCcrSwitch) {
134+
const bool old_enable_feature_binlog = config::enable_feature_binlog;
135+
const bool old_disable_auto_compaction = config::disable_auto_compaction;
136+
const bool old_enable_compaction_pause_on_high_memory =
137+
config::enable_compaction_pause_on_high_memory;
138+
Defer restore_config {[&]() {
139+
config::enable_feature_binlog = old_enable_feature_binlog;
140+
config::disable_auto_compaction = old_disable_auto_compaction;
141+
config::enable_compaction_pause_on_high_memory = old_enable_compaction_pause_on_high_memory;
142+
}};
143+
144+
auto* sp = SyncPoint::get_instance();
145+
sp->enable_processing();
146+
int generation_count = 0;
147+
sp->set_call_back("StorageEngine::_adjust_compaction_thread_num.return_void",
148+
[](auto&& args) { *try_any_cast<bool*>(args.back()) = true; });
149+
sp->set_call_back("olap_server::_generate_compaction_tasks.return_empty",
150+
[&generation_count](auto&& values) {
151+
++generation_count;
152+
auto* ret =
153+
try_any_cast_ret<std::vector<TabletCompactionContext>>(values);
154+
ret->second = true;
155+
});
156+
Defer clear_sync_points {[&]() {
157+
sp->clear_all_call_backs();
158+
sp->disable_processing();
159+
}};
160+
161+
config::enable_feature_binlog = false;
162+
config::disable_auto_compaction = false;
163+
config::enable_compaction_pause_on_high_memory = false;
164+
_storage_engine->_stop_background_threads_latch.count_down();
165+
_storage_engine->_binlog_compaction_tasks_producer_callback();
166+
EXPECT_EQ(generation_count, 1);
167+
168+
config::disable_auto_compaction = true;
169+
_storage_engine->_binlog_compaction_tasks_producer_callback();
170+
EXPECT_EQ(generation_count, 1);
171+
}
172+
133173
TEST_F(CompactionTaskTest, TestAutoSetCompactionIncreaseTaskNum) {
134174
auto st = ThreadPoolBuilder("BaseCompactionTaskThreadPool")
135175
.set_min_threads(2)

‎be/test/storage/tablet/tablet_test.cpp‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
#include <memory>
2828

29+
#include "common/config.h"
2930
#include "gtest/gtest_pred_impl.h"
3031
#include "io/fs/local_file_system.h"
3132
#include "json2pb/json_to_pb.h"
@@ -39,6 +40,7 @@
3940
#include "storage/tablet/tablet_meta.h"
4041
#include "storage/utils.h"
4142
#include "testutil/mock_rowset.h"
43+
#include "util/defer_op.h"
4244
#include "util/time.h"
4345
#include "util/uid_util.h"
4446

@@ -269,6 +271,30 @@ TEST_F(TestTablet, delete_expired_stale_rowset) {
269271
_tablet.reset();
270272
}
271273

274+
TEST_F(TestTablet, RowBinlogCompactionDoesNotDependOnCcrSwitch) {
275+
const bool old_enable_feature_binlog = config::enable_feature_binlog;
276+
Defer restore_config {[&]() { config::enable_feature_binlog = old_enable_feature_binlog; }};
277+
config::enable_feature_binlog = false;
278+
279+
ASSERT_TRUE(_data_dir->init().ok());
280+
_tablet_meta->set_tablet_role(TabletRolePB::TABLET_ROLE_ROW_BINLOG);
281+
TabletSharedPtr row_binlog_tablet(
282+
new Tablet(*k_engine, _tablet_meta, _data_dir.get(), CUMULATIVE_SIZE_BASED_POLICY));
283+
ASSERT_TRUE(row_binlog_tablet->init().ok());
284+
EXPECT_TRUE(row_binlog_tablet->can_do_compaction(_data_dir->path_hash(),
285+
CompactionType::CUMU_BINLOG_COMPACTION));
286+
EXPECT_FALSE(row_binlog_tablet->can_do_compaction(_data_dir->path_hash(),
287+
CompactionType::CUMULATIVE_COMPACTION));
288+
289+
auto data_tablet_meta = new_tablet_meta(TTabletSchema());
290+
data_tablet_meta->set_tablet_role(TabletRolePB::TABLET_ROLE_DATA);
291+
TabletSharedPtr data_tablet(
292+
new Tablet(*k_engine, data_tablet_meta, _data_dir.get(), CUMULATIVE_SIZE_BASED_POLICY));
293+
ASSERT_TRUE(data_tablet->init().ok());
294+
EXPECT_FALSE(data_tablet->can_do_compaction(_data_dir->path_hash(),
295+
CompactionType::CUMU_BINLOG_COMPACTION));
296+
}
297+
272298
TEST_F(TestTablet, pad_rowset) {
273299
std::vector<RowsetMetaSharedPtr> rs_metas;
274300
auto ptr1 = std::make_shared<RowsetMeta>();

0 commit comments

Comments
 (0)