|
31 | 31 | #include "common/status.h" |
32 | 32 | #include "cpp/sync_point.h" |
33 | 33 | #include "io/fs/local_file_system.h" |
| 34 | +#include "runtime/cluster_info.h" |
| 35 | +#include "runtime/exec_env.h" |
| 36 | +#include "storage/changelog.h" |
34 | 37 | #include "storage/compaction/compaction.h" |
35 | 38 | #include "storage/compaction/cumulative_compaction.h" |
36 | 39 | #include "storage/compaction/cumulative_compaction_policy.h" |
|
39 | 42 | #include "storage/storage_engine.h" |
40 | 43 | #include "storage/tablet/tablet_manager.h" |
41 | 44 | #include "util/threadpool.h" |
| 45 | +#include "util/time.h" |
42 | 46 |
|
43 | 47 | namespace doris { |
44 | 48 | using namespace config; |
@@ -246,4 +250,82 @@ TEST_F(CompactionMetricsTest, TestCompactionReadWriteThroughput) { |
246 | 250 | EXPECT_EQ(DorisMetrics::instance()->local_compaction_read_rows_total->value(), 50 * 28); |
247 | 251 | } |
248 | 252 |
|
| 253 | +TEST_F(CompactionMetricsTest, TestChangelogTtlCleanupCountedAsCompactionFinishes) { |
| 254 | + ClusterInfo cluster; |
| 255 | + auto* previous_cluster = ExecEnv::GetInstance()->cluster_info(); |
| 256 | + ExecEnv::GetInstance()->set_cluster_info(&cluster); |
| 257 | + Defer restore_cluster {[&]() { ExecEnv::GetInstance()->set_cluster_info(previous_cluster); }}; |
| 258 | + const int64_t reference_tso = 2000L << kTsoLogicalBits; |
| 259 | + const int64_t cutoff = changelog_ttl_cutoff_tso(reference_tso, 1); |
| 260 | + cluster.advance_changelog_ttl_reference_tso(reference_tso); |
| 261 | + |
| 262 | + TabletMetaSharedPtr tablet_meta; |
| 263 | + tablet_meta.reset(new TabletMeta(1, 2, 15673, 15674, 4, 5, TTabletSchema(), 6, {{7, 8}}, |
| 264 | + UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK, |
| 265 | + TCompressionType::LZ4F)); |
| 266 | + tablet_meta->set_tablet_role(TabletRolePB::TABLET_ROLE_CHANGELOG); |
| 267 | + BinlogConfig binlog_config(true, 1, 1024, 10, BinlogFormatPB::CHANGELOG, false); |
| 268 | + tablet_meta->set_binlog_config(binlog_config); |
| 269 | + TabletSharedPtr tablet(new Tablet(*(_storage_engine.get()), tablet_meta, _data_dir.get(), |
| 270 | + CUMULATIVE_SIZE_BASED_POLICY)); |
| 271 | + ASSERT_TRUE(tablet->init().ok()); |
| 272 | + auto visible = std::make_shared<VersionWithTime>(); |
| 273 | + visible->version.store(3); |
| 274 | + tablet->set_visible_version(visible); |
| 275 | + auto expired = create_rowset({2, 2}, 1, false, 1024); |
| 276 | + expired->rowset_meta()->set_commit_tso(cutoff); |
| 277 | + auto retained = create_rowset({3, 3}, 1, false, 2048); |
| 278 | + retained->rowset_meta()->set_commit_tso(cutoff + 1); |
| 279 | + |
| 280 | + // The compaction runs up to its merge, and the execution ends there with the status injected here: the merge and |
| 281 | + // what follows it need the segment files these rowsets do not have. |
| 282 | + Status execution_status; |
| 283 | + auto* sp = SyncPoint::get_instance(); |
| 284 | + sp->enable_processing(); |
| 285 | + auto return_ok = [](auto&& values) { |
| 286 | + try_any_cast<std::pair<Status, bool>*>(values.back())->second = true; |
| 287 | + }; |
| 288 | + sp->set_call_back("compaction::CompactionMixin::build_basic_info", return_ok); |
| 289 | + sp->set_call_back("compaction::CompactionMixin::execute_compact_impl", [&](auto&& values) { |
| 290 | + auto* pairs = try_any_cast<std::pair<Status, bool>*>(values.back()); |
| 291 | + pairs->first = execution_status; |
| 292 | + pairs->second = true; |
| 293 | + }); |
| 294 | + sp->set_call_back("compaction::CompactionMixin::execute_compact", return_ok); |
| 295 | + sp->set_call_back("cumulative_compaction::CumulativeCompaction::execute_compact", return_ok); |
| 296 | + Defer defer {[&]() { |
| 297 | + sp->clear_call_back("compaction::CompactionMixin::build_basic_info"); |
| 298 | + sp->clear_call_back("compaction::CompactionMixin::execute_compact_impl"); |
| 299 | + sp->clear_call_back("compaction::CompactionMixin::execute_compact"); |
| 300 | + sp->clear_call_back("cumulative_compaction::CumulativeCompaction::execute_compact"); |
| 301 | + }}; |
| 302 | + auto execute = [&](Status status) { |
| 303 | + execution_status = std::move(status); |
| 304 | + CumulativeCompaction compaction(*_storage_engine, tablet); |
| 305 | + compaction.snapshot_changelog_ttl(); |
| 306 | + compaction._input_rowsets = {expired, retained}; |
| 307 | + return compaction.execute_compact(); |
| 308 | + }; |
| 309 | + |
| 310 | + auto* metrics = DorisMetrics::instance(); |
| 311 | + const int64_t successes = metrics->changelog_ttl_cleanup_success_total->value(); |
| 312 | + const int64_t failures = metrics->changelog_ttl_cleanup_failed_total->value(); |
| 313 | + const int64_t reclaimed_bytes = metrics->changelog_ttl_reclaimed_bytes_total->value(); |
| 314 | + const int64_t started_s = UnixSeconds(); |
| 315 | + ASSERT_TRUE(execute(Status::OK()).ok()); |
| 316 | + EXPECT_EQ(metrics->changelog_ttl_cleanup_success_total->value(), successes + 1); |
| 317 | + EXPECT_EQ(metrics->changelog_ttl_cleanup_failed_total->value(), failures); |
| 318 | + // the expired rowset is dropped, the retained one is merged |
| 319 | + EXPECT_EQ(metrics->changelog_ttl_reclaimed_bytes_total->value(), reclaimed_bytes + 1024); |
| 320 | + const int64_t last_success_s = metrics->changelog_ttl_last_success_timestamp_seconds->value(); |
| 321 | + EXPECT_GE(last_success_s, started_s); |
| 322 | + EXPECT_LE(last_success_s, UnixSeconds()); |
| 323 | + |
| 324 | + EXPECT_FALSE(execute(Status::InternalError("injected execution failure")).ok()); |
| 325 | + EXPECT_EQ(metrics->changelog_ttl_cleanup_success_total->value(), successes + 1); |
| 326 | + EXPECT_EQ(metrics->changelog_ttl_cleanup_failed_total->value(), failures + 1); |
| 327 | + EXPECT_EQ(metrics->changelog_ttl_reclaimed_bytes_total->value(), reclaimed_bytes + 1024); |
| 328 | + EXPECT_EQ(metrics->changelog_ttl_last_success_timestamp_seconds->value(), last_success_s); |
| 329 | +} |
| 330 | + |
249 | 331 | } // namespace doris |
0 commit comments