Skip to content

Commit 66aeac9

Browse files
neethuhaneeshaneethuhaneesha
committed
Rocksdb stats level knob.
1 parent ae4d66c commit 66aeac9

File tree

3 files changed

+28
-18
lines changed

3 files changed

+28
-18
lines changed

‎fdbclient/ServerKnobs.cpp‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -407,8 +407,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
407407
init( ROCKSDB_FETCH_QUEUE_SOFT_MAX, 50 );
408408
init( ROCKSDB_HISTOGRAMS_SAMPLE_RATE, 0.001 ); if( randomize && BUGGIFY ) ROCKSDB_HISTOGRAMS_SAMPLE_RATE = 0;
409409
init( ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME, 30.0 ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME = 0.1;
410-
init( ROCKSDB_READ_RANGE_REUSE_ITERATORS, true ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_REUSE_ITERATORS = deterministicRandom()->coinflip() ? true : false;
411-
init( ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS, false ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS = deterministicRandom()->coinflip() ? true : false;
410+
init( ROCKSDB_READ_RANGE_REUSE_ITERATORS, true ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_REUSE_ITERATORS = deterministicRandom()->coinflip();
411+
init( ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS, false ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS = deterministicRandom()->coinflip();
412412
init( ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT, 200 );
413413
// Set to 0 to disable rocksdb write rate limiting. Rate limiter unit: bytes per second.
414414
init( ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC, 0 );
@@ -417,7 +417,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
417417
init( DEFAULT_FDB_ROCKSDB_COLUMN_FAMILY, "fdb");
418418
init( ROCKSDB_DISABLE_AUTO_COMPACTIONS, false ); // RocksDB default
419419

420-
init( ROCKSDB_PERFCONTEXT_ENABLE, false ); if( randomize && BUGGIFY ) ROCKSDB_PERFCONTEXT_ENABLE = deterministicRandom()->coinflip() ? false : true;
420+
init( ROCKSDB_PERFCONTEXT_ENABLE, false ); if( randomize && BUGGIFY ) ROCKSDB_PERFCONTEXT_ENABLE = deterministicRandom()->coinflip();
421421
init( ROCKSDB_PERFCONTEXT_SAMPLE_RATE, 0.0001 );
422422
init( ROCKSDB_METRICS_SAMPLE_INTERVAL, 0.0);
423423
init( ROCKSDB_MAX_SUBCOMPACTIONS, 2 );
@@ -429,9 +429,11 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
429429
init( ROCKSDB_DISABLE_WAL_EXPERIMENTAL, false );
430430
// If ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE is enabled, disable ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS knob.
431431
// These knobs have contrary functionality.
432-
init( ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE, false ); if( randomize && BUGGIFY ) ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE = deterministicRandom()->coinflip() ? false : true;
432+
init( ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE, false ); if( randomize && BUGGIFY ) ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE = deterministicRandom()->coinflip();
433433
init( ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT, 200000 ); // 200KB
434-
init( ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS, true ); if( randomize && BUGGIFY ) ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS = deterministicRandom()->coinflip() ? false : true;
434+
init( ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS, true ); if( randomize && BUGGIFY ) ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS = deterministicRandom()->coinflip();
435+
// ROCKSDB_STATS_LEVEL=1 indicates rocksdb::StatsLevel::kExceptHistogramOrTimers
436+
init( ROCKSDB_STATS_LEVEL, 1 ); if( randomize && BUGGIFY ) ROCKSDB_STATS_LEVEL = deterministicRandom()->randomInt(0, 6);
435437
// Can commit will delay ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD seconds for
436438
// ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD times, if rocksdb overloaded.
437439
// Set ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD to 0, to disable

‎fdbclient/include/fdbclient/ServerKnobs.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -354,6 +354,7 @@ class ServerKnobs : public KnobsImpl<ServerKnobs> {
354354
bool ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE;
355355
int64_t ROCKSDB_SINGLEKEY_DELETES_BYTES_LIMIT;
356356
bool ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS;
357+
int ROCKSDB_STATS_LEVEL;
357358
int64_t ROCKSDB_COMPACTION_READAHEAD_SIZE;
358359
int64_t ROCKSDB_BLOCK_SIZE;
359360
bool ENABLE_SHARDED_ROCKSDB;

‎fdbserver/KeyValueStoreRocksDB.actor.cpp‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,7 @@ rocksdb::DBOptions SharedRocksDBState::initialDbOptions() {
200200
}
201201

202202
options.statistics = rocksdb::CreateDBStatistics();
203-
options.statistics->set_stats_level(rocksdb::kExceptHistogramOrTimers);
203+
options.statistics->set_stats_level(rocksdb::StatsLevel(SERVER_KNOBS->ROCKSDB_STATS_LEVEL));
204204

205205
options.db_log_dir = SERVER_KNOBS->LOG_DIRECTORY;
206206

@@ -893,14 +893,19 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
893893
{ "CountIterSkippedKeys", rocksdb::NUMBER_ITER_SKIP, 0 },
894894
};
895895

896+
// To control the rocksdb::StatsLevel, use ROCKSDB_STATS_LEVEL knob.
896897
state std::vector<std::pair<const char*, uint32_t>> histogramStats = {
897-
{ "CompactionTime", rocksdb::COMPACTION_TIME },
898-
{ "CompactionCPUTime", rocksdb::COMPACTION_CPU_TIME },
899-
{ "CompressionTimeNanos", rocksdb::COMPRESSION_TIMES_NANOS },
900-
{ "DecompressionTimeNanos", rocksdb::DECOMPRESSION_TIMES_NANOS },
901-
{ "HardRateLimitDelayCount", rocksdb::HARD_RATE_LIMIT_DELAY_COUNT },
902-
{ "SoftRateLimitDelayCount", rocksdb::SOFT_RATE_LIMIT_DELAY_COUNT },
903-
{ "WriteStall", rocksdb::WRITE_STALL },
898+
{ "CompactionTime", rocksdb::COMPACTION_TIME }, // enabled if rocksdb::StatsLevel > kExceptTimers(2)
899+
{ "CompactionCPUTime", rocksdb::COMPACTION_CPU_TIME }, // enabled if rocksdb::StatsLevel > kExceptTimers(2)
900+
{ "CompressionTimeNanos",
901+
rocksdb::COMPRESSION_TIMES_NANOS }, // enabled if rocksdb::StatsLevel > kExceptDetailedTimers(3)
902+
{ "DecompressionTimeNanos",
903+
rocksdb::DECOMPRESSION_TIMES_NANOS }, // enabled if rocksdb::StatsLevel > kExceptDetailedTimers(3)
904+
{ "HardRateLimitDelayCount",
905+
rocksdb::HARD_RATE_LIMIT_DELAY_COUNT }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
906+
{ "SoftRateLimitDelayCount",
907+
rocksdb::SOFT_RATE_LIMIT_DELAY_COUNT }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
908+
{ "WriteStall", rocksdb::WRITE_STALL }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
904909
};
905910

906911
state std::vector<std::pair<const char*, std::string>> intPropertyStats = {
@@ -957,11 +962,13 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
957962
cum = stat;
958963
}
959964

960-
for (auto& [name, histogram] : histogramStats) {
961-
rocksdb::HistogramData histogram_data;
962-
statistics->histogramData(histogram, &histogram_data);
963-
e.detail(format("%s%d", name, "P95"), histogram_data.percentile95);
964-
e.detail(format("%s%d", name, "P99"), histogram_data.percentile99);
965+
if (SERVER_KNOBS->ROCKSDB_STATS_LEVEL > rocksdb::kExceptHistogramOrTimers) {
966+
for (auto& [name, histogram] : histogramStats) {
967+
rocksdb::HistogramData histogram_data;
968+
statistics->histogramData(histogram, &histogram_data);
969+
e.detail(format("%s%s", name, "P95"), histogram_data.percentile95);
970+
e.detail(format("%s%s", name, "P99"), histogram_data.percentile99);
971+
}
965972
}
966973

967974
for (const auto& [name, property] : intPropertyStats) {

0 commit comments

Comments
 (0)