Skip to content

Commit 8d0eb03

Browse files
RocksDB 7.10.2 version upgrade
1 parent cf65c26 commit 8d0eb03

File tree

5 files changed

+109
-74
lines changed

5 files changed

+109
-74
lines changed

cmake/CompileRocksDB.cmake

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# FindRocksDB
22

3-
find_package(RocksDB 7.7.3)
3+
find_package(RocksDB 7.10.2)
44

55
include(ExternalProject)
66

@@ -52,8 +52,8 @@ if(ROCKSDB_FOUND)
5252
${BINARY_DIR}/librocksdb.a)
5353
else()
5454
ExternalProject_Add(rocksdb
55-
URL https://github.com/facebook/rocksdb/archive/refs/tags/v7.7.3.tar.gz
56-
URL_HASH SHA256=b8ac9784a342b2e314c821f6d701148912215666ac5e9bdbccd93cf3767cb611
55+
URL https://github.com/facebook/rocksdb/archive/refs/tags/v7.10.2.tar.gz
56+
URL_HASH SHA256=4619ae7308cd3d11cdd36f0bfad3fb03a1ad399ca333f192b77b6b95b08e2f78
5757
CMAKE_ARGS ${RocksDB_CMAKE_ARGS}
5858
BUILD_BYPRODUCTS <BINARY_DIR>/librocksdb.a
5959
INSTALL_COMMAND ""

fdbserver/KeyValueStoreRocksDB.actor.cpp

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,9 @@
6868

6969
#ifdef SSD_ROCKSDB_EXPERIMENTAL
7070

71-
// Enforcing rocksdb version to be 7.7.3.
72-
static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 7 && ROCKSDB_PATCH == 3),
73-
"Unsupported rocksdb version. Update the rocksdb to 7.7.3 version");
71+
// Enforcing rocksdb version to be 7.10.2
72+
static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 10 && ROCKSDB_PATCH == 2),
73+
"Unsupported rocksdb version. Update the rocksdb to 7.10.2 version");
7474

7575
namespace {
7676
using rocksdb::BackgroundErrorReason;
@@ -354,10 +354,14 @@ rocksdb::ExportImportFilesMetaData getMetaData(const CheckpointMetaData& checkpo
354354

355355
for (const LiveFileMetaData& fileMetaData : rocksCF.sstFiles) {
356356
rocksdb::LiveFileMetaData liveFileMetaData;
357-
liveFileMetaData.size = fileMetaData.size;
358-
liveFileMetaData.name = fileMetaData.name;
357+
liveFileMetaData.relative_filename = fileMetaData.relative_filename;
358+
liveFileMetaData.directory = fileMetaData.directory;
359359
liveFileMetaData.file_number = fileMetaData.file_number;
360-
liveFileMetaData.db_path = fileMetaData.db_path;
360+
liveFileMetaData.file_type = static_cast<rocksdb::FileType>(fileMetaData.file_type);
361+
liveFileMetaData.size = fileMetaData.size;
362+
liveFileMetaData.temperature = static_cast<rocksdb::Temperature>(fileMetaData.temperature);
363+
liveFileMetaData.file_checksum = fileMetaData.file_checksum;
364+
liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
361365
liveFileMetaData.smallest_seqno = fileMetaData.smallest_seqno;
362366
liveFileMetaData.largest_seqno = fileMetaData.largest_seqno;
363367
liveFileMetaData.smallestkey = fileMetaData.smallestkey;
@@ -366,12 +370,12 @@ rocksdb::ExportImportFilesMetaData getMetaData(const CheckpointMetaData& checkpo
366370
liveFileMetaData.being_compacted = fileMetaData.being_compacted;
367371
liveFileMetaData.num_entries = fileMetaData.num_entries;
368372
liveFileMetaData.num_deletions = fileMetaData.num_deletions;
369-
liveFileMetaData.temperature = static_cast<rocksdb::Temperature>(fileMetaData.temperature);
370373
liveFileMetaData.oldest_blob_file_number = fileMetaData.oldest_blob_file_number;
371374
liveFileMetaData.oldest_ancester_time = fileMetaData.oldest_ancester_time;
372375
liveFileMetaData.file_creation_time = fileMetaData.file_creation_time;
373-
liveFileMetaData.file_checksum = fileMetaData.file_checksum;
374-
liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
376+
liveFileMetaData.epoch_number = fileMetaData.epoch_number;
377+
liveFileMetaData.name = fileMetaData.name;
378+
liveFileMetaData.db_path = fileMetaData.db_path;
375379
liveFileMetaData.column_family_name = fileMetaData.column_family_name;
376380
liveFileMetaData.level = fileMetaData.level;
377381
metaData.files.push_back(liveFileMetaData);
@@ -385,10 +389,14 @@ void populateMetaData(CheckpointMetaData* checkpoint, const rocksdb::ExportImpor
385389
rocksCF.dbComparatorName = metaData.db_comparator_name;
386390
for (const rocksdb::LiveFileMetaData& fileMetaData : metaData.files) {
387391
LiveFileMetaData liveFileMetaData;
388-
liveFileMetaData.size = fileMetaData.size;
389-
liveFileMetaData.name = fileMetaData.name;
392+
liveFileMetaData.relative_filename = fileMetaData.relative_filename;
393+
liveFileMetaData.directory = fileMetaData.directory;
390394
liveFileMetaData.file_number = fileMetaData.file_number;
391-
liveFileMetaData.db_path = fileMetaData.db_path;
395+
liveFileMetaData.file_type = static_cast<int>(fileMetaData.file_type);
396+
liveFileMetaData.size = fileMetaData.size;
397+
liveFileMetaData.temperature = static_cast<uint8_t>(fileMetaData.temperature);
398+
liveFileMetaData.file_checksum = fileMetaData.file_checksum;
399+
liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
392400
liveFileMetaData.smallest_seqno = fileMetaData.smallest_seqno;
393401
liveFileMetaData.largest_seqno = fileMetaData.largest_seqno;
394402
liveFileMetaData.smallestkey = fileMetaData.smallestkey;
@@ -397,12 +405,12 @@ void populateMetaData(CheckpointMetaData* checkpoint, const rocksdb::ExportImpor
397405
liveFileMetaData.being_compacted = fileMetaData.being_compacted;
398406
liveFileMetaData.num_entries = fileMetaData.num_entries;
399407
liveFileMetaData.num_deletions = fileMetaData.num_deletions;
400-
liveFileMetaData.temperature = static_cast<uint8_t>(fileMetaData.temperature);
401408
liveFileMetaData.oldest_blob_file_number = fileMetaData.oldest_blob_file_number;
402409
liveFileMetaData.oldest_ancester_time = fileMetaData.oldest_ancester_time;
403410
liveFileMetaData.file_creation_time = fileMetaData.file_creation_time;
404-
liveFileMetaData.file_checksum = fileMetaData.file_checksum;
405-
liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
411+
liveFileMetaData.epoch_number = fileMetaData.epoch_number;
412+
liveFileMetaData.name = fileMetaData.name;
413+
liveFileMetaData.db_path = fileMetaData.db_path;
406414
liveFileMetaData.column_family_name = fileMetaData.column_family_name;
407415
liveFileMetaData.level = fileMetaData.level;
408416
rocksCF.sstFiles.push_back(liveFileMetaData);

fdbserver/KeyValueStoreShardedRocksDB.actor.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,9 @@
4242

4343
#ifdef SSD_ROCKSDB_EXPERIMENTAL
4444

45-
// Enforcing rocksdb version to be 7.7.3.
46-
static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 7 && ROCKSDB_PATCH == 3),
47-
"Unsupported rocksdb version. Update the rocksdb to 7.7.3 version");
45+
// Enforcing rocksdb version to be 7.10.2
46+
static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 10 && ROCKSDB_PATCH == 2),
47+
"Unsupported rocksdb version. Update the rocksdb to 7.10.2 version");
4848

4949
const std::string rocksDataFolderSuffix = "-data";
5050
const std::string METADATA_SHARD_ID = "kvs-metadata";

fdbserver/RocksDBCheckpointUtils.actor.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,9 @@
4343
#include "flow/actorcompiler.h" // has to be last include
4444

4545
#ifdef SSD_ROCKSDB_EXPERIMENTAL
46-
// Enforcing rocksdb version to be 7.7.3.
47-
static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 7 && ROCKSDB_PATCH == 3),
48-
"Unsupported rocksdb version. Update the rocksdb to 7.7.3 version");
46+
// Enforcing rocksdb version to be 7.10.2
47+
static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 10 && ROCKSDB_PATCH == 2),
48+
"Unsupported rocksdb version. Update the rocksdb to 7.10.2 version");
4949

5050
namespace {
5151

fdbserver/include/fdbserver/RocksDBCheckpointUtils.actor.h

Lines changed: 77 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -55,55 +55,74 @@ struct CheckpointFile {
5555
struct SstFileMetaData {
5656
constexpr static FileIdentifier file_identifier = 3804347;
5757
SstFileMetaData()
58-
: size(0), file_number(0), smallest_seqno(0), largest_seqno(0), num_reads_sampled(0), being_compacted(false),
59-
num_entries(0), num_deletions(0), temperature(0), oldest_blob_file_number(0), oldest_ancester_time(0),
60-
file_creation_time(0) {}
58+
: file_number(0), file_type(2), size(0), temperature(0), smallest_seqno(0), largest_seqno(0),
59+
num_reads_sampled(0), being_compacted(false), num_entries(0), num_deletions(0), oldest_blob_file_number(0),
60+
oldest_ancester_time(0), file_creation_time(0), epoch_number(0) {}
6161

62-
SstFileMetaData(const std::string& _file_name,
62+
SstFileMetaData(const std::string& _relative_filename,
63+
const std::string& _directory,
6364
uint64_t _file_number,
64-
const std::string& _path,
65-
size_t _size,
65+
int _file_type,
66+
uint64_t _size,
67+
int _temperature,
68+
std::string& _file_checksum,
69+
std::string& _file_checksum_func_name,
6670
uint64_t _smallest_seqno,
6771
uint64_t _largest_seqno,
6872
const std::string& _smallestkey,
6973
const std::string& _largestkey,
7074
uint64_t _num_reads_sampled,
7175
bool _being_compacted,
72-
int _temperature,
76+
uint64_t _num_entries,
77+
uint64_t _num_deletions,
7378
uint64_t _oldest_blob_file_number,
7479
uint64_t _oldest_ancester_time,
7580
uint64_t _file_creation_time,
76-
std::string& _file_checksum,
77-
std::string& _file_checksum_func_name)
78-
: size(_size), name(_file_name), file_number(_file_number), db_path(_path), smallest_seqno(_smallest_seqno),
81+
uint64_t _epoch_number,
82+
const std::string& _name,
83+
const std::string& _db_path)
84+
: relative_filename(_relative_filename), directory(_directory), file_number(_file_number), file_type(_file_type),
85+
size(_size), temperature(_temperature), file_checksum(_file_checksum),
86+
file_checksum_func_name(_file_checksum_func_name), smallest_seqno(_smallest_seqno),
7987
largest_seqno(_largest_seqno), smallestkey(_smallestkey), largestkey(_largestkey),
80-
num_reads_sampled(_num_reads_sampled), being_compacted(_being_compacted), num_entries(0), num_deletions(0),
81-
temperature(_temperature), oldest_blob_file_number(_oldest_blob_file_number),
88+
num_reads_sampled(_num_reads_sampled), being_compacted(_being_compacted), num_entries(_num_entries),
89+
num_deletions(_num_deletions), oldest_blob_file_number(_oldest_blob_file_number),
8290
oldest_ancester_time(_oldest_ancester_time), file_creation_time(_file_creation_time),
83-
file_checksum(_file_checksum), file_checksum_func_name(_file_checksum_func_name) {}
84-
85-
// File size in bytes.
86-
size_t size;
87-
// The name of the file.
88-
std::string name;
89-
// The id of the file.
91+
epoch_number(_epoch_number), name(_name), db_path(_db_path) {}
92+
93+
// The name of the file within its directory (e.g. "123456.sst")
94+
std::string relative_filename;
95+
// The directory containing the file, without a trailing '/'. This could be
96+
// a DB path, wal_dir, etc.
97+
std::string directory;
98+
// The id of the file within a single DB. Set to 0 if the file does not have
99+
// a number (e.g. CURRENT)
90100
uint64_t file_number;
91-
// The full path where the file locates.
92-
std::string db_path;
101+
// The type of the file as part of a DB.
102+
int file_type;
103+
// File size in bytes. See also `trim_to_size`.
104+
uint64_t size;
105+
// This feature is experimental and subject to change.
106+
int temperature;
107+
// The checksum of a SST file, the value is decided by the file content and
108+
// the checksum algorithm used for this SST file. The checksum function is
109+
// identified by the file_checksum_func_name. If the checksum function is
110+
// not specified, file_checksum is "0" by default.
111+
std::string file_checksum;
112+
// The name of the checksum function used to generate the file checksum
113+
// value. If file checksum is not enabled (e.g., sst_file_checksum_func is
114+
// null), file_checksum_func_name is UnknownFileChecksumFuncName, which is
115+
// "Unknown".
116+
std::string file_checksum_func_name;
93117

94118
uint64_t smallest_seqno; // Smallest sequence number in file.
95119
uint64_t largest_seqno; // Largest sequence number in file.
96120
std::string smallestkey; // Smallest user defined key in the file.
97121
std::string largestkey; // Largest user defined key in the file.
98122
uint64_t num_reads_sampled; // How many times the file is read.
99123
bool being_compacted; // true if the file is currently being compacted.
100-
101124
uint64_t num_entries;
102125
uint64_t num_deletions;
103-
104-
// This feature is experimental and subject to change.
105-
int temperature;
106-
107126
uint64_t oldest_blob_file_number; // The id of the oldest blob file
108127
// referenced by the file.
109128
// An SST file may be generated by compactions whose input files may
@@ -117,26 +136,30 @@ struct SstFileMetaData {
117136
// Timestamp when the SST file is created, provided by
118137
// SystemClock::GetCurrentTime(). 0 if the information is not available.
119138
uint64_t file_creation_time;
120-
121-
// The checksum of a SST file, the value is decided by the file content and
122-
// the checksum algorithm used for this SST file. The checksum function is
123-
// identified by the file_checksum_func_name. If the checksum function is
124-
// not specified, file_checksum is "0" by default.
125-
std::string file_checksum;
126-
127-
// The name of the checksum function used to generate the file checksum
128-
// value. If file checksum is not enabled (e.g., sst_file_checksum_func is
129-
// null), file_checksum_func_name is UnknownFileChecksumFuncName, which is
130-
// "Unknown".
131-
std::string file_checksum_func_name;
139+
// The order of a file being flushed or ingested/imported.
140+
// Compaction output file will be assigned with the minimum `epoch_number`
141+
// among input files'.
142+
// For L0, larger `epoch_number` indicates newer L0 file.
143+
// 0 if the information is not available.
144+
uint64_t epoch_number;
145+
// DEPRECATED: The name of the file within its directory with a
146+
// leading slash (e.g. "/123456.sst"). Use relative_filename from base struct
147+
// instead.
148+
std::string name;
149+
// DEPRECATED: replaced by `directory` in base struct
150+
std::string db_path;
132151

133152
template <class Ar>
134153
void serialize(Ar& ar) {
135154
serializer(ar,
136-
size,
137-
name,
155+
relative_filename,
156+
directory,
138157
file_number,
139-
db_path,
158+
file_type,
159+
size,
160+
temperature,
161+
file_checksum,
162+
file_checksum_func_name,
140163
smallest_seqno,
141164
largest_seqno,
142165
smallestkey,
@@ -145,12 +168,12 @@ struct SstFileMetaData {
145168
being_compacted,
146169
num_entries,
147170
num_deletions,
148-
temperature,
149171
oldest_blob_file_number,
150172
oldest_ancester_time,
151173
file_creation_time,
152-
file_checksum,
153-
file_checksum_func_name);
174+
epoch_number,
175+
name,
176+
db_path);
154177
}
155178
};
156179

@@ -165,10 +188,14 @@ struct LiveFileMetaData : public SstFileMetaData {
165188
template <class Ar>
166189
void serialize(Ar& ar) {
167190
serializer(ar,
168-
SstFileMetaData::size,
169-
SstFileMetaData::name,
191+
SstFileMetaData::relative_filename,
192+
SstFileMetaData::directory,
170193
SstFileMetaData::file_number,
171-
SstFileMetaData::db_path,
194+
SstFileMetaData::file_type,
195+
SstFileMetaData::size,
196+
SstFileMetaData::temperature,
197+
SstFileMetaData::file_checksum,
198+
SstFileMetaData::file_checksum_func_name,
172199
SstFileMetaData::smallest_seqno,
173200
SstFileMetaData::largest_seqno,
174201
SstFileMetaData::smallestkey,
@@ -177,12 +204,12 @@ struct LiveFileMetaData : public SstFileMetaData {
177204
SstFileMetaData::being_compacted,
178205
SstFileMetaData::num_entries,
179206
SstFileMetaData::num_deletions,
180-
SstFileMetaData::temperature,
181207
SstFileMetaData::oldest_blob_file_number,
182208
SstFileMetaData::oldest_ancester_time,
183209
SstFileMetaData::file_creation_time,
184-
SstFileMetaData::file_checksum,
185-
SstFileMetaData::file_checksum_func_name,
210+
SstFileMetaData::epoch_number,
211+
SstFileMetaData::name,
212+
SstFileMetaData::db_path,
186213
column_family_name,
187214
level,
188215
fetched);

0 commit comments

Comments
 (0)