Skip to content

Commit 1c5ee77

Browse files
RocksDB 7.10.2 version upgrade
1 parent 3e641a8 commit 1c5ee77

File tree

4 files changed

+106
-69
lines changed

4 files changed

+106
-69
lines changed

cmake/CompileRocksDB.cmake

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# FindRocksDB
22

3-
find_package(RocksDB 7.7.3)
3+
find_package(RocksDB 7.10.2)
44

55
include(ExternalProject)
66

@@ -54,8 +54,8 @@ if(ROCKSDB_FOUND)
5454
${BINARY_DIR}/librocksdb.a)
5555
else()
5656
ExternalProject_Add(rocksdb
57-
URL https://github.com/facebook/rocksdb/archive/refs/tags/v7.7.3.tar.gz
58-
URL_HASH SHA256=b8ac9784a342b2e314c821f6d701148912215666ac5e9bdbccd93cf3767cb611
57+
URL https://github.com/facebook/rocksdb/archive/refs/tags/v7.10.2.tar.gz
58+
URL_HASH SHA256=4619ae7308cd3d11cdd36f0bfad3fb03a1ad399ca333f192b77b6b95b08e2f78
5959
CMAKE_ARGS ${RocksDB_CMAKE_ARGS}
6060
BUILD_BYPRODUCTS <BINARY_DIR>/librocksdb.a
6161
INSTALL_COMMAND ""

documentation/sphinx/source/release-notes/release-notes-710.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ Release Notes
1515
* Fixed unnecessary transaction system recovery when excluding the servers that are already excluded/failed. `(PR #9809) <https://github.com/apple/foundationdb/pull/9809>`_
1616
* Fixed the exclusion of stateless processes by skipping the free capacity check. `(PR #9789) <https://github.com/apple/foundationdb/pull/9789>`_
1717
* Fixed an issue where the new worker cannot get ServerDBInfo update. `(PR #9778) <https://github.com/apple/foundationdb/pull/9778>`_
18+
* Added RocksDB bloom filter knobs. `(PR #9770) <https://github.com/apple/foundationdb/pull/9770>`_
19+
* Upgraded RocksDB to version 7.10.2. `(PR #9829) <https://github.com/apple/foundationdb/pull/9829>`_
1820

1921
7.1.29
2022
======

fdbserver/KeyValueStoreRocksDB.actor.cpp

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,9 @@
6868

6969
#ifdef SSD_ROCKSDB_EXPERIMENTAL
7070

71-
// Enforcing rocksdb version to be 7.7.3.
72-
static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 7 && ROCKSDB_PATCH == 3),
73-
"Unsupported rocksdb version. Update the rocksdb to 7.7.3 version");
71+
// Enforcing rocksdb version to be 7.10.2
72+
static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 10 && ROCKSDB_PATCH == 2),
73+
"Unsupported rocksdb version. Update the rocksdb to 7.10.2 version");
7474

7575
namespace {
7676
using rocksdb::BackgroundErrorReason;
@@ -226,10 +226,14 @@ rocksdb::ExportImportFilesMetaData getMetaData(const CheckpointMetaData& checkpo
226226

227227
for (const LiveFileMetaData& fileMetaData : rocksCF.sstFiles) {
228228
rocksdb::LiveFileMetaData liveFileMetaData;
229-
liveFileMetaData.size = fileMetaData.size;
230-
liveFileMetaData.name = fileMetaData.name;
229+
liveFileMetaData.relative_filename = fileMetaData.relative_filename;
230+
liveFileMetaData.directory = fileMetaData.directory;
231231
liveFileMetaData.file_number = fileMetaData.file_number;
232-
liveFileMetaData.db_path = fileMetaData.db_path;
232+
liveFileMetaData.file_type = static_cast<rocksdb::FileType>(fileMetaData.file_type);
233+
liveFileMetaData.size = fileMetaData.size;
234+
liveFileMetaData.temperature = static_cast<rocksdb::Temperature>(fileMetaData.temperature);
235+
liveFileMetaData.file_checksum = fileMetaData.file_checksum;
236+
liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
233237
liveFileMetaData.smallest_seqno = fileMetaData.smallest_seqno;
234238
liveFileMetaData.largest_seqno = fileMetaData.largest_seqno;
235239
liveFileMetaData.smallestkey = fileMetaData.smallestkey;
@@ -238,12 +242,12 @@ rocksdb::ExportImportFilesMetaData getMetaData(const CheckpointMetaData& checkpo
238242
liveFileMetaData.being_compacted = fileMetaData.being_compacted;
239243
liveFileMetaData.num_entries = fileMetaData.num_entries;
240244
liveFileMetaData.num_deletions = fileMetaData.num_deletions;
241-
liveFileMetaData.temperature = static_cast<rocksdb::Temperature>(fileMetaData.temperature);
242245
liveFileMetaData.oldest_blob_file_number = fileMetaData.oldest_blob_file_number;
243246
liveFileMetaData.oldest_ancester_time = fileMetaData.oldest_ancester_time;
244247
liveFileMetaData.file_creation_time = fileMetaData.file_creation_time;
245-
liveFileMetaData.file_checksum = fileMetaData.file_checksum;
246-
liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
248+
liveFileMetaData.epoch_number = fileMetaData.epoch_number;
249+
liveFileMetaData.name = fileMetaData.name;
250+
liveFileMetaData.db_path = fileMetaData.db_path;
247251
liveFileMetaData.column_family_name = fileMetaData.column_family_name;
248252
liveFileMetaData.level = fileMetaData.level;
249253
metaData.files.push_back(liveFileMetaData);
@@ -257,10 +261,14 @@ void populateMetaData(CheckpointMetaData* checkpoint, const rocksdb::ExportImpor
257261
rocksCF.dbComparatorName = metaData.db_comparator_name;
258262
for (const rocksdb::LiveFileMetaData& fileMetaData : metaData.files) {
259263
LiveFileMetaData liveFileMetaData;
260-
liveFileMetaData.size = fileMetaData.size;
261-
liveFileMetaData.name = fileMetaData.name;
264+
liveFileMetaData.relative_filename = fileMetaData.relative_filename;
265+
liveFileMetaData.directory = fileMetaData.directory;
262266
liveFileMetaData.file_number = fileMetaData.file_number;
263-
liveFileMetaData.db_path = fileMetaData.db_path;
267+
liveFileMetaData.file_type = static_cast<int>(fileMetaData.file_type);
268+
liveFileMetaData.size = fileMetaData.size;
269+
liveFileMetaData.temperature = static_cast<uint8_t>(fileMetaData.temperature);
270+
liveFileMetaData.file_checksum = fileMetaData.file_checksum;
271+
liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
264272
liveFileMetaData.smallest_seqno = fileMetaData.smallest_seqno;
265273
liveFileMetaData.largest_seqno = fileMetaData.largest_seqno;
266274
liveFileMetaData.smallestkey = fileMetaData.smallestkey;
@@ -269,12 +277,12 @@ void populateMetaData(CheckpointMetaData* checkpoint, const rocksdb::ExportImpor
269277
liveFileMetaData.being_compacted = fileMetaData.being_compacted;
270278
liveFileMetaData.num_entries = fileMetaData.num_entries;
271279
liveFileMetaData.num_deletions = fileMetaData.num_deletions;
272-
liveFileMetaData.temperature = static_cast<uint8_t>(fileMetaData.temperature);
273280
liveFileMetaData.oldest_blob_file_number = fileMetaData.oldest_blob_file_number;
274281
liveFileMetaData.oldest_ancester_time = fileMetaData.oldest_ancester_time;
275282
liveFileMetaData.file_creation_time = fileMetaData.file_creation_time;
276-
liveFileMetaData.file_checksum = fileMetaData.file_checksum;
277-
liveFileMetaData.file_checksum_func_name = fileMetaData.file_checksum_func_name;
283+
liveFileMetaData.epoch_number = fileMetaData.epoch_number;
284+
liveFileMetaData.name = fileMetaData.name;
285+
liveFileMetaData.db_path = fileMetaData.db_path;
278286
liveFileMetaData.column_family_name = fileMetaData.column_family_name;
279287
liveFileMetaData.level = fileMetaData.level;
280288
rocksCF.sstFiles.push_back(liveFileMetaData);
@@ -2472,7 +2480,7 @@ TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/CheckpointRestore") {
24722480
TEST_CASE("noSim/fdbserver/KeyValueStoreRocksDB/RocksDBTypes") {
24732481
// If the following assertion fails, update SstFileMetaData and LiveFileMetaData in RocksDBCheckpointUtils.actor.h
24742482
// to be the same as rocksdb::SstFileMetaData and rocksdb::LiveFileMetaData.
2475-
ASSERT_EQ(sizeof(rocksdb::LiveFileMetaData), 184);
2483+
ASSERT_EQ(sizeof(rocksdb::LiveFileMetaData), 192);
24762484
ASSERT_EQ(sizeof(rocksdb::ExportImportFilesMetaData), 32);
24772485
return Void();
24782486
}

fdbserver/RocksDBCheckpointUtils.actor.h

Lines changed: 77 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -35,55 +35,74 @@
3535
struct SstFileMetaData {
3636
constexpr static FileIdentifier file_identifier = 3804347;
3737
SstFileMetaData()
38-
: size(0), file_number(0), smallest_seqno(0), largest_seqno(0), num_reads_sampled(0), being_compacted(false),
39-
num_entries(0), num_deletions(0), temperature(0), oldest_blob_file_number(0), oldest_ancester_time(0),
40-
file_creation_time(0) {}
38+
: file_number(0), file_type(2), size(0), temperature(0), smallest_seqno(0), largest_seqno(0),
39+
num_reads_sampled(0), being_compacted(false), num_entries(0), num_deletions(0), oldest_blob_file_number(0),
40+
oldest_ancester_time(0), file_creation_time(0), epoch_number(0) {}
4141

42-
SstFileMetaData(const std::string& _file_name,
42+
SstFileMetaData(const std::string& _relative_filename,
43+
const std::string& _directory,
4344
uint64_t _file_number,
44-
const std::string& _path,
45-
size_t _size,
45+
int _file_type,
46+
uint64_t _size,
47+
int _temperature,
48+
std::string& _file_checksum,
49+
std::string& _file_checksum_func_name,
4650
uint64_t _smallest_seqno,
4751
uint64_t _largest_seqno,
4852
const std::string& _smallestkey,
4953
const std::string& _largestkey,
5054
uint64_t _num_reads_sampled,
5155
bool _being_compacted,
52-
int _temperature,
56+
uint64_t _num_entries,
57+
uint64_t _num_deletions,
5358
uint64_t _oldest_blob_file_number,
5459
uint64_t _oldest_ancester_time,
5560
uint64_t _file_creation_time,
56-
std::string& _file_checksum,
57-
std::string& _file_checksum_func_name)
58-
: size(_size), name(_file_name), file_number(_file_number), db_path(_path), smallest_seqno(_smallest_seqno),
61+
uint64_t _epoch_number,
62+
const std::string& _name,
63+
const std::string& _db_path)
64+
: relative_filename(_relative_filename), directory(_directory), file_number(_file_number), file_type(_file_type),
65+
size(_size), temperature(_temperature), file_checksum(_file_checksum),
66+
file_checksum_func_name(_file_checksum_func_name), smallest_seqno(_smallest_seqno),
5967
largest_seqno(_largest_seqno), smallestkey(_smallestkey), largestkey(_largestkey),
60-
num_reads_sampled(_num_reads_sampled), being_compacted(_being_compacted), num_entries(0), num_deletions(0),
61-
temperature(_temperature), oldest_blob_file_number(_oldest_blob_file_number),
68+
num_reads_sampled(_num_reads_sampled), being_compacted(_being_compacted), num_entries(_num_entries),
69+
num_deletions(_num_deletions), oldest_blob_file_number(_oldest_blob_file_number),
6270
oldest_ancester_time(_oldest_ancester_time), file_creation_time(_file_creation_time),
63-
file_checksum(_file_checksum), file_checksum_func_name(_file_checksum_func_name) {}
64-
65-
// File size in bytes.
66-
size_t size;
67-
// The name of the file.
68-
std::string name;
69-
// The id of the file.
71+
epoch_number(_epoch_number), name(_name), db_path(_db_path) {}
72+
73+
// The name of the file within its directory (e.g. "123456.sst")
74+
std::string relative_filename;
75+
// The directory containing the file, without a trailing '/'. This could be
76+
// a DB path, wal_dir, etc.
77+
std::string directory;
78+
// The id of the file within a single DB. Set to 0 if the file does not have
79+
// a number (e.g. CURRENT)
7080
uint64_t file_number;
71-
// The full path where the file locates.
72-
std::string db_path;
81+
// The type of the file as part of a DB.
82+
int file_type;
83+
// File size in bytes. See also `trim_to_size`.
84+
uint64_t size;
85+
// This feature is experimental and subject to change.
86+
int temperature;
87+
// The checksum of a SST file, the value is decided by the file content and
88+
// the checksum algorithm used for this SST file. The checksum function is
89+
// identified by the file_checksum_func_name. If the checksum function is
90+
// not specified, file_checksum is "0" by default.
91+
std::string file_checksum;
92+
// The name of the checksum function used to generate the file checksum
93+
// value. If file checksum is not enabled (e.g., sst_file_checksum_func is
94+
// null), file_checksum_func_name is UnknownFileChecksumFuncName, which is
95+
// "Unknown".
96+
std::string file_checksum_func_name;
7397

7498
uint64_t smallest_seqno; // Smallest sequence number in file.
7599
uint64_t largest_seqno; // Largest sequence number in file.
76100
std::string smallestkey; // Smallest user defined key in the file.
77101
std::string largestkey; // Largest user defined key in the file.
78102
uint64_t num_reads_sampled; // How many times the file is read.
79103
bool being_compacted; // true if the file is currently being compacted.
80-
81104
uint64_t num_entries;
82105
uint64_t num_deletions;
83-
84-
// This feature is experimental and subject to change.
85-
int temperature;
86-
87106
uint64_t oldest_blob_file_number; // The id of the oldest blob file
88107
// referenced by the file.
89108
// An SST file may be generated by compactions whose input files may
@@ -97,26 +116,30 @@ struct SstFileMetaData {
97116
// Timestamp when the SST file is created, provided by
98117
// SystemClock::GetCurrentTime(). 0 if the information is not available.
99118
uint64_t file_creation_time;
100-
101-
// The checksum of a SST file, the value is decided by the file content and
102-
// the checksum algorithm used for this SST file. The checksum function is
103-
// identified by the file_checksum_func_name. If the checksum function is
104-
// not specified, file_checksum is "0" by default.
105-
std::string file_checksum;
106-
107-
// The name of the checksum function used to generate the file checksum
108-
// value. If file checksum is not enabled (e.g., sst_file_checksum_func is
109-
// null), file_checksum_func_name is UnknownFileChecksumFuncName, which is
110-
// "Unknown".
111-
std::string file_checksum_func_name;
119+
// The order of a file being flushed or ingested/imported.
120+
// Compaction output file will be assigned with the minimum `epoch_number`
121+
// among input files'.
122+
// For L0, larger `epoch_number` indicates newer L0 file.
123+
// 0 if the information is not available.
124+
uint64_t epoch_number;
125+
// DEPRECATED: The name of the file within its directory with a
126+
// leading slash (e.g. "/123456.sst"). Use relative_filename from base struct
127+
// instead.
128+
std::string name;
129+
// DEPRECATED: replaced by `directory` in base struct
130+
std::string db_path;
112131

113132
template <class Ar>
114133
void serialize(Ar& ar) {
115134
serializer(ar,
116-
size,
117-
name,
135+
relative_filename,
136+
directory,
118137
file_number,
119-
db_path,
138+
file_type,
139+
size,
140+
temperature,
141+
file_checksum,
142+
file_checksum_func_name,
120143
smallest_seqno,
121144
largest_seqno,
122145
smallestkey,
@@ -125,12 +148,12 @@ struct SstFileMetaData {
125148
being_compacted,
126149
num_entries,
127150
num_deletions,
128-
temperature,
129151
oldest_blob_file_number,
130152
oldest_ancester_time,
131153
file_creation_time,
132-
file_checksum,
133-
file_checksum_func_name);
154+
epoch_number,
155+
name,
156+
db_path);
134157
}
135158
};
136159

@@ -145,10 +168,14 @@ struct LiveFileMetaData : public SstFileMetaData {
145168
template <class Ar>
146169
void serialize(Ar& ar) {
147170
serializer(ar,
148-
SstFileMetaData::size,
149-
SstFileMetaData::name,
171+
SstFileMetaData::relative_filename,
172+
SstFileMetaData::directory,
150173
SstFileMetaData::file_number,
151-
SstFileMetaData::db_path,
174+
SstFileMetaData::file_type,
175+
SstFileMetaData::size,
176+
SstFileMetaData::temperature,
177+
SstFileMetaData::file_checksum,
178+
SstFileMetaData::file_checksum_func_name,
152179
SstFileMetaData::smallest_seqno,
153180
SstFileMetaData::largest_seqno,
154181
SstFileMetaData::smallestkey,
@@ -157,12 +184,12 @@ struct LiveFileMetaData : public SstFileMetaData {
157184
SstFileMetaData::being_compacted,
158185
SstFileMetaData::num_entries,
159186
SstFileMetaData::num_deletions,
160-
SstFileMetaData::temperature,
161187
SstFileMetaData::oldest_blob_file_number,
162188
SstFileMetaData::oldest_ancester_time,
163189
SstFileMetaData::file_creation_time,
164-
SstFileMetaData::file_checksum,
165-
SstFileMetaData::file_checksum_func_name,
190+
SstFileMetaData::epoch_number,
191+
SstFileMetaData::name,
192+
SstFileMetaData::db_path,
166193
column_family_name,
167194
level,
168195
fetched);

0 commit comments

Comments
 (0)