Skip to content

Obfuscate chainstate #6650

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
laanwj merged 1 commit into from
Oct 6, 2015
Merged

Obfuscate chainstate #6650

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/Makefile.test.include
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ BITCOIN_TESTS =\
test/hash_tests.cpp \
test/key_tests.cpp \
test/limitedmap_tests.cpp \
test/leveldbwrapper_tests.cpp \
test/main_tests.cpp \
test/mempool_tests.cpp \
test/miner_tests.cpp \
Expand All @@ -72,6 +73,7 @@ BITCOIN_TESTS =\
test/sighash_tests.cpp \
test/sigopcount_tests.cpp \
test/skiplist_tests.cpp \
test/streams_tests.cpp \
test/test_bitcoin.cpp \
test/test_bitcoin.h \
test/timedata_tests.cpp \
Expand Down
60 changes: 59 additions & 1 deletion src/leveldbwrapper.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@
#include "leveldbwrapper.h"

#include "util.h"
#include "random.h"

#include <boost/filesystem.hpp>

#include <leveldb/cache.h>
#include <leveldb/env.h>
#include <leveldb/filter_policy.h>
#include <memenv.h>
#include <stdint.h>

void HandleError(const leveldb::Status& status) throw(leveldb_error)
{
Expand Down Expand Up @@ -43,7 +45,7 @@ static leveldb::Options GetOptions(size_t nCacheSize)
return options;
}

CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe)
CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
{
penv = NULL;
readoptions.verify_checksums = true;
Expand All @@ -67,6 +69,25 @@ CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCa
leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb);
HandleError(status);
LogPrintf("Opened LevelDB successfully\n");

// The base-case obfuscation key, which is a noop.
obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');

bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);

if (!key_exists && obfuscate && IsEmpty()) {
// Initialize non-degenerate obfuscation if it won't upset
// existing, non-obfuscated data.
std::vector<unsigned char> new_key = CreateObfuscateKey();

// Write `new_key` so we don't obfuscate the key with itself
Write(OBFUSCATE_KEY_KEY, new_key);
obfuscate_key = new_key;

LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), GetObfuscateKeyHex());
}

LogPrintf("Using obfuscation key for %s: %s\n", path.string(), GetObfuscateKeyHex());
}

CLevelDBWrapper::~CLevelDBWrapper()
Expand All @@ -87,3 +108,40 @@ bool CLevelDBWrapper::WriteBatch(CLevelDBBatch& batch, bool fSync) throw(leveldb
HandleError(status);
return true;
}

// Prefixed with null character to avoid collisions with other keys
//
// We must use a string constructor which specifies length so that we copy
// past the null-terminator.
const std::string CLevelDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);

const unsigned int CLevelDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;

/**
* Returns a string (consisting of 8 random bytes) suitable for use as an
* obfuscating XOR key.
*/
std::vector<unsigned char> CLevelDBWrapper::CreateObfuscateKey() const
{
unsigned char buff[OBFUSCATE_KEY_NUM_BYTES];
GetRandBytes(buff, OBFUSCATE_KEY_NUM_BYTES);
return std::vector<unsigned char>(&buff[0], &buff[OBFUSCATE_KEY_NUM_BYTES]);

}

bool CLevelDBWrapper::IsEmpty()
{
boost::scoped_ptr<leveldb::Iterator> it(NewIterator());
it->SeekToFirst();
return !(it->Valid());
}

const std::vector<unsigned char>& CLevelDBWrapper::GetObfuscateKey() const
{
return obfuscate_key;
}

std::string CLevelDBWrapper::GetObfuscateKeyHex() const
{
return HexStr(obfuscate_key);
}
53 changes: 49 additions & 4 deletions src/leveldbwrapper.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include "serialize.h"
#include "streams.h"
#include "util.h"
#include "utilstrencodings.h"
#include "version.h"

#include <boost/filesystem/path.hpp>
Expand All @@ -31,8 +32,14 @@ class CLevelDBBatch

private:
leveldb::WriteBatch batch;
const std::vector<unsigned char> obfuscate_key;

public:
/**
* @param[in] obfuscate_key If passed, XOR data with this key.
*/
CLevelDBBatch(const std::vector<unsigned char>& obfuscate_key) : obfuscate_key(obfuscate_key) { };

template <typename K, typename V>
void Write(const K& key, const V& value)
{
Expand All @@ -44,6 +51,7 @@ class CLevelDBBatch
CDataStream ssValue(SER_DISK, CLIENT_VERSION);
ssValue.reserve(ssValue.GetSerializeSize(value));
ssValue << value;
ssValue.Xor(obfuscate_key);
leveldb::Slice slValue(&ssValue[0], ssValue.size());

batch.Put(slKey, slValue);
Expand Down Expand Up @@ -85,8 +93,27 @@ class CLevelDBWrapper
//! the database itself
leveldb::DB* pdb;

//! a key used for optional XOR-obfuscation of the database
std::vector<unsigned char> obfuscate_key;

//! the key under which the obfuscation key is stored
static const std::string OBFUSCATE_KEY_KEY;

//! the length of the obfuscate key in number of bytes
static const unsigned int OBFUSCATE_KEY_NUM_BYTES;

std::vector<unsigned char> CreateObfuscateKey() const;

public:
CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false);
/**
* @param[in] path Location in the filesystem where leveldb data will be stored.
* @param[in] nCacheSize Configures various leveldb cache settings.
* @param[in] fMemory If true, use leveldb's memory environment.
* @param[in] fWipe If true, remove all existing data.
* @param[in] obfuscate If true, store data obfuscated via simple XOR. If false, XOR
* with a zero'd byte array.
*/
CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false);
~CLevelDBWrapper();

template <typename K, typename V>
Expand All @@ -107,6 +134,7 @@ class CLevelDBWrapper
}
try {
CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION);
ssValue.Xor(obfuscate_key);
ssValue >> value;
} catch (const std::exception&) {
return false;
Expand All @@ -117,7 +145,7 @@ class CLevelDBWrapper
template <typename K, typename V>
bool Write(const K& key, const V& value, bool fSync = false) throw(leveldb_error)
{
CLevelDBBatch batch;
CLevelDBBatch batch(obfuscate_key);
batch.Write(key, value);
return WriteBatch(batch, fSync);
}
Expand All @@ -144,7 +172,7 @@ class CLevelDBWrapper
template <typename K>
bool Erase(const K& key, bool fSync = false) throw(leveldb_error)
{
CLevelDBBatch batch;
CLevelDBBatch batch(obfuscate_key);
batch.Erase(key);
return WriteBatch(batch, fSync);
}
Expand All @@ -159,7 +187,7 @@ class CLevelDBWrapper

bool Sync() throw(leveldb_error)
{
CLevelDBBatch batch;
CLevelDBBatch batch(obfuscate_key);
return WriteBatch(batch, true);
}

Expand All @@ -168,6 +196,23 @@ class CLevelDBWrapper
{
return pdb->NewIterator(iteroptions);
}

/**
Copy link
Member

@laanwj laanwj Sep 29, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd put the implementation of these functions in leveldbwrapper.cpp instead of the header file (unless there's a reason I'm overlooking)

Copy link
Contributor Author

@jamesob jamesob Sep 29, 2015

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I implemented these in the header to maintain consistency with much of the rest of CLevelDBWrapper, which is also defined here. I'll move what's below over to the .cpp file, but we should probably think about moving the rest of it out of the header at some point.

* Return true if the database managed by this class contains no entries.
*/
bool IsEmpty();

/**
* Accessor for obfuscate_key.
*/
const std::vector<unsigned char>& GetObfuscateKey() const;

/**
* Return the obfuscate_key as a hex-formatted string.
*/
std::string GetObfuscateKeyHex() const;

};

#endif // BITCOIN_LEVELDBWRAPPER_H

23 changes: 23 additions & 0 deletions src/streams.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,29 @@ class CDataStream
data.insert(data.end(), begin(), end());
clear();
}

/**
* XOR the contents of this stream with a certain key.
*
* @param[in] key The key used to XOR the data in this stream.
*/
void Xor(const std::vector<unsigned char>& key)
{
if (key.size() == 0) {
return;
}

for (size_type i = 0, j = 0; i != size(); i++) {
vch[i] ^= key[j++];

// This potentially acts on very many bytes of data, so it's
// important that we calculate `j`, i.e. the `key` index in this
// way instead of doing a %, which would effectively be a division
// for each byte Xor'd -- much slower than need be.
if (j == key.size())
j = 0;
}
}
};


Expand Down
128 changes: 128 additions & 0 deletions src/test/leveldbwrapper_tests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
// Copyright (c) 2012-2013 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "leveldbwrapper.h"
#include "uint256.h"
#include "random.h"
#include "test/test_bitcoin.h"

#include <boost/assign/std/vector.hpp> // for 'operator+=()'
#include <boost/assert.hpp>
#include <boost/test/unit_test.hpp>

using namespace std;
using namespace boost::assign; // bring 'operator+=()' into scope
using namespace boost::filesystem;

// Test if a string consists entirely of null characters
bool is_null_key(const vector<unsigned char>& key) {
bool isnull = true;

for (unsigned int i = 0; i < key.size(); i++)
isnull &= (key[i] == '\x00');

return isnull;
}

BOOST_FIXTURE_TEST_SUITE(leveldbwrapper_tests, BasicTestingSetup)

BOOST_AUTO_TEST_CASE(leveldbwrapper)
{
// Perform tests both obfuscated and non-obfuscated.
for (int i = 0; i < 2; i++) {
bool obfuscate = (bool)i;
path ph = temp_directory_path() / unique_path();
CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate);
char key = 'k';
uint256 in = GetRandHash();
uint256 res;

// Ensure that we're doing real obfuscation when obfuscate=true
BOOST_CHECK(obfuscate != is_null_key(dbw.GetObfuscateKey()));

BOOST_CHECK(dbw.Write(key, in));
BOOST_CHECK(dbw.Read(key, res));
BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
}
}

// Test that we do not obfuscation if there is existing data.
BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate)
{
// We're going to share this path between two wrappers
path ph = temp_directory_path() / unique_path();
create_directories(ph);

// Set up a non-obfuscated wrapper to write some initial data.
CLevelDBWrapper* dbw = new CLevelDBWrapper(ph, (1 << 10), false, false, false);
char key = 'k';
uint256 in = GetRandHash();
uint256 res;

BOOST_CHECK(dbw->Write(key, in));
BOOST_CHECK(dbw->Read(key, res));
BOOST_CHECK_EQUAL(res.ToString(), in.ToString());

// Call the destructor to free leveldb LOCK
delete dbw;

// Now, set up another wrapper that wants to obfuscate the same directory
CLevelDBWrapper odbw(ph, (1 << 10), false, false, true);

// Check that the key/val we wrote with unobfuscated wrapper exists and
// is readable.
uint256 res2;
BOOST_CHECK(odbw.Read(key, res2));
BOOST_CHECK_EQUAL(res2.ToString(), in.ToString());

BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data
BOOST_CHECK(is_null_key(odbw.GetObfuscateKey())); // The key should be an empty string

uint256 in2 = GetRandHash();
uint256 res3;

// Check that we can write successfully
BOOST_CHECK(odbw.Write(key, in2));
BOOST_CHECK(odbw.Read(key, res3));
BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString());
}

// Ensure that we start obfuscating during a reindex.
BOOST_AUTO_TEST_CASE(existing_data_reindex)
{
// We're going to share this path between two wrappers
path ph = temp_directory_path() / unique_path();
create_directories(ph);

// Set up a non-obfuscated wrapper to write some initial data.
CLevelDBWrapper* dbw = new CLevelDBWrapper(ph, (1 << 10), false, false, false);
char key = 'k';
uint256 in = GetRandHash();
uint256 res;

BOOST_CHECK(dbw->Write(key, in));
BOOST_CHECK(dbw->Read(key, res));
BOOST_CHECK_EQUAL(res.ToString(), in.ToString());

// Call the destructor to free leveldb LOCK
delete dbw;

// Simulate a -reindex by wiping the existing data store
CLevelDBWrapper odbw(ph, (1 << 10), false, true, true);

// Check that the key/val we wrote with unobfuscated wrapper doesn't exist
uint256 res2;
BOOST_CHECK(!odbw.Read(key, res2));
BOOST_CHECK(!is_null_key(odbw.GetObfuscateKey()));

uint256 in2 = GetRandHash();
uint256 res3;

// Check that we can write successfully
BOOST_CHECK(odbw.Write(key, in2));
BOOST_CHECK(odbw.Read(key, res3));
BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString());
}

BOOST_AUTO_TEST_SUITE_END()
Loading