Skip to content

Commit 4790f3c

Browse files
sipasipa
committed
Add address-based index
1) Maintain a salt to perturbate the address index (protection against collisions). 2) Add support for address index entries in the block index, and maintain those if -addrindex is specified. It indexes the use of every >8-byte data push in output script or consumed script - or in case of no such push, the entire script. 3) Add a searchrawtransactions RPC call, which can look up raw transactions by address.
1 parent fd867c7 commit 4790f3c

File tree

8 files changed

+259
-17
lines changed

8 files changed

+259
-17
lines changed

‎src/bitcoinrpc.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,7 @@ static const CRPCCommand vRPCCommands[] =
249249
{ "importwallet", &importwallet, false, false },
250250
{ "listunspent", &listunspent, false, false },
251251
{ "getrawtransaction", &getrawtransaction, false, false },
252+
{ "searchrawtransactions", &searchrawtransactions, false, false },
252253
{ "createrawtransaction", &createrawtransaction, false, false },
253254
{ "decoderawtransaction", &decoderawtransaction, false, false },
254255
{ "signrawtransaction", &signrawtransaction, false, false },
@@ -1189,6 +1190,9 @@ Array RPCConvertValues(const std::string &strMethod, const std::vector<std::stri
11891190
if (strMethod == "listunspent" && n > 2) ConvertTo<Array>(params[2]);
11901191
if (strMethod == "getblock" && n > 1) ConvertTo<bool>(params[1]);
11911192
if (strMethod == "getrawtransaction" && n > 1) ConvertTo<boost::int64_t>(params[1]);
1193+
if (strMethod == "searchrawtransactions" && n > 1) ConvertTo<boost::int64_t>(params[1]);
1194+
if (strMethod == "searchrawtransactions" && n > 2) ConvertTo<boost::int64_t>(params[2]);
1195+
if (strMethod == "searchrawtransactions" && n > 3) ConvertTo<boost::int64_t>(params[3]);
11921196
if (strMethod == "createrawtransaction" && n > 0) ConvertTo<Array>(params[0]);
11931197
if (strMethod == "createrawtransaction" && n > 1) ConvertTo<Object>(params[1]);
11941198
if (strMethod == "signrawtransaction" && n > 1) ConvertTo<Array>(params[1], true);

‎src/bitcoinrpc.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,7 @@ extern json_spirit::Value validateaddress(const json_spirit::Array& params, bool
192192
extern json_spirit::Value getinfo(const json_spirit::Array& params, bool fHelp);
193193

194194
extern json_spirit::Value getrawtransaction(const json_spirit::Array& params, bool fHelp); // in rcprawtransaction.cpp
195+
extern json_spirit::Value searchrawtransactions(const json_spirit::Array& params, bool fHelp);
195196
extern json_spirit::Value listunspent(const json_spirit::Array& params, bool fHelp);
196197
extern json_spirit::Value lockunspent(const json_spirit::Array& params, bool fHelp);
197198
extern json_spirit::Value listlockunspent(const json_spirit::Array& params, bool fHelp);

‎src/init.cpp‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -710,7 +710,7 @@ bool AppInit2(boost::thread_group& threadGroup)
710710
if (nTotalCache < (1 << 22))
711711
nTotalCache = (1 << 22); // total cache cannot be less than 4 MiB
712712
size_t nBlockTreeDBCache = nTotalCache / 8;
713-
if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", false))
713+
if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", false) && !GetBoolArg("-addrindex", false))
714714
nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB
715715
nTotalCache -= nBlockTreeDBCache;
716716
size_t nCoinDBCache = nTotalCache / 2; // use half of the remaining cache for coindb cache
@@ -761,6 +761,12 @@ bool AppInit2(boost::thread_group& threadGroup)
761761
break;
762762
}
763763

764+
// Check for changed -addrindex state
765+
if (fAddrIndex != GetBoolArg("-addrindex", false)) {
766+
strLoadError = _("You need to rebuild the database using -reindex to change -addrindex");
767+
break;
768+
}
769+
764770
uiInterface.InitMessage(_("Verifying blocks..."));
765771
if (!VerifyDB(GetArg("-checklevel", 3),
766772
GetArg( "-checkblocks", 288))) {

‎src/main.cpp‎

Lines changed: 95 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ bool fImporting = false;
4646
bool fReindex = false;
4747
bool fBenchmark = false;
4848
bool fTxIndex = false;
49+
bool fAddrIndex = false;
4950
unsigned int nCoinCacheSize = 5000;
5051
bool fHaveGUI = false;
5152

@@ -1095,6 +1096,42 @@ bool CWalletTx::AcceptWalletTransaction()
10951096
return false;
10961097
}
10971098

1099+
bool ReadTransaction(CTransaction& tx, const CDiskTxPos &pos, uint256 &hashBlock) {
1100+
CAutoFile file(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION);
1101+
CBlockHeader header;
1102+
try {
1103+
file >> header;
1104+
fseek(file, pos.nTxOffset, SEEK_CUR);
1105+
file >> tx;
1106+
} catch (std::exception &e) {
1107+
return error("%s() : deserialize or I/O error", __PRETTY_FUNCTION__);
1108+
}
1109+
hashBlock = header.GetHash();
1110+
return true;
1111+
}
1112+
1113+
bool FindTransactionsByDestination(const CTxDestination &dest, std::set<CExtDiskTxPos> &setpos) {
1114+
uint160 addrid = 0;
1115+
const CKeyID *pkeyid = boost::get<CKeyID>(&dest);
1116+
if (pkeyid)
1117+
addrid = static_cast<uint160>(*pkeyid);
1118+
if (!addrid) {
1119+
const CScriptID *pscriptid = boost::get<CScriptID>(&dest);
1120+
if (pscriptid)
1121+
addrid = static_cast<uint160>(*pscriptid);
1122+
}
1123+
if (!addrid)
1124+
return false;
1125+
1126+
LOCK(cs_main);
1127+
if (!fAddrIndex)
1128+
return false;
1129+
std::vector<CExtDiskTxPos> vPos;
1130+
if (!pblocktree->ReadAddrIndex(addrid, vPos))
1131+
return false;
1132+
setpos.insert(vPos.begin(), vPos.end());
1133+
return true;
1134+
}
10981135

10991136
// Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock
11001137
bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock, bool fAllowSlow)
@@ -1114,16 +1151,8 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock
11141151
if (fTxIndex) {
11151152
CDiskTxPos postx;
11161153
if (pblocktree->ReadTxIndex(hash, postx)) {
1117-
CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION);
1118-
CBlockHeader header;
1119-
try {
1120-
file >> header;
1121-
fseek(file, postx.nTxOffset, SEEK_CUR);
1122-
file >> txOut;
1123-
} catch (std::exception &e) {
1124-
return error("%s() : deserialize or I/O error", __PRETTY_FUNCTION__);
1125-
}
1126-
hashBlock = header.GetHash();
1154+
if (!ReadTransaction(txOut, postx, hashBlock))
1155+
return false;
11271156
if (txOut.GetHash() != hash)
11281157
return error("%s() : txid mismatch", __PRETTY_FUNCTION__);
11291158
return true;
@@ -1748,6 +1777,32 @@ void ThreadScriptCheck() {
17481777
scriptcheckqueue.Thread();
17491778
}
17501779

1780+
// Index either: a) every data push >=8 bytes, b) if no such pushes, the entire script
1781+
void static BuildAddrIndex(const CScript &script, const CExtDiskTxPos &pos, std::vector<std::pair<uint160, CExtDiskTxPos> > &out) {
1782+
CScript::const_iterator pc = script.begin();
1783+
CScript::const_iterator pend = script.end();
1784+
std::vector<unsigned char> data;
1785+
opcodetype opcode;
1786+
bool fHaveData = false;
1787+
while (pc < pend) {
1788+
script.GetOp(pc, opcode, data);
1789+
if (0 <= opcode && opcode <= OP_PUSHDATA4 && data.size() >= 8) { // data element
1790+
uint160 addrid = 0;
1791+
if (data.size() <= 20) {
1792+
memcpy(&addrid, &data[0], data.size());
1793+
} else {
1794+
addrid = Hash160(data);
1795+
}
1796+
out.push_back(std::make_pair(addrid, pos));
1797+
fHaveData = true;
1798+
}
1799+
}
1800+
if (!fHaveData) {
1801+
uint160 addrid = Hash160(script);
1802+
out.push_back(std::make_pair(addrid, pos));
1803+
}
1804+
}
1805+
17511806
bool ConnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, bool fJustCheck)
17521807
{
17531808
// Check it again in case a previous version let a bad block in
@@ -1805,10 +1860,14 @@ bool ConnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, C
18051860
int64 nFees = 0;
18061861
int nInputs = 0;
18071862
unsigned int nSigOps = 0;
1808-
CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size()));
1809-
std::vector<std::pair<uint256, CDiskTxPos> > vPos;
1810-
vPos.reserve(block.vtx.size());
1811-
for (unsigned int i = 0; i < block.vtx.size(); i++)
1863+
CExtDiskTxPos pos(CDiskTxPos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())), pindex->nHeight);
1864+
std::vector<std::pair<uint256, CDiskTxPos> > vPosTxid;
1865+
std::vector<std::pair<uint160, CExtDiskTxPos> > vPosAddrid;
1866+
if (fTxIndex)
1867+
vPosTxid.reserve(block.vtx.size());
1868+
if (fAddrIndex)
1869+
vPosAddrid.reserve(4*block.vtx.size());
1870+
for (unsigned int i=0; i<block.vtx.size(); i++)
18121871
{
18131872
const CTransaction &tx = block.vtx[i];
18141873

@@ -1840,12 +1899,24 @@ bool ConnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, C
18401899
control.Add(vChecks);
18411900
}
18421901

1902+
if (fTxIndex)
1903+
vPosTxid.push_back(std::make_pair(block.GetTxHash(i), pos));
1904+
if (fAddrIndex) {
1905+
if (!tx.IsCoinBase()) {
1906+
BOOST_FOREACH(const CTxIn &txin, tx.vin) {
1907+
const CCoins &coins = view.GetCoins(txin.prevout.hash);
1908+
BuildAddrIndex(coins.vout[txin.prevout.n].scriptPubKey, pos, vPosAddrid);
1909+
}
1910+
}
1911+
BOOST_FOREACH(const CTxOut &txout, tx.vout)
1912+
BuildAddrIndex(txout.scriptPubKey, pos, vPosAddrid);
1913+
}
1914+
18431915
CTxUndo txundo;
18441916
UpdateCoins(tx, state, view, txundo, pindex->nHeight, block.GetTxHash(i));
18451917
if (!tx.IsCoinBase())
18461918
blockundo.vtxundo.push_back(txundo);
18471919

1848-
vPos.push_back(std::make_pair(block.GetTxHash(i), pos));
18491920
pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION);
18501921
}
18511922
int64 nTime = GetTimeMicros() - nStart;
@@ -1887,8 +1958,11 @@ bool ConnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, C
18871958
}
18881959

18891960
if (fTxIndex)
1890-
if (!pblocktree->WriteTxIndex(vPos))
1961+
if (!pblocktree->WriteTxIndex(vPosTxid))
18911962
return state.Abort(_("Failed to write transaction index"));
1963+
if (fAddrIndex)
1964+
if (!pblocktree->AddAddrIndex(vPosAddrid))
1965+
return state.Abort(_("Failed to write address index"));
18921966

18931967
// add this block to the view's block chain
18941968
assert(view.SetBestBlock(pindex));
@@ -2745,6 +2819,9 @@ bool static LoadBlockIndexDB()
27452819
pblocktree->ReadFlag("txindex", fTxIndex);
27462820
printf("LoadBlockIndexDB(): transaction index %s\n", fTxIndex ? "enabled" : "disabled");
27472821

2822+
pblocktree->ReadFlag("addrindex", fAddrIndex);
2823+
printf("LoadBlockIndexDB(): address index %s\n", fAddrIndex ? "enabled" : "disabled");
2824+
27482825
// Load hashBestChain pointer to end of best chain
27492826
pindexBest = pcoinsTip->GetBestBlock();
27502827
if (pindexBest == NULL)
@@ -2869,6 +2946,8 @@ bool InitBlockIndex() {
28692946
// Use the provided setting for -txindex in the new database
28702947
fTxIndex = GetBoolArg("-txindex", false);
28712948
pblocktree->WriteFlag("txindex", fTxIndex);
2949+
fAddrIndex = GetBoolArg("-addrindex", false);
2950+
pblocktree->WriteFlag("addrindex", fAddrIndex);
28722951
printf("Initializing databases...\n");
28732952

28742953
// Only add the genesis block if not reindexing (in which case we reuse the one already on disk)

‎src/main.h‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ extern bool fReindex;
9292
extern bool fBenchmark;
9393
extern int nScriptCheckThreads;
9494
extern bool fTxIndex;
95+
extern bool fAddrIndex;
9596
extern unsigned int nCoinCacheSize;
9697
extern bool fHaveGUI;
9798

@@ -258,9 +259,57 @@ struct CDiskTxPos : public CDiskBlockPos
258259
CDiskBlockPos::SetNull();
259260
nTxOffset = 0;
260261
}
262+
263+
friend bool operator==(const CDiskTxPos &a, const CDiskTxPos &b) {
264+
return (a.nFile == b.nFile && a.nPos == b.nPos && a.nTxOffset == b.nTxOffset);
265+
}
266+
267+
friend bool operator!=(const CDiskTxPos &a, const CDiskTxPos &b) {
268+
return !(a == b);
269+
}
270+
271+
friend bool operator<(const CDiskTxPos &a, const CDiskTxPos &b) {
272+
return (a.nFile < b.nFile || (
273+
(a.nFile == b.nFile) && (a.nPos < b.nPos || (
274+
(a.nPos == b.nPos) && (a.nTxOffset < b.nTxOffset)))));
275+
}
261276
};
262277

278+
struct CExtDiskTxPos : public CDiskTxPos
279+
{
280+
unsigned int nHeight;
281+
282+
IMPLEMENT_SERIALIZE(
283+
READWRITE(*(CDiskTxPos*)this);
284+
READWRITE(VARINT(nHeight));
285+
)
286+
287+
CExtDiskTxPos(const CDiskTxPos &pos, int nHeightIn) : CDiskTxPos(pos), nHeight(nHeightIn) {
288+
}
289+
290+
CExtDiskTxPos() {
291+
SetNull();
292+
}
293+
294+
void SetNull() {
295+
CDiskTxPos::SetNull();
296+
nHeight = 0;
297+
}
298+
299+
friend bool operator==(const CExtDiskTxPos &a, const CExtDiskTxPos &b) {
300+
return (a.nHeight == b.nHeight && a.nFile == b.nFile && a.nPos == b.nPos && a.nTxOffset == b.nTxOffset);
301+
}
263302

303+
friend bool operator!=(const CExtDiskTxPos &a, const CExtDiskTxPos &b) {
304+
return !(a == b);
305+
}
306+
307+
friend bool operator<(const CExtDiskTxPos &a, const CExtDiskTxPos &b) {
308+
if (a.nHeight < b.nHeight) return true;
309+
if (a.nHeight > b.nHeight) return false;
310+
return ((const CDiskTxPos)a < (const CDiskTxPos)b);
311+
}
312+
};
264313

265314
enum GetMinFee_mode
266315
{
@@ -586,8 +635,10 @@ class CPartialMerkleTree
586635

587636
/** Functions for disk access for blocks */
588637
bool WriteBlockToDisk(CBlock& block, CDiskBlockPos& pos);
638+
bool ReadTransaction(CTransaction& tx, const CDiskTxPos &pos, uint256 &hashBlock);
589639
bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos);
590640
bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex);
641+
bool FindTransactionsByDestination(const CTxDestination &dest, std::set<CExtDiskTxPos> &setpos);
591642

592643

593644
/** Functions for validating blocks and updating the block tree */

‎src/rpcrawtransaction.cpp‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,66 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry)
131131
}
132132
}
133133

134+
Value searchrawtransactions(const Array &params, bool fHelp)
135+
{
136+
if (fHelp || params.size() < 1 || params.size() > 4)
137+
throw runtime_error(
138+
"searchrawtransactions <address> [verbose=1] [skip=0] [count=100]\n");
139+
140+
if (!fAddrIndex)
141+
throw JSONRPCError(RPC_MISC_ERROR, "Address index not enabled");
142+
143+
CBitcoinAddress address(params[0].get_str());
144+
if (!address.IsValid())
145+
throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address");
146+
CTxDestination dest = address.Get();
147+
148+
std::set<CExtDiskTxPos> setpos;
149+
if (!FindTransactionsByDestination(dest, setpos))
150+
throw JSONRPCError(RPC_DATABASE_ERROR, "Cannot search for address");
151+
152+
int nSkip = 0;
153+
int nCount = 100;
154+
bool fVerbose = true;
155+
if (params.size() > 1)
156+
fVerbose = (params[1].get_int() != 0);
157+
if (params.size() > 2)
158+
nSkip = params[2].get_int();
159+
if (params.size() > 3)
160+
nCount = params[3].get_int();
161+
162+
if (nSkip < 0)
163+
nSkip += setpos.size();
164+
if (nSkip < 0)
165+
nSkip = 0;
166+
if (nCount < 0)
167+
nCount = 0;
168+
169+
std::set<CExtDiskTxPos>::const_iterator it = setpos.begin();
170+
while (it != setpos.end() && nSkip--) it++;
171+
172+
Array result;
173+
while (it != setpos.end() && nCount--) {
174+
CTransaction tx;
175+
uint256 hashBlock;
176+
if (!ReadTransaction(tx, *it, hashBlock))
177+
throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Cannot read transaction from disk");
178+
CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION);
179+
ssTx << tx;
180+
string strHex = HexStr(ssTx.begin(), ssTx.end());
181+
if (fVerbose) {
182+
Object object;
183+
TxToJSON(tx, hashBlock, object);
184+
object.push_back(Pair("hex", strHex));
185+
result.push_back(object);
186+
} else {
187+
result.push_back(strHex);
188+
}
189+
it++;
190+
}
191+
return result;
192+
}
193+
134194
Value getrawtransaction(const Array& params, bool fHelp)
135195
{
136196
if (fHelp || params.size() < 1 || params.size() > 2)

0 commit comments

Comments
 (0)