Skip to content

Commit 0713529

Browse files
romanzhodlinatorl0rinc
committed
rest: allow reading partial block data from storage
It will allow fetching specific transactions using an external index, following bitcoin#32541 (comment). Co-authored-by: Hodlinator <[email protected]> Co-authored-by: Lőrinc <[email protected]>
1 parent 4e2af1c commit 0713529

File tree

4 files changed

+106
-16
lines changed

4 files changed

+106
-16
lines changed

‎doc/REST-interface.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,11 @@ The HTTP request and response are both handled entirely in-memory.
4747

4848
With the /notxdetails/ option JSON response will only contain the transaction hash instead of the complete transaction details. The option only affects the JSON response.
4949

50+
- `GET /rest/blockpart/<BLOCK-HASH>.<bin|hex>?offset=<OFFSET>&size=<SIZE>`
51+
52+
Given a block hash: returns a block part, in binary or hex-encoded binary formats.
53+
Responds with 404 if the block or the byte range doesn't exist.
54+
5055
#### Blockheaders
5156
`GET /rest/headers/<BLOCK-HASH>.<bin|hex|json>?count=<COUNT=5>`
5257

‎doc/release-notes-33657.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
New REST API
2+
------------
3+
4+
- A new REST API endpoint (`/rest/blockpart/BLOCKHASH.bin?offset=X&size=Y`) has been introduced
5+
for efficiently fetching a range of bytes from block `BLOCKHASH`.

‎src/rest.cpp‎

Lines changed: 44 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -379,10 +379,17 @@ static bool rest_spent_txouts(const std::any& context, HTTPRequest* req, const s
379379
}
380380
}
381381

382+
/**
383+
* This handler is used by multiple HTTP endpoints:
384+
* - `/block/` via `rest_block_extended()`
385+
* - `/block/notxdetails/` via `rest_block_notxdetails()`
386+
* - `/blockpart/` via `rest_block_part()` (doesn't support JSON response, so `tx_verbosity` is unset)
387+
*/
382388
static bool rest_block(const std::any& context,
383389
HTTPRequest* req,
384390
const std::string& uri_part,
385-
TxVerbosity tx_verbosity)
391+
std::optional<TxVerbosity> tx_verbosity,
392+
std::optional<std::pair<size_t, size_t>> block_part = std::nullopt)
386393
{
387394
if (!CheckWarmup(req))
388395
return false;
@@ -416,12 +423,14 @@ static bool rest_block(const std::any& context,
416423
pos = pblockindex->GetBlockPos();
417424
}
418425

419-
const auto block_data{chainman.m_blockman.ReadRawBlock(pos)};
426+
const auto block_data{chainman.m_blockman.ReadRawBlock(pos, block_part)};
420427
if (!block_data) {
421428
switch (block_data.error()) {
422429
case node::ReadRawError::IO: return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "I/O error reading " + hashStr);
423-
case node::ReadRawError::BadPartRange: break; // can happen only when reading a block part
424-
}
430+
case node::ReadRawError::BadPartRange:
431+
assert(block_part);
432+
return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Bad block part offset/size %d/%d for %s", block_part->first, block_part->second, hashStr));
433+
} // no default case, so the compiler can warn about missing cases
425434
assert(false);
426435
}
427436

@@ -440,14 +449,17 @@ static bool rest_block(const std::any& context,
440449
}
441450

442451
case RESTResponseFormat::JSON: {
443-
CBlock block{};
444-
DataStream block_stream{*block_data};
445-
block_stream >> TX_WITH_WITNESS(block);
446-
UniValue objBlock = blockToJSON(chainman.m_blockman, block, *tip, *pblockindex, tx_verbosity, chainman.GetConsensus().powLimit);
447-
std::string strJSON = objBlock.write() + "\n";
448-
req->WriteHeader("Content-Type", "application/json");
449-
req->WriteReply(HTTP_OK, strJSON);
450-
return true;
452+
if (tx_verbosity) {
453+
CBlock block{};
454+
DataStream block_stream{*block_data};
455+
block_stream >> TX_WITH_WITNESS(block);
456+
UniValue objBlock = blockToJSON(chainman.m_blockman, block, *tip, *pblockindex, *tx_verbosity, chainman.GetConsensus().powLimit);
457+
std::string strJSON = objBlock.write() + "\n";
458+
req->WriteHeader("Content-Type", "application/json");
459+
req->WriteReply(HTTP_OK, strJSON);
460+
return true;
461+
}
462+
return RESTERR(req, HTTP_BAD_REQUEST, "JSON output is not supported for this request type");
451463
}
452464

453465
default: {
@@ -466,6 +478,25 @@ static bool rest_block_notxdetails(const std::any& context, HTTPRequest* req, co
466478
return rest_block(context, req, uri_part, TxVerbosity::SHOW_TXID);
467479
}
468480

481+
static bool rest_block_part(const std::any& context, HTTPRequest* req, const std::string& uri_part)
482+
{
483+
try {
484+
if (const auto opt_offset{ToIntegral<size_t>(req->GetQueryParameter("offset").value_or(""))}) {
485+
if (const auto opt_size{ToIntegral<size_t>(req->GetQueryParameter("size").value_or(""))}) {
486+
return rest_block(context, req, uri_part,
487+
/*tx_verbosity=*/std::nullopt,
488+
/*block_part=*/{{*opt_offset, *opt_size}});
489+
} else {
490+
return RESTERR(req, HTTP_BAD_REQUEST, "Block part size missing or invalid");
491+
}
492+
} else {
493+
return RESTERR(req, HTTP_BAD_REQUEST, "Block part offset missing or invalid");
494+
}
495+
} catch (const std::runtime_error& e) {
496+
return RESTERR(req, HTTP_BAD_REQUEST, e.what());
497+
}
498+
}
499+
469500
static bool rest_filter_header(const std::any& context, HTTPRequest* req, const std::string& uri_part)
470501
{
471502
if (!CheckWarmup(req)) return false;
@@ -1114,6 +1145,7 @@ static const struct {
11141145
{"/rest/tx/", rest_tx},
11151146
{"/rest/block/notxdetails/", rest_block_notxdetails},
11161147
{"/rest/block/", rest_block_extended},
1148+
{"/rest/blockpart/", rest_block_part},
11171149
{"/rest/blockfilter/", rest_block_filter},
11181150
{"/rest/blockfilterheaders/", rest_filter_header},
11191151
{"/rest/chaininfo", rest_chaininfo},

‎test/functional/interface_rest.py‎

Lines changed: 52 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
MiniWallet,
2929
getnewdestination,
3030
)
31-
from typing import Optional
3231

3332

3433
INVALID_PARAM = "abc"
@@ -66,13 +65,16 @@ def test_rest_request(
6665
body: str = '',
6766
status: int = 200,
6867
ret_type: RetType = RetType.JSON,
69-
query_params: Optional[dict[str, typing.Any]] = None,
68+
query_params: typing.Union[dict[str, typing.Any], str, None] = None,
7069
) -> typing.Union[http.client.HTTPResponse, bytes, str, None]:
7170
rest_uri = '/rest' + uri
7271
if req_type in ReqType:
7372
rest_uri += f'.{req_type.name.lower()}'
7473
if query_params:
75-
rest_uri += f'?{urllib.parse.urlencode(query_params)}'
74+
if isinstance(query_params, str):
75+
rest_uri += f'?{query_params}'
76+
else:
77+
rest_uri += f'?{urllib.parse.urlencode(query_params)}'
7678

7779
conn = http.client.HTTPConnection(self.url.hostname, self.url.port)
7880
self.log.debug(f'{http_method} {rest_uri} {body}')
@@ -82,7 +84,7 @@ def test_rest_request(
8284
conn.request('POST', rest_uri, body)
8385
resp = conn.getresponse()
8486

85-
assert_equal(resp.status, status)
87+
assert resp.status == status, f"Expected: {status}, Got: {resp.status} - Response: {str(resp.read())}"
8688

8789
if ret_type == RetType.OBJ:
8890
return resp
@@ -455,6 +457,52 @@ def run_test(self):
455457
expected = [(p["scriptPubKey"], p["value"]) for p in prevouts]
456458
assert_equal(expected, actual)
457459

460+
self.log.info("Test the /blockpart URI")
461+
462+
blockhash = self.nodes[0].getbestblockhash()
463+
block_bin = self.test_rest_request(f"/block/{blockhash}", req_type=ReqType.BIN, ret_type=RetType.BYTES)
464+
for req_type in (ReqType.BIN, ReqType.HEX):
465+
def get_block_part(status: int = 200, **kwargs):
466+
resp = self.test_rest_request(f"/blockpart/{blockhash}", status=status,
467+
req_type=req_type, ret_type=RetType.BYTES, **kwargs)
468+
assert isinstance(resp, bytes)
469+
if req_type is ReqType.HEX and status == 200:
470+
resp = bytes.fromhex(resp.decode().strip())
471+
return resp
472+
473+
assert_equal(block_bin, get_block_part(query_params={"offset": 0, "size": len(block_bin)}))
474+
475+
assert len(block_bin) >= 500
476+
assert_equal(block_bin[20:320], get_block_part(query_params={"offset": 20, "size": 300}))
477+
assert_equal(block_bin[-5:], get_block_part(query_params={"offset": len(block_bin) - 5, "size": 5}))
478+
479+
get_block_part(status=400, query_params={"offset": 10})
480+
get_block_part(status=400, query_params={"size": 100})
481+
get_block_part(status=400, query_params={"offset": "x"})
482+
get_block_part(status=400, query_params={"size": "y"})
483+
get_block_part(status=400, query_params={"offset": "x", "size": "y"})
484+
assert get_block_part(status=400, query_params="%XY").decode("utf-8").startswith("URI parsing failed")
485+
486+
get_block_part(status=400, query_params={"offset": 0, "size": 0})
487+
get_block_part(status=400, query_params={"offset": len(block_bin), "size": 0})
488+
get_block_part(status=400, query_params={"offset": len(block_bin) + 1, "size": 1})
489+
get_block_part(status=400, query_params={"offset": len(block_bin), "size": 1})
490+
get_block_part(status=400, query_params={"offset": len(block_bin) + 1, "size": 1})
491+
get_block_part(status=400, query_params={"offset": 0, "size": len(block_bin) + 1})
492+
493+
self.test_rest_request(f"/blockpart/{blockhash}", status=400, req_type=ReqType.JSON, ret_type=RetType.OBJ)
494+
495+
self.log.info("Missing block data should cause REST API to fail")
496+
497+
self.test_rest_request(f"/block/{blockhash}", status=200, req_type=ReqType.BIN, ret_type=RetType.OBJ)
498+
self.test_rest_request(f"/blockpart/{blockhash}", query_params={"offset": 0, "size": 1}, status=200, req_type=ReqType.BIN, ret_type=RetType.OBJ)
499+
blk_files = list(self.nodes[0].blocks_path.glob("blk*.dat"))
500+
for blk_file in blk_files:
501+
blk_file.rename(blk_file.with_suffix('.bkp'))
502+
self.test_rest_request(f"/block/{blockhash}", status=500, req_type=ReqType.BIN, ret_type=RetType.OBJ)
503+
self.test_rest_request(f"/blockpart/{blockhash}", query_params={"offset": 0, "size": 1}, status=500, req_type=ReqType.BIN, ret_type=RetType.OBJ)
504+
for blk_file in blk_files:
505+
blk_file.with_suffix('.bkp').rename(blk_file)
458506

459507
self.log.info("Test the /deploymentinfo URI")
460508

0 commit comments

Comments
 (0)