Skip to content

Commit d5c4af2

Browse files
kdemekdeme
authored
Portal: rework db paths and general architecture (#3755)
1 parent 552755a commit d5c4af2

File tree

12 files changed

+160
-71
lines changed

12 files changed

+160
-71
lines changed

‎portal/client/nimbus_portal_client.nim‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,13 +165,14 @@ proc run(portalClient: PortalClient, config: PortalConf) {.raises: [CatchableErr
165165
d.open()
166166

167167
## Force pruning - optional
168+
## Forced on history network database only currently
168169
if config.forcePrune:
169170
let db = ContentDB.new(
170-
dataDir / config.network.getDbDirectory() / "contentdb_" &
171-
d.localNode.id.toBytesBE().toOpenArray(0, 8).toHex(),
171+
dataDir / dbDir,
172172
storageCapacity = config.storageCapacityMB * 1_000_000,
173173
radiusConfig = config.radiusConfig,
174174
localId = d.localNode.id,
175+
subnetwork = PortalSubnetwork.history,
175176
manualCheckpoint = true,
176177
)
177178

‎portal/database/content_db.nim‎

Lines changed: 38 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
{.push raises: [].}
99

1010
import
11+
std/os,
1112
chronicles,
1213
metrics,
1314
stint,
@@ -19,18 +20,27 @@ import
1920

2021
export kvstore_sqlite3, portal_protocol_config
2122

22-
# This version of content db is the most basic, simple solution where data is
23-
# stored no matter what content type or content network in the same kvstore with
24-
# the content id as key. The content id is derived from the content key, and the
25-
# deriviation is different depending on the content type. As we use content id,
26-
# this part is currently out of the scope / API of the ContentDB.
27-
# In the future it is likely that that either:
28-
# 1. More kvstores are added per network, and thus depending on the network a
29-
# different kvstore needs to be selected.
30-
# 2. Or more kvstores are added per network and per content type, and thus
31-
# content key fields are required to access the data.
32-
# 3. Or databases are created per network (and kvstores pre content type) and
33-
# thus depending on the network the right db needs to be selected.
23+
# ContentDB stores all content types in a single kvstore, using the content id
24+
# as the key. The content id is derived from the content key, with derivation
25+
# logic varying by content type. ContentDB does not handle content key
26+
# derivation; it only stores and retrieves by content id.
27+
#
28+
# Each Portal subnetwork should use a separate ContentDB instance, resulting in
29+
# a distinct sqlite database file per subnetwork. This design was chosen for
30+
# several reasons:
31+
# - Storing all subnetworks in a single ContentDB would complicate queries and
32+
# slow down access, as unrelated content would be mixed and require more
33+
# complex key logic or additional columns.
34+
# - Using multiple tables (kvstores) within a single database would make manual
35+
# deletion of a specific subnetwork's data more difficult, which is useful for
36+
# experimental or disabled networks.
37+
#
38+
# Note:
39+
# Currently, only one ContentDB instance is used for the history subnetwork.
40+
# When additional subnetworks that use ContentDB are supported, a global manager
41+
# (e.g., ContentDBManager or CommonRadiusManager) will be needed to coordinate
42+
# radius, storage capacity, and pruning across all ContentDBs.
43+
#
3444

3545
declareCounter portal_pruning_counter,
3646
"Number of pruning events which occured during the node's uptime",
@@ -50,6 +60,7 @@ type
5060
storageCapacity*: uint64
5161
dataRadius*: UInt256
5262
localId: NodeId
63+
subnetwork: PortalSubnetwork
5364
sizeStmt: SqliteStmt[NoParams, int64]
5465
unusedSizeStmt: SqliteStmt[NoParams, int64]
5566
vacuumStmt: SqliteStmt[NoParams, void]
@@ -187,18 +198,23 @@ proc new*(
187198
storageCapacity: uint64,
188199
radiusConfig: RadiusConfig,
189200
localId: NodeId,
201+
subnetwork: PortalSubnetwork,
190202
inMemory = false,
191203
manualCheckpoint = false,
192204
): ContentDB =
193205
doAssert(storageCapacity <= uint64(int64.high))
194-
195-
let db =
196-
if inMemory:
197-
SqStoreRef.init("", "contentdb-test", inMemory = true).expect(
198-
"working database (out of memory?)"
199-
)
200-
else:
201-
SqStoreRef.init(path, "contentdb", manualCheckpoint = false).expectDb()
206+
let
207+
subnetworkName = subnetwork.symbolName()
208+
fullPath = path / "contentdb-" & subnetworkName
209+
db =
210+
if inMemory:
211+
SqStoreRef.init("", "contentdb_test_" & subnetworkName, inMemory = true).expect(
212+
"working database (out of memory?)"
213+
)
214+
else:
215+
SqStoreRef
216+
.init(fullPath, "contentdb_" & subnetworkName, manualCheckpoint = false)
217+
.expectDb()
202218

203219
db.createCustomFunction("xorDistance", 2, xorDistance).expect(
204220
"Custom function xorDistance creation OK"
@@ -244,6 +260,7 @@ proc new*(
244260
manualCheckpoint: manualCheckpoint,
245261
storageCapacity: storageCapacity,
246262
localId: localId,
263+
subnetwork: subnetwork,
247264
sizeStmt: sizeStmt,
248265
unusedSizeStmt: unusedSizeStmt,
249266
vacuumStmt: vacuumStmt,
@@ -258,6 +275,7 @@ proc new*(
258275

259276
proc close*(db: ContentDB) =
260277
discard db.kv.close()
278+
db.backend.close()
261279

262280
## Private ContentDB calls
263281

‎portal/network/beacon/beacon_db.nim‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
{.push raises: [].}
99

1010
import
11+
std/os,
1112
chronicles,
1213
metrics,
1314
eth/db/kvstore,
@@ -57,6 +58,7 @@ type
5758
forkDigests: ForkDigests
5859
cfg*: RuntimeConfig
5960
beaconDbCache*: BeaconDbCache
61+
sizeStmt: SqliteStmt[NoParams, int64]
6062

6163
BeaconDbCache* = ref object
6264
finalityUpdateCache*: Opt[LightClientFinalityUpdateCache]
@@ -317,6 +319,13 @@ proc initHistoricalSummariesStore(
317319
keepFromStmt: keepFromStmt,
318320
)
319321

322+
proc size*(db: BeaconDb): int64 =
323+
var size: int64 = 0
324+
discard (
325+
db.sizeStmt.exec do(res: int64):
326+
size = res).expectDb()
327+
return size
328+
320329
func close(store: var BestLightClientUpdateStore) =
321330
store.getStmt.disposeSafe()
322331
store.getBulkStmt.disposeSafe()
@@ -340,20 +349,26 @@ proc new*(
340349
T: type BeaconDb, networkData: NetworkInitData, path: string, inMemory = false
341350
): BeaconDb =
342351
let
352+
fullPath = path / "beacondb"
343353
db =
344354
if inMemory:
345-
SqStoreRef.init("", "lc-test", inMemory = true).expect(
355+
SqStoreRef.init("", "beacondb_test", inMemory = true).expect(
346356
"working database (out of memory?)"
347357
)
348358
else:
349-
SqStoreRef.init(path, "lc").expectDb()
359+
SqStoreRef.init(fullPath, "beacondb").expectDb()
350360

351361
kvStore = kvStore db.openKvStore().expectDb()
352362
bootstraps = initBootstrapStore(db, "lc_bootstraps").expectDb()
353363
bestUpdates = initBestUpdateStore(db, "lc_best_updates").expectDb()
354364
historicalSummaries =
355365
initHistoricalSummariesStore(db, "beacon_historical_summaries").expectDb()
356366

367+
sizeStmt = db.prepareStmt(
368+
"SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size();",
369+
NoParams, int64,
370+
)[]
371+
357372
BeaconDb(
358373
backend: db,
359374
kv: kvStore,
@@ -364,6 +379,7 @@ proc new*(
364379
cfg: networkData.metadata.cfg,
365380
forkDigests: (newClone networkData.forks)[],
366381
beaconDbCache: BeaconDbCache(),
382+
sizeStmt: sizeStmt,
367383
)
368384

369385
proc close*(db: BeaconDb) =

‎portal/network/beacon/beacon_network.nim‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -481,6 +481,7 @@ proc statusLogLoop(n: BeaconNetwork) {.async: (raises: []).} =
481481
await sleepAsync(60.seconds)
482482

483483
info "Beacon network status",
484+
dbSize = $(n.beaconDb.size() div 1_000_000) & "mb",
484485
routingTableNodes = n.portalProtocol.routingTable.len()
485486
except CancelledError:
486487
trace "statusLogLoop canceled"

‎portal/network/history/history_network.nim‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,7 @@ proc statusLogLoop(n: HistoryNetwork) {.async: (raises: []).} =
237237
await sleepAsync(60.seconds)
238238

239239
info "History network status",
240+
dbSize = $(n.contentDB.size() div 1_000_000) & "mb",
240241
routingTableNodes = n.portalProtocol.routingTable.len()
241242
except CancelledError:
242243
trace "statusLogLoop canceled"
@@ -263,5 +264,7 @@ proc stop*(n: HistoryNetwork) {.async: (raises: []).} =
263264
futures.add(n.statusLogLoop.cancelAndWait())
264265
await noCancel(allFutures(futures))
265266

267+
n.contentDB.close()
268+
266269
n.processContentLoops.setLen(0)
267270
n.statusLogLoop = nil

‎portal/network/portal_node.nim‎

Lines changed: 23 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import
1212
chronos,
1313
eth/p2p/discoveryv5/protocol,
1414
beacon_chain/spec/forks,
15-
stew/byteutils,
1615
../eth_history/history_data_ssz_e2s,
1716
../database/content_db,
1817
./wire/[portal_stream, portal_protocol_config],
@@ -35,7 +34,6 @@ type
3534

3635
PortalNode* = ref object
3736
discovery: protocol.Protocol
38-
contentDB: ContentDB
3937
streamManager: StreamManager
4038
historyNetwork*: Opt[HistoryNetwork]
4139
beaconNetwork*: Opt[BeaconNetwork]
@@ -58,13 +56,7 @@ proc onOptimisticHeader(
5856
when lcDataFork > LightClientDataFork.None:
5957
info "New LC optimistic header", optimistic_header = shortLog(forkyHeader)
6058

61-
proc getDbDirectory*(network: PortalNetwork): string =
62-
if network == PortalNetwork.mainnet:
63-
"db"
64-
else:
65-
"db_" & network.symbolName()
66-
67-
const dbDir = "portaldb"
59+
const dbDir* = "portaldb"
6860

6961
proc new*(
7062
T: type PortalNode,
@@ -77,17 +69,6 @@ proc new*(
7769
rng = newRng(),
7870
): T =
7971
let
80-
# Store the database at contentdb prefixed with the first 8 chars of node id.
81-
# This is done because the content in the db is dependant on the `NodeId` and
82-
# the selected `Radius`.
83-
contentDB = ContentDB.new(
84-
config.dataDir / dbDir / "contentdb_" &
85-
discovery.localNode.id.toBytesBE().toOpenArray(0, 8).toHex(),
86-
storageCapacity = config.storageCapacity,
87-
radiusConfig = config.portalConfig.radiusConfig,
88-
localId = discovery.localNode.id,
89-
)
90-
9172
networkData =
9273
case network
9374
of PortalNetwork.mainnet:
@@ -99,6 +80,16 @@ proc new*(
9980

10081
historyNetwork =
10182
if PortalSubnetwork.history in subnetworks:
83+
# Store the database at contentdb prefixed with the first 8 chars of node id.
84+
# This is done because the content in the db is dependant on the `NodeId` and
85+
# the selected `Radius`.
86+
let contentDB = ContentDB.new(
87+
config.dataDir / dbDir,
88+
storageCapacity = config.storageCapacity,
89+
radiusConfig = config.portalConfig.radiusConfig,
90+
localId = discovery.localNode.id,
91+
subnetwork = PortalSubnetwork.history,
92+
)
10293
Opt.some(
10394
HistoryNetwork.new(
10495
network,
@@ -119,7 +110,7 @@ proc new*(
119110
beaconNetwork =
120111
if PortalSubnetwork.beacon in subnetworks:
121112
let
122-
beaconDb = BeaconDb.new(networkData, config.dataDir / dbDir / "beacondb")
113+
beaconDb = BeaconDb.new(networkData, config.dataDir / dbDir)
123114
beaconNetwork = BeaconNetwork.new(
124115
network,
125116
discovery,
@@ -158,7 +149,6 @@ proc new*(
158149

159150
PortalNode(
160151
discovery: discovery,
161-
contentDB: contentDB,
162152
streamManager: streamManager,
163153
historyNetwork: historyNetwork,
164154
beaconNetwork: beaconNetwork,
@@ -172,16 +162,16 @@ proc statusLogLoop(n: PortalNode) {.async: (raises: []).} =
172162
# drop a lot when using the logbase2 scale, namely `/ 2` per 1 logaritmic
173163
# radius drop.
174164
# TODO: Get some float precision calculus?
175-
let
176-
radius = n.contentDB.dataRadius
177-
radiusPercentage = radius div (UInt256.high() div u256(100))
178-
logRadius = logDistance(radius, u256(0))
165+
if n.historyNetwork.isSome():
166+
let
167+
radius = n.historyNetwork.value.contentDB.dataRadius
168+
radiusPercentage = radius div (UInt256.high() div u256(100))
169+
logRadius = logDistance(radius, u256(0))
179170

180-
info "Portal node status",
181-
dbSize = $(n.contentDB.size() div 1_000_000) & "mb",
182-
radiusPercentage = radiusPercentage.toString(10) & "%",
183-
radius = radius.toHex(),
184-
logRadius
171+
info "Portal node status",
172+
radiusPercentage = radiusPercentage.toString(10) & "%",
173+
radius = radius.toHex(),
174+
logRadius
185175

186176
await sleepAsync(60.seconds)
187177
except CancelledError:
@@ -206,17 +196,16 @@ proc stop*(n: PortalNode) {.async: (raises: []).} =
206196

207197
var futures: seq[Future[void]]
208198

199+
if not n.statusLogLoop.isNil():
200+
futures.add(n.statusLogLoop.cancelAndWait())
209201
if n.historyNetwork.isSome():
210202
futures.add(n.historyNetwork.value.stop())
211203
if n.beaconNetwork.isSome():
212204
futures.add(n.beaconNetwork.value.stop())
213205
if n.beaconLightClient.isSome():
214206
futures.add(n.beaconLightClient.value.stop())
215-
if not n.statusLogLoop.isNil():
216-
futures.add(n.statusLogLoop.cancelAndWait())
217207

218208
await noCancel(allFutures(futures))
219209

220210
await n.discovery.closeWait()
221-
n.contentDB.close()
222211
n.statusLogLoop = nil

‎portal/tests/history_network_tests/history_test_helpers.nim‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ proc newHistoryNetwork*(
3636
uint32.high,
3737
RadiusConfig(kind: Static, logRadius: 256),
3838
node.localNode.id,
39+
PortalSubnetwork.history,
3940
inMemory = true,
4041
)
4142
streamManager = StreamManager.new(node)

‎portal/tests/rpc_tests/test_portal_rpc_client.nim‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,12 @@ proc newHistoryNode(rng: ref HmacDrbgContext, port: int): HistoryNode =
3131
let
3232
node = initDiscoveryNode(rng, PrivateKey.random(rng[]), localAddress(port))
3333
db = ContentDB.new(
34-
"", uint32.high, RadiusConfig(kind: Dynamic), node.localNode.id, inMemory = true
34+
"",
35+
uint32.high,
36+
RadiusConfig(kind: Dynamic),
37+
node.localNode.id,
38+
PortalSubnetwork.history,
39+
inMemory = true,
3540
)
3641
streamManager = StreamManager.new(node)
3742
historyNetwork = HistoryNetwork.new(PortalNetwork.mainnet, node, db, streamManager)

0 commit comments

Comments
 (0)