Skip to content

Commit 4eacf7d

Browse files
authored
Snap sync download framework (#3912)
* Snap sync download framework details + downloads some account data and throws it away + Some peer management for slow or unusable peers * Code cosmetics
1 parent 084bc45 commit 4eacf7d

File tree

19 files changed

+1906
-57
lines changed

19 files changed

+1906
-57
lines changed

execution_chain/sync/beacon/worker/headers/headers_fetch.nim

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ template fetchHeadersReversed*(
198198
# Update download statistics
199199
let bps = buddy.hdrSampleSize(elapsed, h.getEncodedLength)
200200

201-
# Request did not fail
201+
# This request did not fail (maybe another one did): reset anyway
202202
buddy.only.failedReq.reset
203203

204204
# Ban an overly slow peer for a while when observed consecutively.

execution_chain/sync/snap.nim

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Nimbus
2-
# Copyright (c) 2023-2025 Status Research & Development GmbH
2+
# Copyright (c) 2025-2026 Status Research & Development GmbH
33
# Licensed under either of
44
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
55
# http://www.apache.org/licenses/LICENSE-2.0)
@@ -15,7 +15,8 @@ import
1515
pkg/stew/[interval_set, sorted_set],
1616
../core/chain,
1717
../networking/p2p,
18-
./snap/[snap_desc, worker, worker/classify],
18+
./snap/[snap_desc, worker],
19+
./snap/worker/helpers,
1920
./[sync_sched, wire_protocol]
2021

2122
from ./beacon
@@ -53,8 +54,7 @@ proc runPool(buddy: SnapPeerRef; last: bool; laps: int): bool =
5354
worker.runPool(buddy, last, laps, "SyncMode")
5455

5556
proc runPeer(buddy: SnapPeerRef): Future[Duration] {.async: (raises: []).} =
56-
let rank = buddy.classifyForFetching()
57-
return worker.runPeer(buddy, rank, "Peer")
57+
return worker.runPeer(buddy, "Peer")
5858

5959
# ------------------------------------------------------------------------------
6060
# Public functions
@@ -93,7 +93,11 @@ proc configTarget*(desc: SnapSyncRef; hex: string): bool =
9393
## Set up inital target root (if any, mainly for debugging)
9494
doAssert not desc.ctx.isNil
9595
try:
96-
desc.ctx.pool.initBlockHash = Hash32.fromHex(hex)
96+
var target: SnapTarget
97+
if desc.ctx.pool.target.isSome():
98+
target = desc.ctx.pool.target.value
99+
target.blockHash = BlockHash(Hash32.fromHex(hex))
100+
desc.ctx.pool.target = Opt.some(target)
97101
return true
98102
except ValueError:
99103
discard
@@ -103,7 +107,11 @@ proc configUpdateFile*(desc: SnapSyncRef; file: string): bool =
103107
## Update file containing the target
104108
doAssert not desc.ctx.isNil
105109
if 0 < file.len:
106-
desc.ctx.pool.stateUpdateFile = file
110+
var target: SnapTarget
111+
if desc.ctx.pool.target.isSome():
112+
target = desc.ctx.pool.target.value
113+
target.updateFile = file
114+
desc.ctx.pool.target = Opt.some(target)
107115
return true
108116
# false
109117

execution_chain/sync/snap/snap_desc.nim

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Nimbus
2-
# Copyright (c) 2023-2025 Status Research & Development GmbH
2+
# Copyright (c) 2025-2026 Status Research & Development GmbH
33
# Licensed under either of
44
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
55
# http://www.apache.org/licenses/LICENSE-2.0)

execution_chain/sync/snap/worker.nim

Lines changed: 49 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Nimbus
2-
# Copyright (c) 2023-2025 Status Research & Development GmbH
2+
# Copyright (c) 2025-2026 Status Research & Development GmbH
33
# Licensed under either of
44
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
55
# http://www.apache.org/licenses/LICENSE-2.0)
@@ -11,28 +11,54 @@
1111
{.push raises:[].}
1212

1313
import
14-
pkg/[chronicles, chronos],
15-
./worker/worker_desc
14+
std/os,
15+
pkg/[chronicles, chronos, minilru, results],
16+
./worker/[account, helpers, start_stop, state_db, worker_desc]
17+
18+
logScope:
19+
topics = "snap sync"
20+
21+
# ------------------------------------------------------------------------------
22+
# Private helpers
23+
# ------------------------------------------------------------------------------
1624

1725
# ------------------------------------------------------------------------------
1826
# Public start/stop and admin functions
1927
# ------------------------------------------------------------------------------
2028

2129
proc setup*(ctx: SnapCtxRef; info: static[string]): bool =
2230
## Global set up
31+
ctx.setupServices info
2332
true
2433

2534
proc release*(ctx: SnapCtxRef; info: static[string]) =
2635
## Global clean up
27-
discard
36+
ctx.destroyServices()
37+
2838

2939
proc start*(buddy: SnapPeerRef; info: static[string]): bool =
3040
## Initialise worker peer
41+
let
42+
peer {.inject,used.} = $buddy.peer # logging only
43+
ctx = buddy.ctx
44+
45+
if not ctx.pool.seenData and buddy.peerID in ctx.pool.failedPeers:
46+
debug info & ": useless peer already tried", peer
47+
return false
48+
49+
if not buddy.startSyncPeer():
50+
debug info & ": failed", peer
51+
return false
52+
53+
debug info & ": new peer", peer, nSyncPeers=ctx.nSyncPeers(),
54+
peerType=buddy.only.peerType, clientId=buddy.peer.clientId
3155
true
3256

3357
proc stop*(buddy: SnapPeerRef; info: static[string]) =
3458
## Clean up this peer
35-
discard
59+
debug info & ": release peer", peer=buddy.peer,
60+
nSyncPeers=(buddy.ctx.nSyncPeers()-1), state=($buddy.syncState)
61+
buddy.stopSyncPeer()
3662

3763
# ------------------------------------------------------------------------------
3864
# Public functions
@@ -56,8 +82,8 @@ template runDaemon*(ctx: SnapCtxRef; info: static[string]): Duration =
5682
##
5783
## The template returns a suggested idle time for after this task.
5884
##
59-
debug info & ": not implemented for snap sync", nSyncPeers=ctx.nSyncPeers()
60-
chronos.seconds(10)
85+
debug info & ": snap sync not implemented yet", nSyncPeers=ctx.nSyncPeers()
86+
chronos.seconds(30)
6187

6288
proc runPool*(
6389
buddy: SnapPeerRef;
@@ -79,13 +105,12 @@ proc runPool*(
79105
##
80106
## Note that this function does not run in `async` mode.
81107
##
82-
debug info & ": not implemented for snap sync", peer=buddy.peer,
108+
debug info & ": snap sync not implemented yet", peer=buddy.peer,
83109
nSyncPeers=buddy.ctx.nSyncPeers()
84110
true # stop
85111

86112
template runPeer*(
87113
buddy: SnapPeerRef;
88-
rank: PeerRanking;
89114
info: static[string];
90115
): Duration =
91116
## Async/template
@@ -95,9 +120,21 @@ template runPeer*(
95120
##
96121
## The template returns a suggested idle time for after this task.
97122
##
98-
debug info & ": not implemented for snap sync", peer=buddy.peer,
99-
nSyncPeers=buddy.ctx.nSyncPeers()
100-
chronos.seconds(10)
123+
var bodyRc = chronos.nanoseconds(0)
124+
block body:
125+
let
126+
ctx = buddy.ctx
127+
peer {.inject,used.} = $buddy.peer # logging only
128+
129+
buddy.accountRangeImport info
130+
131+
debug info & ": snap sync not implemented yet", peer,
132+
nSyncPeers=ctx.nSyncPeers()
133+
134+
bodyRc = chronos.seconds(10)
135+
# End block: `body`
136+
137+
bodyRc
101138

102139
# ------------------------------------------------------------------------------
103140
# End
Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
# Nimbus
2+
# Copyright (c) 2025-2026 Status Research & Development GmbH
3+
# Licensed under either of
4+
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
5+
# http://www.apache.org/licenses/LICENSE-2.0)
6+
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
7+
# http://opensource.org/licenses/MIT)
8+
# at your option. This file may not be copied, modified, or distributed
9+
# except according to those terms.
10+
11+
{.push raises: [].}
12+
13+
import
14+
pkg/[chronicles, chronos],
15+
account/account_fetch,
16+
./[helpers, header, state_db, worker_desc]
17+
18+
# ------------------------------------------------------------------------------
19+
# Private helpers
20+
# ------------------------------------------------------------------------------
21+
22+
template updateTarget(
23+
buddy: SnapPeerRef;
24+
info: static[string];
25+
) =
26+
## Async/template
27+
##
28+
block body:
29+
# Check whether explicit target setup is configured
30+
if buddy.ctx.pool.target.isSome():
31+
let
32+
peer {.inject,used.} = $buddy.peer # logging only
33+
ctx = buddy.ctx
34+
35+
# Single target block hash
36+
if ctx.pool.target.value.blockHash != BlockHash(zeroHash32):
37+
let rc = buddy.headerStateSet(ctx.pool.target.value.blockHash)
38+
if rc.isErr and rc.error: # real error
39+
trace info & ": failed fetching pivot hash", peer,
40+
hash=ctx.pool.target.value.blockHash.toStr
41+
elif ctx.pool.target.value.updateFile.len == 0:
42+
ctx.pool.target = Opt.none(SnapTarget) # No more target entries
43+
break body # noting more to do here
44+
else:
45+
var target = ctx.pool.target.value
46+
target.blockHash = BlockHash(zeroHash32)
47+
ctx.pool.target = Opt.some(target)
48+
49+
# Check whether a file target setup is configured
50+
if 0 < ctx.pool.target.value.updateFile.len:
51+
discard buddy.headerStateLoad( ctx.pool.target.value.updateFile, info)
52+
53+
discard # visual alignment
54+
55+
56+
template download(
57+
buddy: SnapPeerRef;
58+
state: StateDataRef;
59+
info: static[string];
60+
) =
61+
## Async/template
62+
##
63+
block body:
64+
let
65+
ctx = buddy.ctx
66+
sdb = ctx.pool.stateDB
67+
68+
peer {.inject,used.} = $buddy.peer # logging only
69+
root {.inject,used.} = state.rootStr # logging only
70+
71+
iv = state.unproc.fetchLeast(unprocAccountsRangeMax).valueOr:
72+
trace info & ": no more unpocessed", peer, root, stateDB=sdb.toStr
73+
break body
74+
75+
accData = buddy.fetchAccounts(state.root, iv).valueOr:
76+
state.unproc.commit(iv, iv) # registry roll back
77+
state.downScore()
78+
trace info & ": account download failed", peer, root,
79+
iv=iv.to(float).toStr, stateDB=sdb.toStr
80+
break body
81+
82+
# Accept, update registry
83+
if 0 < accData.accounts.len:
84+
let accTop = accData.accounts[^1].accHash.to(ItemKey)
85+
state.unproc.commit(iv, accTop + 1, iv.maxPt)
86+
state.unproc.overCommit(iv.maxPt + 1, accTop)
87+
state.upScore()
88+
else:
89+
state.downScore()
90+
91+
debug info & ": accounts downloaded", peer, root, iv=iv.to(float).toStr,
92+
nAccounts=accData.accounts.len, nProof=accData.proof.len,
93+
stateDB=sdb.toStr
94+
discard
95+
96+
# ------------------------------------------------------------------------------
97+
# Public function
98+
# ------------------------------------------------------------------------------
99+
100+
template accountRangeImport*(buddy: SnapPeerRef; info: static[string]) =
101+
## Async/template
102+
##
103+
## Fetch and stash account ranges -- TBD
104+
##
105+
block body:
106+
let
107+
ctx = buddy.ctx
108+
sdb = ctx.pool.stateDB
109+
peer {.inject,used.} = $buddy.peer # logging only
110+
111+
# Update state db, add new state records if pivots are available
112+
if buddy.only.pivotRoot.isNone():
113+
let ethPeer = buddy.getEthPeer() # get `ethXX` peer if avail
114+
if not ethPeer.isNil:
115+
trace info & ": processing best/latest pivotHash", peer,
116+
hash=ethPeer.only.pivotHash.short
117+
buddy.headerStateSet(BlockHash(ethPeer.only.pivotHash)).isErrOr:
118+
buddy.only.pivotRoot = Opt.some(value)
119+
120+
# Check for maual target settings
121+
buddy.updateTarget info
122+
123+
if ctx.pool.stateDB.len == 0:
124+
trace info & ": no state records", peer, stateDB=sdb.toStr
125+
break body
126+
127+
# Fetch for state DB items, start with pivot root
128+
for state in sdb.items(startWith=buddy.only.pivotRoot, truncate=true):
129+
if buddy.ctrl.stopped:
130+
break
131+
trace info & ": download state", peer, root=state.rootStr,
132+
stateDB=sdb.toStr
133+
buddy.download(state, info)
134+
135+
discard # visual alignment
136+
137+
# ------------------------------------------------------------------------------
138+
# End
139+
# ------------------------------------------------------------------------------

0 commit comments

Comments
 (0)