Skip to content

Commit d4ada9e

Browse files
mjfhmjfh
authored
Snap sync implement code dowloader (#3982)
* Re-factor DB cache/assembly why Storage slots cache should have `account` as part of the lookup key * Cosmetics, code clean up/beautification, small fixes * Implement byre codes downloader
1 parent bb120ec commit d4ada9e

18 files changed

+876
-156
lines changed

‎execution_chain/sync/snap/worker/account/account_download.nim‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,11 +63,13 @@ template accountDownload*(
6363
nProof {.inject,used.} = data.proof.len # logging only
6464

6565
# Stash accounts data packet to be processed later
66-
adb.putRawAccounts(state.root,ivReq.minPt,limit,data,buddy.peerID).isOkOr:
67-
sdb.rollbackAccountRange(state, ivReq) # registry roll back
68-
debug info & ": caching accounts failed", peer, root, iv,
69-
nAccounts, nProof
70-
break body # return err()
66+
adb.putRawAccounts(
67+
state.root, ivReq.minPt, limit, data.accounts, data.proof,
68+
buddy.peerID).isOkOr:
69+
sdb.rollbackAccountRange(state, ivReq) # registry roll back
70+
debug info & ": caching accounts failed", peer, root, iv,
71+
nAccounts, nProof
72+
break body # return err()
7173

7274
state.upScore() # got some data
7375
sdb.commitAccountRange(state, ivReq, limit) # update registry

‎execution_chain/sync/snap/worker/account/account_fetch.nim‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -151,12 +151,15 @@ template fetchAccounts*(
151151
let
152152
nRespAcc {.inject.} = rc.value.packet.accounts.len
153153
nRespProof {.inject.} = rc.value.packet.proof.len
154+
var
155+
respAcc {.inject,used.} = "n/a" # logging only
154156

155157
if 0 < nRespAcc:
156158
let
157159
accMin = rc.value.packet.accounts[0].accHash.to(ItemKey)
158160
accMax = rc.value.packet.accounts[^1].accHash.to(ItemKey)
159-
respAcc {.inject,used.} = (accMin,accMax).flStr # logging only
161+
162+
respAcc = (accMin,accMax).flStr # logging only
160163

161164
if accMin < ivReq.minPt:
162165
trace recvInfo & " min account too low", peer, root, reqAcc, nReqAcc,
@@ -185,8 +188,6 @@ template fetchAccounts*(
185188
# will know about that. What will happen when a proof is missing
186189
# is that the trie `validation()` function will fail at a later
187190
# stage.
188-
trace recvInfo, peer, root, reqAcc, nReqAcc, respAcc, nRespAcc,
189-
nRespProof, ela, state, nErrors=buddy.nErrors.fetch.acc
190191

191192
elif nRespProof == 0:
192193
# No data available for this state root.
@@ -197,17 +198,16 @@ template fetchAccounts*(
197198
bodyRc = FetchAccountsResult.err(ENoDataAvailable)
198199
break body # return err()
199200

200-
else:
201-
trace recvInfo, peer, root, reqAcc, nReqAcc, nRespAcc,
202-
nRespProof, ela, state, nErrors=buddy.nErrors.fetch.acc
203-
204201
# Ban an overly slow peer for a while when observed consecutively.
205202
if fetchAccountSnapTimeout < elapsed:
206203
buddy.accFetchRegisterError(slowPeer=true)
207204
else:
208205
buddy.nErrors.fetch.acc = 0 # reset error count
209206
buddy.ctx.pool.lastSlowPeer = Opt.none(Hash) # not last one/error
210207

208+
trace recvInfo, peer, root, reqAcc, nReqAcc, respAcc, nRespAcc,
209+
nRespProof, ela, state, nErrors=buddy.nErrors.fetch.acc
210+
211211
bodyRc = FetchAccountsResult.ok(rc.value.packet)
212212

213213
bodyRc

‎execution_chain/sync/snap/worker/account/account_requeue.nim‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,12 @@ proc accountRequeue*(ctx: SnapCtxRef; info: static[string]): bool =
3636
let
3737
then = Moment.now()
3838
mpt = block:
39-
let rc = w.root.validate(w.start, w.packet)
39+
let rc = w.root.validate(w.start, w.accounts, w.proof)
4040
block cleanUp:
4141
adb.delRawAccounts(w.root, w.start).isOkOr:
4242
trace info & ": error deleting packet", root=w.root.toStr,
4343
iv=(w.start,w.limit).to(float).toStr,
44-
nAccounts=w.packet.accounts.len, nProof=w.packet.proof.len,
45-
`error`=error
44+
nAccounts=w.accounts.len, nProof=w.proof.len,`error`=error
4645
break cleanUp
4746

4847
if rc.isErr:
@@ -54,10 +53,10 @@ proc accountRequeue*(ctx: SnapCtxRef; info: static[string]): bool =
5453
# Done for now
5554
ctx.pool.mptEla += (Moment.now() - then)
5655
debug info & ": accounts validation failed", root=w.root.toStr,
57-
iv=(w.start,w.limit).to(float).toStr,
58-
nAccounts=w.packet.accounts.len, nProof=w.packet.proof.len,
59-
elaSum=ctx.pool.mptEla.toStr
60-
doAssert dumpAccFailFile.dumpToFile(w.root, w.start, w.packet)
56+
iv=(w.start,w.limit).to(float).toStr, nAccounts=w.accounts.len,
57+
nProof=w.proof.len, elaSum=ctx.pool.mptEla.toStr
58+
doAssert dumpAccFailFile.dumpToFile(
59+
w.root, w.start, w.accounts, w.proof)
6160
return true # failed, but did something
6261

6362
rc.value
@@ -69,17 +68,16 @@ proc accountRequeue*(ctx: SnapCtxRef; info: static[string]): bool =
6968

7069
if rc.isErr:
7170
debug info & ": caching accounts failed", root=w.root.toStr,
72-
iv=(w.start,w.limit).to(float).toStr,
73-
nAccounts=w.packet.accounts.len, nProof=w.packet.proof.len,
74-
elaSum=ctx.pool.mptEla.toStr, error=rc.error
75-
doAssert dumpAccFailFile.dumpToFile(w.root, w.start, w.packet)
71+
iv=(w.start,w.limit).to(float).toStr, nAccounts=w.accounts.len,
72+
nProof=w.proof.len, elaSum=ctx.pool.mptEla.toStr, error=rc.error
73+
doAssert dumpAccFailFile.dumpToFile(
74+
w.root, w.start, w.accounts, w.proof)
7675
return true # failed, but did something
7776

7877
# Successfully stored
7978
debug info & ": accounts stored", root=w.root.toStr,
80-
iv=(w.start,w.limit).to(float).toStr,
81-
nAccounts=w.packet.accounts.len, nProof=w.packet.proof.len,
82-
elaSum=ctx.pool.mptEla.toStr
79+
iv=(w.start,w.limit).to(float).toStr, nAccounts=w.accounts.len,
80+
nProof=w.proof.len, elaSum=ctx.pool.mptEla.toStr
8381
return true
8482

8583
# false # no serious work done

‎execution_chain/sync/snap/worker/code.nim‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Nimbus
2+
# Copyright (c) 2025-2026 Status Research & Development GmbH
3+
# Licensed under either of
4+
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
5+
# http://www.apache.org/licenses/LICENSE-2.0)
6+
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
7+
# http://opensource.org/licenses/MIT)
8+
# at your option. This file may not be copied, modified, or distributed
9+
# except according to those terms.
10+
11+
{.push raises: [].}
12+
13+
import
14+
./code/code_download
15+
16+
export
17+
code_download
18+
19+
# End

‎execution_chain/sync/snap/worker/code/code_download.nim‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
# Nimbus
2+
# Copyright (c) 2025-2026 Status Research & Development GmbH
3+
# Licensed under either of
4+
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
5+
# http://www.apache.org/licenses/LICENSE-2.0)
6+
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
7+
# http://opensource.org/licenses/MIT)
8+
# at your option. This file may not be copied, modified, or distributed
9+
# except according to those terms.
10+
11+
{.push raises: [].}
12+
13+
import
14+
std/sequtils,
15+
pkg/[chronicles, chronos],
16+
../[helpers, mpt, state_db, worker_desc],
17+
./code_fetch
18+
19+
# ------------------------------------------------------------------------------
20+
# Private helpers
21+
# ------------------------------------------------------------------------------
22+
23+
proc register(state: StateDataRef, acc: seq[(ItemKey,CodeHash)]) =
24+
for (key,val) in acc:
25+
state.register(key,val)
26+
27+
# ------------------------------------------------------------------------------
28+
# Private functions
29+
# ------------------------------------------------------------------------------
30+
31+
template downloadImpl(
32+
buddy: SnapPeerRef; # Snap peer
33+
state: StateDataRef; # Current state
34+
accounts: seq[(ItemKey,CodeHash)]; # Acoounts with contracts
35+
info: static[string]; # Log message prefix
36+
): bool =
37+
## Async/template
38+
##
39+
## The template will return `true` if there were some data that could be
40+
## downloaded and processed.
41+
##
42+
var bodyRc = false
43+
block body:
44+
let
45+
ctx = buddy.ctx
46+
adb = ctx.pool.mptAsm
47+
sRoot = state.root
48+
peerID = buddy.peerID
49+
50+
peer {.inject,used.} = $buddy.peer # logging only
51+
root {.inject,used.} = state.rootStr # logging only
52+
53+
# Fetch storage slots from argument list `accounts`
54+
var start {.inject.} = 0
55+
while start < accounts.len:
56+
let
57+
accLeft = if start == 0: accounts else: accounts[start .. ^1]
58+
codeHashes = accLeft.mapIt(it[1])
59+
60+
# Fetch from network
61+
let data = buddy.fetchCodes(sRoot, codeHashes).valueOr:
62+
state.register accLeft # stash data and return
63+
trace info & ": fetching codes failed", peer, root,
64+
start, nAccLeft=accLeft.len
65+
break body # error => return
66+
67+
# Store byte codes on database
68+
adb.putRawAyteCode(
69+
sRoot, accLeft[0][0], accLeft[^1][0],
70+
codeHashes.zip data.codes, peerID).isOkOr:
71+
state.register(accLeft) # stash data and return
72+
trace info & ": storing codes failed", peer, root,
73+
start, nAccLeft=accLeft.len
74+
break body # error => return
75+
76+
start += data.codes.len
77+
bodyRc = true # did something
78+
79+
# End `while`
80+
81+
bodyRc
82+
83+
template downloadFromQueue(
84+
buddy: SnapPeerRef; # Snap peer
85+
state: StateDataRef; # Current state
86+
info: static[string]; # Log message prefix
87+
): bool =
88+
## Async/template
89+
##
90+
## Process stashed unprocessed byte codes from the state DB.
91+
##
92+
## The template will return `true` if there were some data that could be
93+
## downloaded and processed.
94+
##
95+
var bodyRc = false
96+
block body:
97+
let
98+
peer {.inject,used.} = $buddy.peer # logging only
99+
root {.inject,used.} = state.rootStr # logging only
100+
var
101+
accQueue: seq[(ItemKey,CodeHash)]
102+
103+
for w in state.stoItems:
104+
accQueue.add (w.key, w.data.code)
105+
state.delCode w.key
106+
107+
if 0 < accQueue.len:
108+
trace info & ": processing from codes queue", peer, root,
109+
nAccQueue=accQueue.len
110+
if buddy.downloadImpl(state, accQueue, info):
111+
bodyRc = true
112+
113+
bodyRc
114+
115+
# ------------------------------------------------------------------------------
116+
# Public function
117+
# ------------------------------------------------------------------------------
118+
119+
template codeDownload*(
120+
buddy: SnapPeerRef; # Snap peer
121+
state: StateDataRef; # Current state
122+
accounts: seq[SnapAccount]; # Acoounts with sub-tries
123+
info: static[string]; # Log message prefix
124+
) =
125+
## Async/template
126+
##
127+
block body:
128+
let acc = accounts
129+
.filterIt(not it.accBody.codeHash.isEmpty)
130+
.mapIt( (it.accHash.to(ItemKey),
131+
it.accBody.codeHash.to(Hash32).to(CodeHash)) )
132+
133+
if buddy.ctrl.stopped:
134+
state.register acc # stash data and return
135+
break body # all done
136+
137+
discard buddy.downloadImpl(state, acc, info)
138+
139+
while not buddy.ctrl.stopped and
140+
0 < state.len and
141+
buddy.downloadFromQueue(state, info):
142+
continue
143+
144+
discard # visual alignment
145+
146+
# ------------------------------------------------------------------------------
147+
# End
148+
# ------------------------------------------------------------------------------

0 commit comments

Comments
 (0)