Skip to content

Commit fcc96a1

Browse files
michaelsproulmichaelsproul
committed
Merge remote-tracking branch 'origin/unstable' into tree-states-hot-rebase
2 parents 9151436 + 70850fe commit fcc96a1

File tree

219 files changed

+3876
-2464
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

219 files changed

+3876
-2464
lines changed

‎Cargo.lock‎

Lines changed: 58 additions & 70 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ members = [
5151
"common/unused_port",
5252
"common/validator_dir",
5353
"common/warp_utils",
54+
"common/workspace_members",
5455

5556
"consensus/fixed_bytes",
5657
"consensus/fork_choice",
@@ -120,6 +121,7 @@ bincode = "1"
120121
bitvec = "1"
121122
byteorder = "1"
122123
bytes = "1"
124+
cargo_metadata = "0.19"
123125
clap = { version = "4.5.4", features = ["derive", "cargo", "wrap_help"] }
124126
# Turn off c-kzg's default features which include `blst/portable`. We can turn on blst's portable
125127
# feature ourselves when desired.
@@ -144,6 +146,7 @@ fnv = "1"
144146
fs2 = "0.4"
145147
futures = "0.3"
146148
graffiti_file = { path = "validator_client/graffiti_file" }
149+
gossipsub = { package = "libp2p-gossipsub", git = "https://github.com/sigp/rust-libp2p.git", rev = "7a36e4c" }
147150
hex = "0.4"
148151
hashlink = "0.9.0"
149152
hyper = "1"
@@ -158,7 +161,7 @@ mockito = "1.5.0"
158161
num_cpus = "1"
159162
parking_lot = "0.12"
160163
paste = "1"
161-
prometheus = "0.13"
164+
prometheus = { version = "0.13", default-features = false }
162165
quickcheck = "1"
163166
quickcheck_macros = "1"
164167
quote = "1"
@@ -173,7 +176,7 @@ reqwest = { version = "0.11", default-features = false, features = [
173176
"rustls-tls",
174177
"native-tls-vendored",
175178
] }
176-
ring = "0.16"
179+
ring = "0.17"
177180
rpds = "0.11"
178181
rusqlite = { version = "0.28", features = ["bundled"] }
179182
serde = { version = "1", features = ["derive"] }
@@ -245,6 +248,7 @@ kzg = { path = "crypto/kzg" }
245248
metrics = { path = "common/metrics" }
246249
lighthouse_network = { path = "beacon_node/lighthouse_network" }
247250
lighthouse_version = { path = "common/lighthouse_version" }
251+
workspace_members = { path = "common/workspace_members" }
248252
lockfile = { path = "common/lockfile" }
249253
logging = { path = "common/logging" }
250254
lru_cache = { path = "common/lru_cache" }
@@ -277,7 +281,7 @@ validator_metrics = { path = "validator_client/validator_metrics" }
277281
validator_store = { path = "validator_client/validator_store" }
278282
validator_test_rig = { path = "testing/validator_test_rig" }
279283
warp_utils = { path = "common/warp_utils" }
280-
xdelta3 = { git = "http://github.com/sigp/xdelta3-rs", rev = "50d63cdf1878e5cf3538e9aae5eed34a22c64e4a" }
284+
xdelta3 = { git = "http://github.com/sigp/xdelta3-rs", rev = "4db64086bb02e9febb584ba93b9d16bb2ae3825a" }
281285
zstd = "0.13"
282286

283287
[profile.maxperf]

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ install-audit:
250250
cargo install --force cargo-audit
251251

252252
audit-CI:
253-
cargo audit --ignore RUSTSEC-2025-0009 --ignore RUSTSEC-2024-0437
253+
cargo audit
254254

255255
# Runs `cargo vendor` to make sure dependencies can be vendored for packaging, reproducibility and archival purpose.
256256
vendor:

‎beacon_node/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "beacon_node"
3-
version = "7.0.0-beta.0"
3+
version = "7.0.0-beta.5"
44
authors = [
55
"Paul Hauner <[email protected]>",
66
"Age Manning <[email protected]",

‎beacon_node/beacon_chain/src/attestation_rewards.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use crate::{BeaconChain, BeaconChainError, BeaconChainTypes};
2-
use eth2::lighthouse::attestation_rewards::{IdealAttestationRewards, TotalAttestationRewards};
3-
use eth2::lighthouse::StandardAttestationRewards;
4-
use eth2::types::ValidatorId;
2+
use eth2::types::{
3+
IdealAttestationRewards, StandardAttestationRewards, TotalAttestationRewards, ValidatorId,
4+
};
55
use safe_arith::SafeArith;
66
use serde_utils::quoted_u64::Quoted;
77
use state_processing::common::base::{self, SqrtTotalActiveBalance};
@@ -51,8 +51,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
5151
.state_root_at_slot(state_slot)?
5252
.ok_or(BeaconChainError::NoStateForSlot(state_slot))?;
5353

54+
// This branch is reached from the HTTP API. We assume the user wants
55+
// to cache states so that future calls are faster.
5456
let state = self
55-
.get_state(&state_root, Some(state_slot))?
57+
.get_state(&state_root, Some(state_slot), true)?
5658
.ok_or(BeaconChainError::MissingBeaconState(state_root))?;
5759

5860
if state.fork_name_unchecked().altair_enabled() {

‎beacon_node/beacon_chain/src/attestation_verification.rs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1126,6 +1126,12 @@ fn verify_head_block_is_known<T: BeaconChainTypes>(
11261126
}
11271127
}
11281128

1129+
if !verify_attestation_is_finalized_checkpoint_or_descendant(attestation.data(), chain) {
1130+
return Err(Error::HeadBlockFinalized {
1131+
beacon_block_root: attestation.data().beacon_block_root,
1132+
});
1133+
}
1134+
11291135
Ok(block)
11301136
} else if chain.is_pre_finalization_block(attestation.data().beacon_block_root)? {
11311137
Err(Error::HeadBlockFinalized {
@@ -1359,6 +1365,29 @@ pub fn verify_committee_index<E: EthSpec>(attestation: AttestationRef<E>) -> Res
13591365
Ok(())
13601366
}
13611367

1368+
fn verify_attestation_is_finalized_checkpoint_or_descendant<T: BeaconChainTypes>(
1369+
attestation_data: &AttestationData,
1370+
chain: &BeaconChain<T>,
1371+
) -> bool {
1372+
// If we have a split block newer than finalization then we also ban attestations which are not
1373+
// descended from that split block. It's important not to try checking `is_descendant` if
1374+
// finality is ahead of the split and the split block has been pruned, as `is_descendant` will
1375+
// return `false` in this case.
1376+
let fork_choice = chain.canonical_head.fork_choice_read_lock();
1377+
let attestation_block_root = attestation_data.beacon_block_root;
1378+
let finalized_slot = fork_choice
1379+
.finalized_checkpoint()
1380+
.epoch
1381+
.start_slot(T::EthSpec::slots_per_epoch());
1382+
let split = chain.store.get_split_info();
1383+
let is_descendant_from_split_block = split.slot == 0
1384+
|| split.slot <= finalized_slot
1385+
|| fork_choice.is_descendant(split.block_root, attestation_block_root);
1386+
1387+
fork_choice.is_finalized_checkpoint_or_descendant(attestation_block_root)
1388+
&& is_descendant_from_split_block
1389+
}
1390+
13621391
/// Assists in readability.
13631392
type CommitteesPerSlot = u64;
13641393

‎beacon_node/beacon_chain/src/attester_cache.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -325,8 +325,10 @@ impl AttesterCache {
325325
return Ok(value);
326326
}
327327

328+
// We use `cache_state = true` here because if we are attesting to the state it's likely
329+
// to be recent and useful for other things.
328330
let mut state: BeaconState<T::EthSpec> = chain
329-
.get_state(&state_root, None)?
331+
.get_state(&state_root, None, true)?
330332
.ok_or(Error::MissingBeaconState(state_root))?;
331333

332334
if state.slot() > slot {

‎beacon_node/beacon_chain/src/beacon_block_reward.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use crate::{BeaconChain, BeaconChainError, BeaconChainTypes, StateSkipConfig};
22
use attesting_indices_base::get_attesting_indices;
3-
use eth2::lighthouse::StandardBlockReward;
3+
use eth2::types::StandardBlockReward;
44
use safe_arith::SafeArith;
55
use state_processing::common::attesting_indices_base;
66
use state_processing::{

‎beacon_node/beacon_chain/src/beacon_chain.rs‎

Lines changed: 81 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ use crate::light_client_optimistic_update_verification::{
4141
Error as LightClientOptimisticUpdateError, VerifiedLightClientOptimisticUpdate,
4242
};
4343
use crate::light_client_server_cache::LightClientServerCache;
44-
use crate::migrate::BackgroundMigrator;
44+
use crate::migrate::{BackgroundMigrator, ManualFinalizationNotification};
4545
use crate::naive_aggregation_pool::{
4646
AggregatedAttestationMap, Error as NaiveAggregationError, NaiveAggregationPool,
4747
SyncContributionAggregateMap,
@@ -117,8 +117,8 @@ use std::sync::Arc;
117117
use std::time::Duration;
118118
use store::iter::{BlockRootsIterator, ParentRootBlockIterator, StateRootsIterator};
119119
use store::{
120-
BlobSidecarListFromRoot, DatabaseBlock, Error as DBError, HotColdDB, KeyValueStore,
121-
KeyValueStoreOp, StoreItem, StoreOp,
120+
BlobSidecarListFromRoot, DatabaseBlock, Error as DBError, HotColdDB, HotStateSummary,
121+
KeyValueStore, KeyValueStoreOp, StoreItem, StoreOp,
122122
};
123123
use task_executor::{ShutdownReason, TaskExecutor};
124124
use tokio::sync::oneshot;
@@ -691,7 +691,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
691691
///
692692
/// - `slot` always increases by `1`.
693693
/// - Skipped slots contain the root of the closest prior
694-
/// non-skipped slot (identical to the way they are stored in `state.block_roots`).
694+
/// non-skipped slot (identical to the way they are stored in `state.block_roots`).
695695
/// - Iterator returns `(Hash256, Slot)`.
696696
///
697697
/// Will return a `BlockOutOfRange` error if the requested start slot is before the period of
@@ -755,7 +755,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
755755
///
756756
/// - `slot` always decreases by `1`.
757757
/// - Skipped slots contain the root of the closest prior
758-
/// non-skipped slot (identical to the way they are stored in `state.block_roots`) .
758+
/// non-skipped slot (identical to the way they are stored in `state.block_roots`) .
759759
/// - Iterator returns `(Hash256, Slot)`.
760760
/// - The provided `block_root` is included as the first item in the iterator.
761761
pub fn rev_iter_block_roots_from(
@@ -765,8 +765,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
765765
let block = self
766766
.get_blinded_block(&block_root)?
767767
.ok_or(Error::MissingBeaconBlock(block_root))?;
768+
// This method is only used in tests, so we may as well cache states to make CI go brr.
769+
// TODO(release-v7) move this method out of beacon chain and into `store_tests`` or something equivalent.
768770
let state = self
769-
.get_state(&block.state_root(), Some(block.slot()))?
771+
.get_state(&block.state_root(), Some(block.slot()), true)?
770772
.ok_or_else(|| Error::MissingBeaconState(block.state_root()))?;
771773
let iter = BlockRootsIterator::owned(&self.store, state);
772774
Ok(std::iter::once(Ok((block_root, block.slot())))
@@ -782,7 +784,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
782784
/// - `slot` always decreases by `1`.
783785
/// - Iterator returns `(Hash256, Slot)`.
784786
/// - As this iterator starts at the `head` of the chain (viz., the best block), the first slot
785-
/// returned may be earlier than the wall-clock slot.
787+
/// returned may be earlier than the wall-clock slot.
786788
pub fn rev_iter_state_roots_from<'a>(
787789
&'a self,
788790
state_root: Hash256,
@@ -1292,8 +1294,9 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
12921294
&self,
12931295
state_root: &Hash256,
12941296
slot: Option<Slot>,
1297+
update_cache: bool,
12951298
) -> Result<Option<BeaconState<T::EthSpec>>, Error> {
1296-
Ok(self.store.get_state(state_root, slot)?)
1299+
Ok(self.store.get_state(state_root, slot, update_cache)?)
12971300
}
12981301

12991302
/// Return the sync committee at `slot + 1` from the canonical chain.
@@ -1466,8 +1469,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
14661469
})?
14671470
.ok_or(Error::NoStateForSlot(slot))?;
14681471

1472+
// This branch is mostly reached from the HTTP API when doing analysis, or in niche
1473+
// situations when producing a block. In the HTTP API case we assume the user wants
1474+
// to cache states so that future calls are faster, and that if the cache is
1475+
// struggling due to non-finality that they will dial down inessential calls. In the
1476+
// block proposal case we want to cache the state so that we can process the block
1477+
// quickly after it has been signed.
14691478
Ok(self
1470-
.get_state(&state_root, Some(slot))?
1479+
.get_state(&state_root, Some(slot), true)?
14711480
.ok_or(Error::NoStateForSlot(slot))?)
14721481
}
14731482
}
@@ -1649,6 +1658,43 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
16491658
}
16501659
}
16511660

1661+
pub fn manually_compact_database(&self) {
1662+
self.store_migrator.process_manual_compaction();
1663+
}
1664+
1665+
pub fn manually_finalize_state(
1666+
&self,
1667+
state_root: Hash256,
1668+
checkpoint: Checkpoint,
1669+
) -> Result<(), Error> {
1670+
let HotStateSummary {
1671+
slot,
1672+
latest_block_root,
1673+
..
1674+
} = self
1675+
.store
1676+
.load_hot_state_summary(&state_root)
1677+
.map_err(BeaconChainError::DBError)?
1678+
.ok_or(BeaconChainError::MissingHotStateSummary(state_root))?;
1679+
1680+
if slot != checkpoint.epoch.start_slot(T::EthSpec::slots_per_epoch())
1681+
|| latest_block_root != *checkpoint.root
1682+
{
1683+
return Err(BeaconChainError::InvalidCheckpoint {
1684+
state_root,
1685+
checkpoint,
1686+
});
1687+
}
1688+
1689+
let notif = ManualFinalizationNotification {
1690+
state_root: state_root.into(),
1691+
checkpoint,
1692+
};
1693+
1694+
self.store_migrator.process_manual_finalization(notif);
1695+
Ok(())
1696+
}
1697+
16521698
/// Returns an aggregated `Attestation`, if any, that has a matching `attestation.data`.
16531699
///
16541700
/// The attestation will be obtained from `self.naive_aggregation_pool`.
@@ -2793,6 +2839,15 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
27932839
chain_segment: Vec<RpcBlock<T::EthSpec>>,
27942840
notify_execution_layer: NotifyExecutionLayer,
27952841
) -> ChainSegmentResult {
2842+
for block in chain_segment.iter() {
2843+
if let Err(error) = self.check_invalid_block_roots(block.block_root()) {
2844+
return ChainSegmentResult::Failed {
2845+
imported_blocks: vec![],
2846+
error,
2847+
};
2848+
}
2849+
}
2850+
27962851
let mut imported_blocks = vec![];
27972852

27982853
// Filter uninteresting blocks from the chain segment in a blocking task.
@@ -2928,6 +2983,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
29282983
pub async fn verify_block_for_gossip(
29292984
self: &Arc<Self>,
29302985
block: Arc<SignedBeaconBlock<T::EthSpec>>,
2986+
custody_columns_count: usize,
29312987
) -> Result<GossipVerifiedBlock<T>, BlockError> {
29322988
let chain = self.clone();
29332989
self.task_executor
@@ -2937,7 +2993,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
29372993
let slot = block.slot();
29382994
let graffiti_string = block.message().body().graffiti().as_utf8_lossy();
29392995

2940-
match GossipVerifiedBlock::new(block, &chain) {
2996+
match GossipVerifiedBlock::new(block, &chain, custody_columns_count) {
29412997
Ok(verified) => {
29422998
let commitments_formatted = verified.block.commitments_formatted();
29432999
debug!(
@@ -3284,6 +3340,15 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
32843340
self.remove_notified(&block_root, r)
32853341
}
32863342

3343+
/// Check for known and configured invalid block roots before processing.
3344+
pub fn check_invalid_block_roots(&self, block_root: Hash256) -> Result<(), BlockError> {
3345+
if self.config.invalid_block_roots.contains(&block_root) {
3346+
Err(BlockError::KnownInvalidExecutionPayload(block_root))
3347+
} else {
3348+
Ok(())
3349+
}
3350+
}
3351+
32873352
/// Returns `Ok(block_root)` if the given `unverified_block` was successfully verified and
32883353
/// imported into the chain.
32893354
///
@@ -6715,9 +6780,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
67156780
})?;
67166781
let beacon_state_root = beacon_block.state_root();
67176782

6783+
// This branch is reached from the HTTP API. We assume the user wants
6784+
// to cache states so that future calls are faster.
67186785
let mut beacon_state = self
67196786
.store
6720-
.get_state(&beacon_state_root, Some(beacon_block.slot()))?
6787+
.get_state(&beacon_state_root, Some(beacon_block.slot()), true)?
67216788
.ok_or_else(|| {
67226789
Error::DBInconsistent(format!("Missing state {:?}", beacon_state_root))
67236790
})?;
@@ -6869,8 +6936,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
68696936

68706937
if signed_beacon_block.slot() % T::EthSpec::slots_per_epoch() == 0 {
68716938
let block = self.get_blinded_block(&block_hash).unwrap().unwrap();
6939+
// This branch is reached from the HTTP API. We assume the user wants
6940+
// to cache states so that future calls are faster.
68726941
let state = self
6873-
.get_state(&block.state_root(), Some(block.slot()))
6942+
.get_state(&block.state_root(), Some(block.slot()), true)
68746943
.unwrap()
68756944
.unwrap();
68766945
finalized_blocks.insert(state.finalized_checkpoint().root);
@@ -7031,10 +7100,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
70317100
block_root: Hash256,
70327101
block_data: AvailableBlockData<T::EthSpec>,
70337102
) -> Result<Option<StoreOp<T::EthSpec>>, String> {
7034-
// TODO(das) we currently store all subnet sampled columns. Tracking issue to exclude non
7035-
// custody columns: https://github.com/sigp/lighthouse/issues/6465
7036-
let _custody_columns_count = self.data_availability_checker.get_sampling_column_count();
7037-
70387103
match block_data {
70397104
AvailableBlockData::NoData => Ok(None),
70407105
AvailableBlockData::Blobs(blobs) => {

‎beacon_node/beacon_chain/src/beacon_proposer_cache.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ pub fn compute_proposer_duties_from_head<T: BeaconChainTypes>(
178178
/// - Returns an error if `state.current_epoch() > target_epoch`.
179179
/// - No-op if `state.current_epoch() == target_epoch`.
180180
/// - It must be the case that `state.canonical_root() == state_root`, but this function will not
181-
/// check that.
181+
/// check that.
182182
pub fn ensure_state_is_in_epoch<E: EthSpec>(
183183
state: &mut BeaconState<E>,
184184
state_root: Hash256,

0 commit comments

Comments
 (0)