Description
Due to the ordering of various operations during initialisation of Lighthouse's networking stack, we are currently unable to defer port selection to the OS (i.e., bind to the 0 port). The ability to do so is desirable for a few reasons; namely that it would avoid racing on bound port numbers which are currently causing some instability in our tests and subsequently CI jobs.
Currently, the following procedure describes the initialisation of the network service:
- Parse relevant CLI flags
- Produce a network service configuration from these flags
- Initialise the UPnP service
|
// try and construct UPnP port mappings if required. |
|
if let Some(upnp_config) = crate::nat::UPnPConfig::from_config(config) { |
|
let upnp_log = network_log.new(o!("service" => "UPnP")); |
|
let upnp_network_send = network_senders.network_send(); |
|
if config.upnp_enabled { |
|
executor.spawn_blocking( |
|
move || { |
|
crate::nat::construct_upnp_mappings( |
|
upnp_config, |
|
upnp_network_send, |
|
upnp_log, |
|
) |
|
}, |
|
"UPnP", |
|
); |
|
} |
|
} |
- Start
lighthouse_network (i.e., our libp2p handle)
|
// launch libp2p service |
|
let (mut libp2p, network_globals) = |
|
Network::new(executor.clone(), service_context, &network_log).await?; |
a) Build ENR
|
// Create an ENR or load from disk if appropriate |
|
let enr = crate::discovery::enr::build_or_load_enr::<TSpec>( |
|
local_keypair.clone(), |
|
&config, |
|
&ctx.enr_fork_id, |
|
&log, |
|
)?; |
b) Initialise network globals using this ENR
|
let globals = NetworkGlobals::new( |
|
enr, |
|
config.listen_addrs().v4().map(|v4_addr| v4_addr.tcp_port), |
|
config.listen_addrs().v6().map(|v6_addr| v6_addr.tcp_port), |
|
meta_data, |
|
config |
|
.trusted_peers |
|
.iter() |
|
.map(|x| PeerId::from(x.clone())) |
|
.collect(), |
|
config.disable_peer_scoring, |
|
&log, |
|
); |
c) Initialise discovery
|
let discovery = { |
|
// Build and start the discovery sub-behaviour |
|
let mut discovery = Discovery::new( |
|
local_keypair.clone(), |
|
&config, |
|
network_globals.clone(), |
|
&log, |
|
) |
|
.await?; |
|
// start searching for peers |
|
discovery.discover_peers(FIND_NODE_QUERY_CLOSEST_PEERS); |
|
discovery |
|
}; |
d) Start libp2p
|
let (swarm, bandwidth) = { |
|
// Set up the transport - tcp/ws with noise and mplex |
|
let (transport, bandwidth) = build_transport(local_keypair.clone()) |
|
.map_err(|e| format!("Failed to build transport: {:?}", e))?; |
|
|
|
// use the executor for libp2p |
|
struct Executor(task_executor::TaskExecutor); |
|
impl libp2p::swarm::Executor for Executor { |
|
fn exec(&self, f: Pin<Box<dyn futures::Future<Output = ()> + Send>>) { |
|
self.0.spawn(f, "libp2p"); |
|
} |
|
} |
|
|
|
// sets up the libp2p connection limits |
|
|
|
( |
|
SwarmBuilder::with_executor( |
|
transport, |
|
behaviour, |
|
local_peer_id, |
|
Executor(executor), |
|
) |
|
.notify_handler_buffer_size(std::num::NonZeroUsize::new(7).expect("Not zero")) |
|
.per_connection_event_buffer_size(4) |
|
.build(), |
|
bandwidth, |
|
) |
|
}; |
- Start network service
|
let network_service = NetworkService { |
|
beacon_chain, |
|
libp2p, |
|
attestation_service, |
|
sync_committee_service, |
|
network_recv, |
|
validator_subscription_recv, |
|
router_send, |
|
store, |
|
network_globals: network_globals.clone(), |
|
upnp_mappings: (None, None), |
|
discovery_auto_update: config.discv5_config.enr_update, |
|
next_fork_update, |
|
next_fork_subscriptions, |
|
next_unsubscribe, |
|
subscribe_all_subnets: config.subscribe_all_subnets, |
|
shutdown_after_sync: config.shutdown_after_sync, |
|
metrics_enabled: config.metrics_enabled, |
|
metrics_update, |
|
gossipsub_parameter_update, |
|
fork_context, |
|
log: network_log, |
|
enable_light_client_server: config.enable_light_client_server, |
|
}; |
|
|
|
network_service.spawn_service(executor); |
Steps to resolve
The new procedure needs to look like the following:
- Parse relevant CLI flags
- Produce a network service configuration from these flags
- Start
lighthouse_network
- Wait for a
NewListenAddr event from libp2p
a) Initialise the UPnP service
b) Update the networking globals accordingly
The associated RPC tests will also need to be updated to bind to port 0.
