Skip to content

Commit 96aa7cf

Browse files
committed
feat(supervisor): dial pool sandbox sessions to the host's gateway replica
HostAccepted now carries the accepting replica's peer endpoint. A shared host's sandbox sessions dial it directly, so each session runs on the replica serving its supervisor. A stopping replica sheds its host streams, and redirected sessions follow the host to its next replica. Closes #5051. Signed-off-by: divesh <[email protected]>
1 parent 465aa71 commit 96aa7cf

12 files changed

Lines changed: 147 additions & 1 deletion

File tree

‎crates/openshell-server/src/grpc/supervisor_host.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,16 @@ impl SupervisorHost for SupervisorHostService {
5959
Status::failed_precondition("this gateway does not host a shared supervisor")
6060
})?
6161
.clone();
62+
// A stopping replica sheds its hosts so they reattach to a live one
63+
// before their sandbox sessions follow them there.
64+
let mut shutdown = self.state.supervisor_sessions.subscribe_shutdown();
65+
if *shutdown.borrow() {
66+
return Err(Status::unavailable("gateway is shutting down"));
67+
}
6268
let outbound = controller.attach(identity, request.into_inner()).await?;
69+
let outbound = futures::StreamExt::take_until(outbound, async move {
70+
let _ = shutdown.wait_for(|shutdown| *shutdown).await;
71+
});
6372
Ok(Response::new(Box::pin(outbound)))
6473
}
6574
}

‎crates/openshell-server/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -836,6 +836,7 @@ pub(crate) async fn run_server(
836836
"shared supervisor assignment stream enabled; minting supervisor tokens"
837837
);
838838
info!("shared supervisor pool enabled; reconciling pool membership and ownership");
839+
controller.set_peer_endpoint(state.peer_endpoint.clone().unwrap_or_default());
839840
supervisor_pool::SupervisorPool::new(
840841
state.store.clone(),
841842
controller.clone(),

‎crates/openshell-server/src/supervisor_session.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,11 @@ impl SupervisorSessionRegistry {
435435
self.admission_closed.store(true, Ordering::Release);
436436
}
437437

438+
/// Watch for [`Self::shutdown`] beginning.
439+
pub(crate) fn subscribe_shutdown(&self) -> watch::Receiver<bool> {
440+
self.shutdown.subscribe()
441+
}
442+
438443
/// Close control sessions and wait for tracked ownership cleanup. Call
439444
/// after compute shutdown so supervisors can finish normal stop reporting.
440445
pub(crate) async fn shutdown(&self, timeout: Duration) -> Result<(), String> {

‎crates/openshell-supervisor-control/src/controller.rs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,8 @@ struct Inner {
356356
/// stream attached or ended, or a member announced leaving.
357357
membership: watch::Sender<u64>,
358358
observed: Mutex<Observed>,
359+
/// This replica's peer endpoint, sent in every `HostAccepted`.
360+
peer_endpoint: Mutex<String>,
359361
}
360362

361363
/// Gateway-owned desired assignments and observed status of the shared
@@ -406,10 +408,17 @@ impl SupervisorHostController {
406408
revision: watch::Sender::new(0),
407409
membership: watch::Sender::new(0),
408410
observed: Mutex::new(Observed::default()),
411+
peer_endpoint: Mutex::new(String::new()),
409412
}),
410413
}
411414
}
412415

416+
/// Tell attaching hosts to dial their sandbox sessions to `peer_endpoint`.
417+
/// Call it before any host attaches.
418+
pub fn set_peer_endpoint(&self, peer_endpoint: String) {
419+
*lock(&self.inner.peer_endpoint) = peer_endpoint;
420+
}
421+
413422
/// Offer every instance's runtime `recovery_timeout_secs` instead of
414423
/// [`POOL_RECOVERY_TIMEOUT_SECS`]. Call it before any host attaches.
415424
pub fn set_recovery_timeout_secs(&self, recovery_timeout_secs: u32) {
@@ -843,13 +852,15 @@ impl SupervisorHostController {
843852
let (tx, rx) = mpsc::channel(OUTBOUND_QUEUE);
844853
let expires_at = identity.expires_at;
845854
let (session_id, superseded) = self.begin_session(identity, &hello)?;
855+
let peer_endpoint = lock(&self.inner.peer_endpoint).clone();
846856
tx.send(Ok(HostGatewayMessage {
847857
payload: Some(host_gateway_message::Payload::Accepted(HostAccepted {
848858
session_id: session_id.to_string(),
849859
heartbeat_interval: Some(prost_types::Duration {
850860
seconds: i64::try_from(heartbeat.as_secs()).unwrap_or(15),
851861
nanos: 0,
852862
}),
863+
peer_endpoint,
853864
})),
854865
}))
855866
.await

‎crates/openshell-supervisor-process/src/delegated.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ pub async fn start_boundary_access(
9393
config_apply_tx: tokio::sync::mpsc::Sender<crate::supervisor_session::ConfigApplyRequest>,
9494
config_push_updates: tokio::sync::watch::Sender<bool>,
9595
host_key: Option<russh::keys::PrivateKey>,
96+
gateway_home: Option<tokio::sync::watch::Receiver<Option<String>>>,
9697
) -> Result<BoundaryAccess> {
9798
let instance_id = uuid::Uuid::new_v4().to_string();
9899
let terminating = Arc::new(AtomicBool::new(false));
@@ -130,6 +131,7 @@ pub async fn start_boundary_access(
130131
session_id_updates: supervisor_session_updates,
131132
config_apply_tx,
132133
config_push_updates,
134+
gateway_home,
133135
},
134136
);
135137
(Some(task), Some(accepted))
@@ -374,6 +376,7 @@ mod tests {
374376
tokio::sync::mpsc::channel(1).0,
375377
tokio::sync::watch::channel(false).0,
376378
None,
379+
None,
377380
)
378381
.await
379382
.expect("access without SSH is portable");
@@ -415,6 +418,7 @@ mod tests {
415418
tokio::sync::mpsc::channel(1).0,
416419
tokio::sync::watch::channel(false).0,
417420
None,
421+
None,
418422
)
419423
.await;
420424
assert!(

‎crates/openshell-supervisor-process/src/supervisor_session.rs‎

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ use openshell_core::transport_errors::is_expected_transport_close_status;
4444

4545
const INITIAL_BACKOFF: Duration = Duration::from_secs(1);
4646
const MAX_BACKOFF: Duration = Duration::from_secs(30);
47+
/// How long a shared host's session waits for the host to attach to a
48+
/// gateway replica before dialing another address.
49+
const GATEWAY_HOME_WAIT: Duration = Duration::from_secs(5);
4750
const CONFIG_APPLY_TIMEOUT: Duration = Duration::from_mins(1);
4851

4952
/// A stream-delivered configuration update awaiting application by the
@@ -129,6 +132,9 @@ pub struct SessionRuntimeContext {
129132
/// Publishes whether the current session pushes configuration, so the
130133
/// runtime stops polling while it does.
131134
pub config_push_updates: watch::Sender<bool>,
135+
/// Peer endpoint of the gateway replica serving this supervisor's shared
136+
/// host. When set, sessions dial it instead of the configured address.
137+
pub gateway_home: Option<watch::Receiver<Option<String>>>,
132138
}
133139

134140
/// Parse a gRPC endpoint URI into an OCSF `Endpoint` (host + port). Falls back
@@ -423,6 +429,7 @@ pub fn spawn_with_readiness(
423429
session_id_updates: runtime.session_id_updates,
424430
config_apply_tx: runtime.config_apply_tx,
425431
config_push_updates: runtime.config_push_updates,
432+
gateway_home: runtime.gateway_home,
426433
ready_tx,
427434
};
428435
(tokio::spawn(run_session_loop(config)), ready_rx)
@@ -442,6 +449,7 @@ struct SessionConfig {
442449
/// Publishes whether the current session pushes configuration. A
443450
/// reconnect to a gateway that does not push resumes polling.
444451
config_push_updates: watch::Sender<bool>,
452+
gateway_home: Option<watch::Receiver<Option<String>>>,
445453
ready_tx: watch::Sender<bool>,
446454
}
447455

@@ -455,9 +463,22 @@ async fn run_session_loop(config: SessionConfig) {
455463
// back to the configured address, which load-balances across replicas.
456464
let mut target = config.gateway.clone();
457465
let mut redirected = false;
466+
// A shared host's sessions dial the replica serving the host, as
467+
// redirected so that replica serves them, unless they are following a
468+
// redirect away from it.
469+
let mut gateway_home = config.gateway_home.clone();
470+
let mut follow_home = true;
458471

459472
loop {
460473
attempt += 1;
474+
if follow_home
475+
&& let Some(home) = gateway_home.as_mut()
476+
&& let Some(endpoint) = wait_for_home(home, None).await
477+
{
478+
target = config.gateway.with_url(endpoint);
479+
redirected = true;
480+
}
481+
follow_home = true;
461482

462483
let result = run_single_session(&config, &target, redirected, attempt).await;
463484
if let Some(updates) = &config.session_id_updates {
@@ -485,8 +506,14 @@ async fn run_session_loop(config: SessionConfig) {
485506
peer_endpoint = %peer_endpoint,
486507
"supervisor session: following gateway redirect"
487508
);
488-
target = config.gateway.with_url(peer_endpoint);
509+
// Prefer the replica the shared host moved to.
510+
let home = match gateway_home.as_mut() {
511+
Some(home) => wait_for_home(home, Some(target.url())).await,
512+
None => None,
513+
};
514+
target = config.gateway.with_url(home.unwrap_or(peer_endpoint));
489515
redirected = true;
516+
follow_home = false;
490517
// No backoff: this is an expected handoff, not a failure.
491518
}
492519
Err(e) => {
@@ -555,6 +582,22 @@ impl ReconnectBackoff {
555582
}
556583
}
557584

585+
/// The peer endpoint of the replica serving the shared host, once it is one
586+
/// other than `avoid`, waiting up to [`GATEWAY_HOME_WAIT`].
587+
async fn wait_for_home(
588+
home: &mut watch::Receiver<Option<String>>,
589+
avoid: Option<&str>,
590+
) -> Option<String> {
591+
tokio::time::timeout(
592+
GATEWAY_HOME_WAIT,
593+
home.wait_for(|home| home.as_deref().is_some_and(|home| Some(home) != avoid)),
594+
)
595+
.await
596+
.ok()?
597+
.ok()
598+
.and_then(|home| (*home).clone())
599+
}
600+
558601
/// Keep asking not to be redirected when a redirected attempt never got a
559602
/// session, so a stale ring pointing at a dead replica cannot bounce us back
560603
/// to it. A session that was accepted and later dropped starts placement over.
@@ -1636,6 +1679,31 @@ mod ocsf_event_tests {
16361679
assert!(!redirect_survives_failure(false, true));
16371680
}
16381681

1682+
#[tokio::test(start_paused = true)]
1683+
async fn shared_host_sessions_wait_for_the_hosts_gateway() {
1684+
let (tx, mut home) = watch::channel(None);
1685+
assert_eq!(wait_for_home(&mut home, None).await, None);
1686+
1687+
tx.send_replace(Some("https://gw-a:8443".to_string()));
1688+
assert_eq!(
1689+
wait_for_home(&mut home, None).await.as_deref(),
1690+
Some("https://gw-a:8443")
1691+
);
1692+
assert_eq!(
1693+
wait_for_home(&mut home, Some("https://gw-a:8443")).await,
1694+
None
1695+
);
1696+
1697+
let mut moving = home.clone();
1698+
let moved =
1699+
tokio::spawn(
1700+
async move { wait_for_home(&mut moving, Some("https://gw-a:8443")).await },
1701+
);
1702+
tokio::task::yield_now().await;
1703+
tx.send_replace(Some("https://gw-b:8443".to_string()));
1704+
assert_eq!(moved.await.unwrap().as_deref(), Some("https://gw-b:8443"));
1705+
}
1706+
16391707
#[test]
16401708
fn reconnect_backoff_doubles_until_a_session_is_accepted() {
16411709
let mut backoff = ReconnectBackoff::new();

‎crates/openshell-supervisor/src/backend_setup/tests.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -834,6 +834,7 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() {
834834
sandbox: None,
835835
container_image: String::new(),
836836
openshell_endpoint: None,
837+
gateway_home: None,
837838
policy_rules: Some(rules.to_string_lossy().into_owned()),
838839
policy_data: Some(data.to_string_lossy().into_owned()),
839840
ssh_socket_path: None,
@@ -1038,6 +1039,7 @@ async fn suspend_signal_exits_without_terminating_boundary() {
10381039
sandbox: None,
10391040
container_image: String::new(),
10401041
openshell_endpoint: None,
1042+
gateway_home: None,
10411043
policy_rules: Some(rules.to_string_lossy().into_owned()),
10421044
policy_data: Some(data.to_string_lossy().into_owned()),
10431045
ssh_socket_path: None,

‎crates/openshell-supervisor/src/legacy_startup_regression_tests.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,7 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl
287287
sandbox: Some("legacy-restart".into()),
288288
container_image: String::new(),
289289
openshell_endpoint: Some(endpoint),
290+
gateway_home: None,
290291
policy_rules: None,
291292
policy_data: None,
292293
ssh_socket_path: None,

‎crates/openshell-supervisor/src/lib.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -896,6 +896,9 @@ pub struct SandboxInstanceConfig {
896896
/// Workload image recorded in this instance's OCSF events.
897897
pub container_image: String,
898898
pub openshell_endpoint: Option<String>,
899+
/// Peer endpoint of the gateway replica serving the shared host, which
900+
/// the session dials instead of `openshell_endpoint` when set.
901+
pub gateway_home: Option<tokio::sync::watch::Receiver<Option<String>>>,
899902
pub policy_rules: Option<String>,
900903
pub policy_data: Option<String>,
901904
pub ssh_socket_path: Option<String>,
@@ -1060,6 +1063,7 @@ pub async fn run_sandbox_instance(
10601063
sandbox,
10611064
container_image: _,
10621065
openshell_endpoint,
1066+
gateway_home,
10631067
policy_rules,
10641068
policy_data,
10651069
ssh_socket_path,
@@ -1704,6 +1708,7 @@ pub async fn run_sandbox_instance(
17041708
config_apply_tx,
17051709
config_push_updates,
17061710
ssh_host_key,
1711+
gateway_home,
17071712
)
17081713
.instrument(tracing::info_span!(parent: &startup, "supervisor.access.start"))
17091714
.await?;
@@ -4628,6 +4633,7 @@ mod tests {
46284633
sandbox: None,
46294634
container_image: String::new(),
46304635
openshell_endpoint: None,
4636+
gateway_home: None,
46314637
policy_rules: None,
46324638
policy_data: None,
46334639
ssh_socket_path: None,

‎crates/openshell-supervisor/src/main.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -607,6 +607,7 @@ fn main() -> Result<()> {
607607
sandbox: args.sandbox,
608608
container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(),
609609
openshell_endpoint: args.openshell_endpoint,
610+
gateway_home: None,
610611
policy_rules: args.policy_rules,
611612
policy_data: args.policy_data,
612613
ssh_socket_path: args.ssh_socket_path,

0 commit comments

Comments
 (0)