Skip to content

Commit b256c3f

Browse files
committed
refactor(sdk): share replica routing helpers and address review feedback
Move owner_replica, with_owner_hint, routed_to and retry_unrouted into openshell-core so the CLI and SDK share one copy. Curated get_sandbox now caches the owner, owner lookup returns auth and availability errors, and a routed Unavailable drops the cached owner. Signed-off-by: divesh <[email protected]>
1 parent 92907e2 commit b256c3f

5 files changed

Lines changed: 199 additions & 224 deletions

File tree

‎crates/openshell-cli/src/tls.rs‎

Lines changed: 4 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ use miette::{IntoDiagnostic, Result, WrapErr};
55
use openshell_core::auth::EdgeAuthInterceptor;
66
use openshell_core::net::set_tcp_nodelay_best_effort;
77
use openshell_core::proto::open_shell_client::OpenShellClient;
8+
pub use openshell_core::replica_routing::{
9+
owner_replica, retry_unrouted, routed_to, with_owner_hint,
10+
};
811
use rustls::{
912
RootCertStore,
1013
client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier},
@@ -18,7 +21,6 @@ use std::path::PathBuf;
1821
use std::sync::OnceLock;
1922
use std::time::Duration;
2023
use tokio::sync::Mutex;
21-
use tonic::metadata::AsciiMetadataValue;
2224
use tonic::service::interceptor::InterceptedService;
2325
use tonic::transport::{Certificate, Channel, ClientTlsConfig, Endpoint, Identity};
2426
use tracing::debug;
@@ -464,127 +466,9 @@ fn interceptor_from_tls(tls: &TlsOptions) -> Result<EdgeAuthInterceptor> {
464466
EdgeAuthInterceptor::new(tls.oidc_token.as_deref(), tls.edge_token.as_deref())
465467
}
466468

467-
/// Gateway replica that owns the sandbox named in `response`, when it says.
468-
pub fn owner_replica<T>(response: &tonic::Response<T>) -> Option<AsciiMetadataValue> {
469-
response
470-
.metadata()
471-
.get(openshell_core::replica_routing::OWNER_REPLICA_HEADER)
472-
.cloned()
473-
}
474-
475-
pub fn with_owner_hint<T>(message: T) -> tonic::Request<T> {
476-
let mut request = tonic::Request::new(message);
477-
request.metadata_mut().insert(
478-
openshell_core::replica_routing::OWNER_REQUEST_HEADER,
479-
AsciiMetadataValue::from_static("1"),
480-
);
481-
request
482-
}
483-
484-
/// Wrap `message` so the edge proxy can route it straight to `replica`.
485-
pub fn routed_to<T>(message: T, replica: Option<&AsciiMetadataValue>) -> tonic::Request<T> {
486-
let mut request = tonic::Request::new(message);
487-
if let Some(replica) = replica {
488-
request.metadata_mut().insert(
489-
openshell_core::replica_routing::ROUTE_REPLICA_HEADER,
490-
replica.clone(),
491-
);
492-
}
493-
request
494-
}
495-
496-
/// A routed call is retried once without routing only when no gateway
497-
/// received it.
498-
///
499-
/// Envoy Gateway answers a replica route whose Service has no ready endpoints
500-
/// with `grpc-status: 14` and no `grpc-message`, while gateway errors and
501-
/// mid-request resets always carry a message.
502-
pub fn retry_unrouted(replica: Option<&AsciiMetadataValue>, status: &tonic::Status) -> bool {
503-
let message = status.message();
504-
replica.is_some()
505-
&& status.code() == tonic::Code::Unavailable
506-
&& (message.is_empty()
507-
|| message.contains("no healthy upstream")
508-
|| message.contains("connection failure"))
509-
}
510-
511469
#[cfg(test)]
512470
mod tests {
513-
use super::{owner_replica, retry_unrouted, routed_to, tls_server_name, with_owner_hint};
514-
use openshell_core::replica_routing::{
515-
OWNER_REPLICA_HEADER, OWNER_REQUEST_HEADER, ROUTE_REPLICA_HEADER,
516-
};
517-
518-
#[test]
519-
fn owner_hint_is_requested_explicitly() {
520-
assert!(
521-
with_owner_hint(())
522-
.metadata()
523-
.contains_key(OWNER_REQUEST_HEADER)
524-
);
525-
}
526-
527-
#[test]
528-
fn owner_replica_is_carried_to_the_routed_request() {
529-
let mut response = tonic::Response::new(());
530-
response
531-
.metadata_mut()
532-
.insert(OWNER_REPLICA_HEADER, "openshell-1".parse().unwrap());
533-
let replica = owner_replica(&response);
534-
let request = routed_to((), replica.as_ref());
535-
assert_eq!(
536-
request.metadata().get(ROUTE_REPLICA_HEADER).unwrap(),
537-
"openshell-1"
538-
);
539-
}
540-
541-
#[test]
542-
fn requests_without_a_known_owner_are_not_routed() {
543-
let replica = owner_replica(&tonic::Response::new(()));
544-
assert!(replica.is_none());
545-
let request = routed_to((), replica.as_ref());
546-
assert!(request.metadata().get(ROUTE_REPLICA_HEADER).is_none());
547-
}
548-
549-
#[test]
550-
fn only_unavailable_routed_calls_are_retried() {
551-
let replica = "openshell-1".parse().unwrap();
552-
let unavailable = tonic::Status::unavailable("no healthy upstream");
553-
assert!(retry_unrouted(Some(&replica), &unavailable));
554-
assert!(!retry_unrouted(None, &unavailable));
555-
assert!(!retry_unrouted(
556-
Some(&replica),
557-
&tonic::Status::permission_denied("denied")
558-
));
559-
}
560-
561-
#[test]
562-
fn calls_are_retried_only_when_no_gateway_received_them() {
563-
let replica = "openshell-1".parse().unwrap();
564-
for never_arrived in [
565-
"",
566-
"no healthy upstream",
567-
"upstream connect error or disconnect/reset before headers. reset reason: connection failure, transport failure reason: delayed connect error: Connection refused",
568-
] {
569-
assert!(retry_unrouted(
570-
Some(&replica),
571-
&tonic::Status::unavailable(never_arrived)
572-
));
573-
}
574-
for may_have_started in [
575-
"supervisor session not connected",
576-
"upstream connect error or disconnect/reset before headers. reset reason: connection termination",
577-
] {
578-
assert!(!retry_unrouted(
579-
Some(&replica),
580-
&tonic::Status::unavailable(may_have_started)
581-
));
582-
}
583-
assert!(!retry_unrouted(
584-
None,
585-
&tonic::Status::unavailable("no healthy upstream")
586-
));
587-
}
471+
use super::tls_server_name;
588472

589473
#[test]
590474
fn tls_server_name_normalizes_bracketed_ipv6_endpoint() {

‎crates/openshell-core/src/replica_routing.rs‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
//! Metadata that lets an edge proxy send a sandbox's long-lived connections
55
//! straight to the gateway replica holding its supervisor session.
66
7+
use tonic::metadata::AsciiMetadataValue;
8+
79
/// Response metadata naming the replica that owns a sandbox's supervisor session.
810
pub const OWNER_REPLICA_HEADER: &str = "x-openshell-owner";
911

@@ -12,3 +14,114 @@ pub const OWNER_REQUEST_HEADER: &str = "x-openshell-want-owner";
1214

1315
/// Request metadata asking the edge proxy to route to a specific replica.
1416
pub const ROUTE_REPLICA_HEADER: &str = "x-openshell-replica";
17+
18+
/// Gateway replica that owns the sandbox named in `response`, when it says.
19+
pub fn owner_replica<T>(response: &tonic::Response<T>) -> Option<AsciiMetadataValue> {
20+
response.metadata().get(OWNER_REPLICA_HEADER).cloned()
21+
}
22+
23+
pub fn with_owner_hint<T>(message: T) -> tonic::Request<T> {
24+
let mut request = tonic::Request::new(message);
25+
request
26+
.metadata_mut()
27+
.insert(OWNER_REQUEST_HEADER, AsciiMetadataValue::from_static("1"));
28+
request
29+
}
30+
31+
/// Wrap `message` so the edge proxy can route it straight to `replica`.
32+
pub fn routed_to<T>(message: T, replica: Option<&AsciiMetadataValue>) -> tonic::Request<T> {
33+
let mut request = tonic::Request::new(message);
34+
if let Some(replica) = replica {
35+
request
36+
.metadata_mut()
37+
.insert(ROUTE_REPLICA_HEADER, replica.clone());
38+
}
39+
request
40+
}
41+
42+
/// A routed call is retried once without routing only when no gateway
43+
/// received it.
44+
///
45+
/// Envoy Gateway answers a replica route whose Service has no ready endpoints
46+
/// with `grpc-status: 14` and no `grpc-message`, while gateway errors and
47+
/// mid-request resets always carry a message.
48+
pub fn retry_unrouted(replica: Option<&AsciiMetadataValue>, status: &tonic::Status) -> bool {
49+
let message = status.message();
50+
replica.is_some()
51+
&& status.code() == tonic::Code::Unavailable
52+
&& (message.is_empty()
53+
|| message.contains("no healthy upstream")
54+
|| message.contains("connection failure"))
55+
}
56+
57+
#[cfg(test)]
58+
mod tests {
59+
use super::*;
60+
61+
#[test]
62+
fn owner_hint_is_requested_explicitly() {
63+
assert!(
64+
with_owner_hint(())
65+
.metadata()
66+
.contains_key(OWNER_REQUEST_HEADER)
67+
);
68+
}
69+
70+
#[test]
71+
fn owner_replica_is_carried_to_the_routed_request() {
72+
let mut response = tonic::Response::new(());
73+
response
74+
.metadata_mut()
75+
.insert(OWNER_REPLICA_HEADER, "openshell-1".parse().unwrap());
76+
let replica = owner_replica(&response);
77+
let request = routed_to((), replica.as_ref());
78+
assert_eq!(
79+
request.metadata().get(ROUTE_REPLICA_HEADER).unwrap(),
80+
"openshell-1"
81+
);
82+
}
83+
84+
#[test]
85+
fn requests_without_a_known_owner_are_not_routed() {
86+
let replica = owner_replica(&tonic::Response::new(()));
87+
assert!(replica.is_none());
88+
let request = routed_to((), replica.as_ref());
89+
assert!(request.metadata().get(ROUTE_REPLICA_HEADER).is_none());
90+
}
91+
92+
#[test]
93+
fn only_unavailable_routed_calls_are_retried() {
94+
let replica = "openshell-1".parse().unwrap();
95+
let unavailable = tonic::Status::unavailable("no healthy upstream");
96+
assert!(retry_unrouted(Some(&replica), &unavailable));
97+
assert!(!retry_unrouted(None, &unavailable));
98+
assert!(!retry_unrouted(
99+
Some(&replica),
100+
&tonic::Status::permission_denied("denied")
101+
));
102+
}
103+
104+
#[test]
105+
fn calls_are_retried_only_when_no_gateway_received_them() {
106+
let replica = "openshell-1".parse().unwrap();
107+
for never_arrived in [
108+
"",
109+
"no healthy upstream",
110+
"upstream connect error or disconnect/reset before headers. reset reason: connection failure, transport failure reason: delayed connect error: Connection refused",
111+
] {
112+
assert!(retry_unrouted(
113+
Some(&replica),
114+
&tonic::Status::unavailable(never_arrived)
115+
));
116+
}
117+
for may_have_started in [
118+
"supervisor session not connected",
119+
"upstream connect error or disconnect/reset before headers. reset reason: connection termination",
120+
] {
121+
assert!(!retry_unrouted(
122+
Some(&replica),
123+
&tonic::Status::unavailable(may_have_started)
124+
));
125+
}
126+
}
127+
}

‎crates/openshell-sdk/src/client.rs‎

Lines changed: 38 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ use crate::error::{Result, SdkError};
1414
use crate::pagination::{Page, Pager};
1515
use crate::raw::AuthedGrpcClient;
1616
use crate::refresh::{RefreshedToken, TokenSource};
17-
use crate::routing::{OwnerCache, owner_replica, retry_unrouted, routed_to, with_owner_hint};
17+
use crate::routing::OwnerCache;
1818
use crate::types::{
1919
DeleteOptions, DeletionResult, ExecOptions, ExecResult, Health, ListOptions, SandboxPhase,
2020
SandboxRef, SandboxSpec, SandboxTemplateCreateSpec, SandboxTemplateListOptions,
@@ -23,6 +23,7 @@ use crate::types::{
2323
use crate::{WatchEvent, WatchOptions, transport};
2424
use futures::{Stream, StreamExt};
2525
use openshell_core::proto;
26+
use openshell_core::replica_routing::{owner_replica, retry_unrouted, routed_to, with_owner_hint};
2627
use std::collections::HashMap;
2728
use std::future::Future;
2829
use std::sync::Arc;
@@ -304,16 +305,7 @@ impl OpenShellClient {
304305

305306
/// Fetch a sandbox by name.
306307
pub async fn get_sandbox(&self, name: &str) -> Result<SandboxRef> {
307-
let response = self
308-
.unary(|mut grpc| {
309-
let request = proto::GetSandboxRequest {
310-
name: name.to_string(),
311-
workspace_scope: Some(proto::workspace_selector("default")),
312-
};
313-
async move { grpc.get_sandbox(request).await }
314-
})
315-
.await?;
316-
sandbox_from_response(response.sandbox)
308+
self.get_sandbox_in("default", name).await
317309
}
318310

319311
/// List sandboxes.
@@ -811,11 +803,31 @@ impl OpenShellClient {
811803
}
812804
}
813805

814-
/// Best effort: a failed lookup leaves the call unrouted.
815-
async fn owner_of(&self, workspace: &str, name: &str) -> Option<AsciiMetadataValue> {
806+
/// Errors that exec would hit anyway are returned; any other failed lookup
807+
/// leaves the call unrouted.
808+
async fn owner_of(&self, workspace: &str, name: &str) -> Result<Option<AsciiMetadataValue>> {
816809
if let Some(cached) = self.owners.get(workspace, name) {
817-
return cached.0;
810+
return Ok(cached.0);
818811
}
812+
match self.get_sandbox_in(workspace, name).await {
813+
Ok(_) => Ok(self.owners.get(workspace, name).and_then(|cached| cached.0)),
814+
Err(err)
815+
if matches!(
816+
err.grpc_status().map(tonic::Status::code),
817+
Some(tonic::Code::Unauthenticated | tonic::Code::Unavailable)
818+
) =>
819+
{
820+
Err(err)
821+
}
822+
Err(err) => {
823+
tracing::debug!(error = %err, "sandbox owner lookup failed; exec is unrouted");
824+
Ok(None)
825+
}
826+
}
827+
}
828+
829+
/// `GetSandbox` that also caches the owner replica the gateway reports.
830+
async fn get_sandbox_in(&self, workspace: &str, name: &str) -> Result<SandboxRef> {
819831
let response = self
820832
.unary_response(|mut grpc| {
821833
let request = with_owner_hint(proto::GetSandboxRequest {
@@ -824,11 +836,10 @@ impl OpenShellClient {
824836
});
825837
async move { grpc.get_sandbox(request).await }
826838
})
827-
.await
828-
.ok()?;
829-
let owner = owner_replica(&response);
830-
self.owners.insert(workspace, name, owner.clone());
831-
owner
839+
.await?;
840+
self.owners
841+
.insert(workspace, name, owner_replica(&response));
842+
sandbox_from_response(response.into_inner().sandbox)
832843
}
833844

834845
/// Open `ExecSandbox` on the owner replica under the unary auth policy.
@@ -837,7 +848,7 @@ impl OpenShellClient {
837848
workspace: &str,
838849
request: proto::ExecSandboxRequest,
839850
) -> Result<tonic::Streaming<proto::ExecSandboxEvent>> {
840-
let owner = self.owner_of(workspace, &request.sandbox).await;
851+
let owner = self.owner_of(workspace, &request.sandbox).await?;
841852
let owners = &self.owners;
842853
self.unary(|mut grpc| {
843854
let request = request.clone();
@@ -851,6 +862,12 @@ impl OpenShellClient {
851862
owners.forget(workspace, &request.sandbox);
852863
grpc.exec_sandbox(request).await
853864
}
865+
Err(status) => {
866+
if owner.is_some() && status.code() == tonic::Code::Unavailable {
867+
owners.forget(workspace, &request.sandbox);
868+
}
869+
Err(status)
870+
}
854871
response => response,
855872
}
856873
}
@@ -1065,17 +1082,7 @@ impl WorkspaceScopedClient {
10651082

10661083
/// Fetch a sandbox by name in this workspace.
10671084
pub async fn get_sandbox(&self, name: &str) -> Result<SandboxRef> {
1068-
let response = self
1069-
.client
1070-
.unary(|mut grpc| {
1071-
let request = proto::GetSandboxRequest {
1072-
name: name.to_string(),
1073-
workspace_scope: Some(proto::workspace_selector(&self.workspace)),
1074-
};
1075-
async move { grpc.get_sandbox(request).await }
1076-
})
1077-
.await?;
1078-
sandbox_from_response(response.sandbox)
1085+
self.client.get_sandbox_in(&self.workspace, name).await
10791086
}
10801087

10811088
/// List sandboxes in this workspace.

0 commit comments

Comments
 (0)