Skip to content

Commit bdffa10

Browse files
authored
feat(api): add durable exec launch admission (NVIDIA#3324)
* feat(api): add durable exec launch admission Fence duplicate exec launches with keyed durable admission and producer-owned terminal completion. Keep uncertain launches unresolved and never replay output or interactive input. Part of NVIDIA#3051 (phase 4a). Signed-off-by: Mrunal Patel <[email protected]> * fix(api): fence exec identity across authorization lookups Signed-off-by: Mrunal Patel <[email protected]> --------- Signed-off-by: Mrunal Patel <[email protected]>
1 parent 1e34e8c commit bdffa10

14 files changed

Lines changed: 1372 additions & 89 deletions

File tree

‎architecture/gateway.md‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,25 @@ in-memory extension. Replay reauthorizes original and current effective scopes,
9595
requires the same effective payload, and reruns current interceptor validation.
9696
Interceptors cannot mutate the request UUID. Server-marked replay suppresses
9797
post-commit observation, which remains best-effort rather than an outbox.
98-
Credential capabilities and streaming execution require separate contracts.
98+
Credential capabilities require separate contracts.
99+
100+
Both exec RPCs share keyed admission but defer completion to the SSH producer.
101+
The initial interactive Start uses the exec request schema under a distinct RPC
102+
namespace; later stdin and resize frames are not replayable inputs. Admission
103+
resolves the public sandbox name and workspace, durably binds both original and
104+
effective sandbox UUIDs, and rejects same-name replacements. When the original
105+
and effective selectors match, both authorization lookups must resolve the same
106+
workspace and sandbox identities before admission. A different effective target
107+
is allowed only when the selector changes. The owner checks the effective UUID
108+
again before relay opening, then hands its CAS-only finalizer to the owned producer,
109+
releasing the shared admission-worker permit after handoff. Only a confirmed
110+
remote exit records a terminal marker and starts 24-hour retention. Synthetic
111+
timeouts, disconnects without exit confirmation, and persistence failures leave
112+
permanent unresolved claims. Duplicates never launch, attach, or replay output:
113+
pending records report uncertainty, and terminal records report stream
114+
unavailability. Existing transport cancellation behavior remains unchanged.
115+
Exec timeouts retain the public duration's precision and presence: an absent
116+
timeout is unbounded, while an explicit zero is a finite timeout.
99117

100118
The gateway listens on one service port and multiplexes gRPC and HTTP traffic.
101119
The default local single-user deployment mode is mTLS user authentication:

‎crates/openshell-cli/src/run.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1897,6 +1897,7 @@ pub async fn sandbox_exec_grpc(
18971897
// Make the streaming gRPC call.
18981898
let mut stream = client
18991899
.exec_sandbox(ExecSandboxRequest {
1900+
request_id: String::new(),
19001901
sandbox: name.to_string(),
19011902
workspace_scope: Some(openshell_core::proto::workspace_selector(
19021903
workspace.to_string(),
@@ -2300,6 +2301,7 @@ async fn sandbox_exec_interactive_grpc(
23002301
input_tx
23012302
.send(ExecSandboxInput {
23022303
payload: Some(exec_sandbox_input::Payload::Start(ExecSandboxRequest {
2304+
request_id: String::new(),
23032305
sandbox: sandbox.object_name().to_string(),
23042306
workspace_scope: Some(openshell_core::proto::workspace_selector(
23052307
(sandbox.object_workspace()).to_string(),

‎crates/openshell-sdk/src/client.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -607,6 +607,7 @@ impl OpenShellClient {
607607
/// call `exec_sandbox` directly.
608608
pub async fn exec(&self, name: &str, cmd: &[String], opts: ExecOptions) -> Result<ExecResult> {
609609
let request = proto::ExecSandboxRequest {
610+
request_id: String::new(),
610611
sandbox: name.to_string(),
611612
workspace_scope: Some(proto::workspace_selector("default")),
612613
command: cmd.to_vec(),
@@ -1185,6 +1186,7 @@ impl WorkspaceScopedClient {
11851186
/// Run a command inside a sandbox and buffer stdout/stderr.
11861187
pub async fn exec(&self, name: &str, cmd: &[String], opts: ExecOptions) -> Result<ExecResult> {
11871188
let request = proto::ExecSandboxRequest {
1189+
request_id: String::new(),
11881190
sandbox: name.to_string(),
11891191
workspace_scope: Some(proto::workspace_selector(&self.workspace)),
11901192
command: cmd.to_vec(),

‎crates/openshell-server/src/grpc/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -430,7 +430,7 @@ impl OpenShell for OpenShellService {
430430
&self,
431431
request: Request<ExecSandboxRequest>,
432432
) -> Result<Response<Self::ExecSandboxStream>, Status> {
433-
sandbox::handle_exec_sandbox(&self.state, request).await
433+
mutation_replay::run(&self.state, request).await
434434
}
435435

436436
type ForwardTcpStream =

‎crates/openshell-server/src/grpc/mutation_replay.rs‎

Lines changed: 114 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ struct Admission {
5656
#[serde(default)]
5757
protection: Option<Protection>,
5858
workspace_id: Option<String>,
59+
#[serde(default)]
60+
target_id: Option<String>,
5961
success: Option<Success>,
6062
completed_at_ms: Option<i64>,
6163
}
@@ -65,6 +67,8 @@ struct Protection {
6567
key_id: String,
6668
effective_payload_hash: String,
6769
effective_workspace_id: Option<String>,
70+
#[serde(default)]
71+
effective_target_id: Option<String>,
6872
}
6973

7074
/// Gateway-owned, in-memory input before hydration and interceptor modification.
@@ -78,19 +82,39 @@ pub(super) enum Success {
7882
Resource { id: String, version: u64 },
7983
Deletion { outcome: i32 },
8084
Ordinary(ordinary::Outcome),
85+
StreamTerminal,
8186
}
8287

8388
pub(super) struct Scope {
8489
name: String,
8590
workspace_id: Option<String>,
91+
// Name-addressed exec requests must remain bound to one sandbox instance.
92+
target_id: Option<String>,
93+
}
94+
95+
#[cfg(test)]
96+
#[derive(Default)]
97+
struct AuthorizationBarrier {
98+
resolved: tokio::sync::Notify,
99+
resume: tokio::sync::Notify,
86100
}
87101

88102
#[tonic::async_trait]
89103
pub(super) trait Mutation: Message + Default + Send + Sync + 'static {
90-
type Output: Message + Default + Send + 'static;
104+
type Output: Send + 'static;
91105
const METHOD: &'static str;
92106
const PROTECTED: bool = false;
107+
// Streaming producers, not returned stream handles, confirm completion.
108+
const DEFERRED: bool = false;
93109
fn request_id(&self) -> &str;
110+
/// Name-addressed targets must keep their identity unless an interceptor
111+
/// actually changes the selector, not merely another request field.
112+
fn target_selector(&self) -> Option<(&str, &WorkspaceSelector)> {
113+
None
114+
}
115+
fn canonical_message(&self) -> Result<DynamicMessage, Status> {
116+
decode_request_message(&format!("{}Request", Self::METHOD), self)
117+
}
94118
async fn authorize(&self, state: &ServerState, principal: &Principal) -> Result<Scope, Status>;
95119
async fn execute(
96120
state: &Arc<ServerState>,
@@ -132,7 +156,19 @@ pub(super) async fn run<M: Mutation>(
132156
));
133157
};
134158
let scope = original.authorize(state, &principal).await?;
159+
#[cfg(test)]
160+
if let Some(barrier) = request.extensions().get::<Arc<AuthorizationBarrier>>() {
161+
barrier.resolved.notify_one();
162+
barrier.resume.notified().await;
163+
}
135164
let effective_scope = request.get_ref().authorize(state, &principal).await?;
165+
if original.target_selector().is_some()
166+
&& original.target_selector() == request.get_ref().target_selector()
167+
&& (scope.workspace_id != effective_scope.workspace_id
168+
|| scope.target_id != effective_scope.target_id)
169+
{
170+
return Err(replay_unavailable());
171+
}
136172
let (payload_hash, protection) = if M::PROTECTED {
137173
let key = fingerprint_key(state).await?;
138174
(
@@ -141,6 +177,7 @@ pub(super) async fn run<M: Mutation>(
141177
key_id: key.id(),
142178
effective_payload_hash: key.fingerprint(request.get_ref())?,
143179
effective_workspace_id: effective_scope.workspace_id,
180+
effective_target_id: effective_scope.target_id,
144181
}),
145182
)
146183
} else {
@@ -207,11 +244,12 @@ async fn execute_owned<M: Mutation>(
207244
protection: Option<Protection>,
208245
scope: Scope,
209246
) -> Result<Response<M::Output>, Status> {
210-
let mut admission = Admission {
247+
let admission = Admission {
211248
format_version: 1,
212249
payload_hash,
213250
protection,
214251
workspace_id: scope.workspace_id,
252+
target_id: scope.target_id,
215253
success: None,
216254
completed_at_ms: None,
217255
};
@@ -256,6 +294,7 @@ async fn execute_owned<M: Mutation>(
256294
));
257295
}
258296
if previous.workspace_id != admission.workspace_id
297+
|| previous.target_id != admission.target_id
259298
|| previous.protection != admission.protection
260299
{
261300
return Err(replay_unavailable());
@@ -302,32 +341,85 @@ async fn execute_owned<M: Mutation>(
302341
};
303342
// Any error or panic from here leaves the claim unresolved. Status codes
304343
// do not establish that a handler performed no effects.
344+
let completion = Completion {
345+
store: state.store.clone(),
346+
claim_id,
347+
key: key.into(),
348+
bucket: bucket.into(),
349+
version,
350+
admission: payload,
351+
};
352+
if M::DEFERRED {
353+
request.extensions_mut().insert(completion.clone());
354+
}
305355
let facts = ordinary::Facts::default();
306356
request.extensions_mut().insert(facts.clone());
307357
let mut response = M::execute(state, request).await?;
358+
if M::DEFERRED {
359+
return Ok(response);
360+
}
308361
response.extensions_mut().insert(facts);
309-
admission.success = Some(M::capture(&response)?);
362+
completion.complete(M::capture(&response)?).await?;
363+
return Ok(response);
364+
}
365+
Err(uncertain())
366+
}
367+
368+
/// Only the admitted owner can finalize this exact incarnation of a claim.
369+
/// A dropped finalizer leaves a permanent unresolved admission, never a lease.
370+
#[derive(Clone)]
371+
pub(super) struct Completion {
372+
store: Arc<Store>,
373+
claim_id: String,
374+
key: String,
375+
bucket: String,
376+
version: u64,
377+
admission: Vec<u8>,
378+
}
379+
380+
impl Completion {
381+
/// Reject replacement between admission authorization and relay creation.
382+
pub(super) fn ensure_target(&self, target_id: &str) -> Result<(), Status> {
383+
let admission: Admission =
384+
serde_json::from_slice(&self.admission).map_err(|_| uncertain())?;
385+
if admission
386+
.protection
387+
.as_ref()
388+
.and_then(|protection| protection.effective_target_id.as_deref())
389+
!= Some(target_id)
390+
{
391+
return Err(uncertain());
392+
}
393+
Ok(())
394+
}
395+
396+
pub(super) async fn stream_terminal(&self) -> Result<(), Status> {
397+
self.complete(Success::StreamTerminal).await
398+
}
399+
400+
async fn complete(&self, success: Success) -> Result<(), Status> {
401+
let mut admission: Admission =
402+
serde_json::from_slice(&self.admission).map_err(|_| uncertain())?;
403+
admission.success = Some(success);
310404
admission.completed_at_ms = Some(current_time_ms());
311405
let payload = serde_json::to_vec(&admission).map_err(|_| uncertain())?;
312406
if payload.len() > 64 * 1024 {
313407
return Err(uncertain());
314408
}
315-
state
316-
.store
409+
self.store
317410
.put_if(
318411
OBJECT_TYPE,
319-
&claim_id,
320-
key,
321-
bucket,
412+
&self.claim_id,
413+
&self.key,
414+
&self.bucket,
322415
&payload,
323416
None,
324-
WriteCondition::MatchResourceVersion(version),
417+
WriteCondition::MatchResourceVersion(self.version),
325418
)
326419
.await
327420
.map_err(|_| uncertain())?;
328-
return Ok(response);
421+
Ok(())
329422
}
330-
Err(uncertain())
331423
}
332424

333425
async fn prune_expired(store: &Store, bucket: &str) -> Result<bool, Status> {
@@ -377,12 +469,16 @@ fn validate_request_id(value: &str) -> Result<String, Status> {
377469
Ok(id.hyphenated().to_string())
378470
}
379471

380-
fn fingerprint<M: Mutation>(request: &M) -> Result<String, Status> {
472+
fn decode_request_message(name: &str, request: &impl Message) -> Result<DynamicMessage, Status> {
381473
let descriptor = DESCRIPTORS
382-
.get_message_by_name(&format!("openshell.v1.{}Request", M::METHOD))
474+
.get_message_by_name(&format!("openshell.v1.{name}"))
383475
.ok_or_else(|| Status::internal("mutation request descriptor missing"))?;
384-
let mut message = DynamicMessage::decode(descriptor, request.encode_to_vec().as_slice())
385-
.map_err(|_| Status::internal("decode mutation request"))?;
476+
DynamicMessage::decode(descriptor, request.encode_to_vec().as_slice())
477+
.map_err(|_| Status::internal("decode mutation request"))
478+
}
479+
480+
fn fingerprint<M: Mutation>(request: &M) -> Result<String, Status> {
481+
let mut message = request.canonical_message()?;
386482
message.clear_field_by_name("request_id");
387483
let value = serde_json::to_value(message)
388484
.map_err(|_| Status::internal("canonicalize mutation request"))?;
@@ -492,6 +588,7 @@ async fn named_scope(state: &ServerState, name: &str) -> Result<Scope, Status> {
492588
Ok(Scope {
493589
name: name.into(),
494590
workspace_id: Some(resource.object_id().into()),
591+
target_id: None,
495592
})
496593
}
497594

@@ -500,6 +597,7 @@ fn global_scope(state: &ServerState, principal: &Principal) -> Result<Scope, Sta
500597
Ok(Scope {
501598
name: String::new(),
502599
workspace_id: None,
600+
target_id: None,
503601
})
504602
}
505603

@@ -719,3 +817,4 @@ deletion_mutation!(
719817
mod tests;
720818

721819
pub(super) mod ordinary;
820+
pub(super) mod streaming;

0 commit comments

Comments
 (0)