Skip to content

Commit 8d1fd3e

Browse files
committed
feat(api): add durable exec launch admission
Fence duplicate exec launches with keyed durable admission and producer-owned terminal completion. Keep uncertain launches unresolved and never replay output or interactive input. Part of #3051 (phase 4a). Signed-off-by: Mrunal Patel <[email protected]>
1 parent a316fd7 commit 8d1fd3e

13 files changed

Lines changed: 1001 additions & 88 deletions

File tree

‎architecture/gateway.md‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,20 @@ in-memory extension. Replay reauthorizes original and current effective scopes,
9191
requires the same effective payload, and reruns current interceptor validation.
9292
Interceptors cannot mutate the request UUID. Server-marked replay suppresses
9393
post-commit observation, which remains best-effort rather than an outbox.
94-
Credential capabilities and streaming execution require separate contracts.
94+
Credential capabilities require separate contracts.
95+
96+
Both exec RPCs share keyed admission but defer completion to the SSH producer.
97+
The initial interactive Start uses the exec request schema under a distinct RPC
98+
namespace; later stdin and resize frames are not replayable inputs. Admission
99+
precedes relay opening and hands its CAS-only finalizer to the owned producer,
100+
releasing the shared admission-worker permit after handoff. Only a confirmed
101+
remote exit records a terminal marker and starts 24-hour retention. Synthetic
102+
timeouts, disconnects without exit confirmation, and persistence failures leave
103+
permanent unresolved claims. Duplicates never launch, attach, or replay output:
104+
pending records report uncertainty, and terminal records report stream
105+
unavailability. Existing transport cancellation behavior remains unchanged.
106+
Exec timeouts retain the public duration's precision and presence: an absent
107+
timeout is unbounded, while an explicit zero is a finite timeout.
95108

96109
The gateway listens on one service port and multiplexes gRPC and HTTP traffic.
97110
The default local single-user deployment mode is mTLS user authentication:

‎crates/openshell-cli/src/run.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1805,6 +1805,7 @@ pub async fn sandbox_exec_grpc(
18051805
// Make the streaming gRPC call.
18061806
let mut stream = client
18071807
.exec_sandbox(ExecSandboxRequest {
1808+
request_id: String::new(),
18081809
sandbox_id: sandbox.object_id().to_string(),
18091810
command: command.to_vec(),
18101811
workdir: workdir.unwrap_or_default().to_string(),
@@ -2189,6 +2190,7 @@ async fn sandbox_exec_interactive_grpc(
21892190
input_tx
21902191
.send(ExecSandboxInput {
21912192
payload: Some(exec_sandbox_input::Payload::Start(ExecSandboxRequest {
2193+
request_id: String::new(),
21922194
sandbox_id: sandbox.object_id().to_string(),
21932195
command: command.to_vec(),
21942196
workdir: workdir.unwrap_or_default().to_string(),

‎crates/openshell-sdk/src/client.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -608,6 +608,7 @@ impl OpenShellClient {
608608
pub async fn exec(&self, name: &str, cmd: &[String], opts: ExecOptions) -> Result<ExecResult> {
609609
let sandbox = self.get_sandbox(name).await?;
610610
let request = proto::ExecSandboxRequest {
611+
request_id: String::new(),
611612
sandbox_id: sandbox.id,
612613
command: cmd.to_vec(),
613614
workdir: opts.workdir.unwrap_or_default(),
@@ -1068,6 +1069,7 @@ impl WorkspaceScopedClient {
10681069
pub async fn exec(&self, name: &str, cmd: &[String], opts: ExecOptions) -> Result<ExecResult> {
10691070
let sandbox = self.get_sandbox(name).await?;
10701071
let request = proto::ExecSandboxRequest {
1072+
request_id: String::new(),
10711073
sandbox_id: sandbox.id,
10721074
command: cmd.to_vec(),
10731075
workdir: opts.workdir.unwrap_or_default(),

‎crates/openshell-server/src/grpc/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,7 @@ impl OpenShell for OpenShellService {
378378
&self,
379379
request: Request<ExecSandboxRequest>,
380380
) -> Result<Response<Self::ExecSandboxStream>, Status> {
381-
sandbox::handle_exec_sandbox(&self.state, request).await
381+
mutation_replay::run(&self.state, request).await
382382
}
383383

384384
type ForwardTcpStream =

‎crates/openshell-server/src/grpc/mutation_replay.rs‎

Lines changed: 64 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ pub(super) enum Success {
7878
Resource { id: String, version: u64 },
7979
Deletion { outcome: i32 },
8080
Ordinary(ordinary::Outcome),
81+
StreamTerminal,
8182
}
8283

8384
pub(super) struct Scope {
@@ -87,10 +88,15 @@ pub(super) struct Scope {
8788

8889
#[tonic::async_trait]
8990
pub(super) trait Mutation: Message + Default + Send + Sync + 'static {
90-
type Output: Message + Default + Send + 'static;
91+
type Output: Send + 'static;
9192
const METHOD: &'static str;
9293
const PROTECTED: bool = false;
94+
// Streaming producers, not returned stream handles, confirm completion.
95+
const DEFERRED: bool = false;
9396
fn request_id(&self) -> &str;
97+
fn canonical_message(&self) -> Result<DynamicMessage, Status> {
98+
decode_request_message(&format!("{}Request", Self::METHOD), self)
99+
}
94100
async fn authorize(&self, state: &ServerState, principal: &Principal) -> Result<Scope, Status>;
95101
async fn execute(
96102
state: &Arc<ServerState>,
@@ -207,7 +213,7 @@ async fn execute_owned<M: Mutation>(
207213
protection: Option<Protection>,
208214
scope: Scope,
209215
) -> Result<Response<M::Output>, Status> {
210-
let mut admission = Admission {
216+
let admission = Admission {
211217
format_version: 1,
212218
payload_hash,
213219
protection,
@@ -302,32 +308,70 @@ async fn execute_owned<M: Mutation>(
302308
};
303309
// Any error or panic from here leaves the claim unresolved. Status codes
304310
// do not establish that a handler performed no effects.
311+
let completion = Completion {
312+
store: state.store.clone(),
313+
claim_id,
314+
key: key.into(),
315+
bucket: bucket.into(),
316+
version,
317+
admission: payload,
318+
};
319+
if M::DEFERRED {
320+
request.extensions_mut().insert(completion.clone());
321+
}
305322
let facts = ordinary::Facts::default();
306323
request.extensions_mut().insert(facts.clone());
307324
let mut response = M::execute(state, request).await?;
325+
if M::DEFERRED {
326+
return Ok(response);
327+
}
308328
response.extensions_mut().insert(facts);
309-
admission.success = Some(M::capture(&response)?);
329+
completion.complete(M::capture(&response)?).await?;
330+
return Ok(response);
331+
}
332+
Err(uncertain())
333+
}
334+
335+
/// Only the admitted owner can finalize this exact incarnation of a claim.
336+
/// A dropped finalizer leaves a permanent unresolved admission, never a lease.
337+
#[derive(Clone)]
338+
pub(super) struct Completion {
339+
store: Arc<Store>,
340+
claim_id: String,
341+
key: String,
342+
bucket: String,
343+
version: u64,
344+
admission: Vec<u8>,
345+
}
346+
347+
impl Completion {
348+
pub(super) async fn stream_terminal(&self) -> Result<(), Status> {
349+
self.complete(Success::StreamTerminal).await
350+
}
351+
352+
async fn complete(&self, success: Success) -> Result<(), Status> {
353+
let mut admission: Admission =
354+
serde_json::from_slice(&self.admission).map_err(|_| uncertain())?;
355+
admission.success = Some(success);
310356
admission.completed_at_ms = Some(current_time_ms());
311357
let payload = serde_json::to_vec(&admission).map_err(|_| uncertain())?;
312358
if payload.len() > 64 * 1024 {
313359
return Err(uncertain());
314360
}
315-
state
316-
.store
361+
self.store
317362
.put_if(
318363
OBJECT_TYPE,
319-
&claim_id,
320-
key,
321-
bucket,
364+
&self.claim_id,
365+
&self.key,
366+
&self.bucket,
322367
&payload,
323368
None,
324-
WriteCondition::MatchResourceVersion(version),
369+
WriteCondition::MatchResourceVersion(self.version),
325370
)
326371
.await
327372
.map_err(|_| uncertain())?;
328-
return Ok(response);
373+
Ok(())
329374
}
330-
Err(uncertain())
331375
}
332376

333377
async fn prune_expired(store: &Store, bucket: &str) -> Result<bool, Status> {
@@ -377,12 +421,16 @@ fn validate_request_id(value: &str) -> Result<String, Status> {
377421
Ok(id.hyphenated().to_string())
378422
}
379423

380-
fn fingerprint<M: Mutation>(request: &M) -> Result<String, Status> {
424+
fn decode_request_message(name: &str, request: &impl Message) -> Result<DynamicMessage, Status> {
381425
let descriptor = DESCRIPTORS
382-
.get_message_by_name(&format!("openshell.v1.{}Request", M::METHOD))
426+
.get_message_by_name(&format!("openshell.v1.{name}"))
383427
.ok_or_else(|| Status::internal("mutation request descriptor missing"))?;
384-
let mut message = DynamicMessage::decode(descriptor, request.encode_to_vec().as_slice())
385-
.map_err(|_| Status::internal("decode mutation request"))?;
428+
DynamicMessage::decode(descriptor, request.encode_to_vec().as_slice())
429+
.map_err(|_| Status::internal("decode mutation request"))
430+
}
431+
432+
fn fingerprint<M: Mutation>(request: &M) -> Result<String, Status> {
433+
let mut message = request.canonical_message()?;
386434
message.clear_field_by_name("request_id");
387435
let value = serde_json::to_value(message)
388436
.map_err(|_| Status::internal("canonicalize mutation request"))?;
@@ -706,3 +754,4 @@ deletion_mutation!(
706754
mod tests;
707755

708756
pub(super) mod ordinary;
757+
pub(super) mod streaming;
Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Launch admission only: output and interactive input are never replayed.
5+
6+
#[cfg(test)]
7+
mod tests;
8+
9+
use std::sync::Arc;
10+
11+
use openshell_core::ObjectWorkspace;
12+
use openshell_core::proto::{
13+
ExecSandboxEvent, ExecSandboxInput, ExecSandboxRequest, exec_sandbox_input,
14+
};
15+
use openshell_core::rpc_error;
16+
use prost_reflect::DynamicMessage;
17+
use tokio::sync::Mutex;
18+
use tokio_stream::wrappers::ReceiverStream;
19+
use tonic::{Request, Response, Status};
20+
21+
use super::{Mutation, Scope, Success, decode_request_message, named_scope, uncertain};
22+
use crate::ServerState;
23+
use crate::auth::principal::Principal;
24+
use crate::grpc::sandbox;
25+
use crate::persistence::Store;
26+
27+
pub(in crate::grpc) type ExecStream = ReceiverStream<Result<ExecSandboxEvent, Status>>;
28+
29+
/// Private, one-owner transport handoff. The fingerprint includes only Start.
30+
#[derive(Clone)]
31+
pub(in crate::grpc) struct InteractiveInput(
32+
pub Arc<Mutex<Option<tonic::Streaming<ExecSandboxInput>>>>,
33+
);
34+
35+
pub(in crate::grpc) fn start(input: &ExecSandboxInput) -> Result<&ExecSandboxRequest, Status> {
36+
match input.payload.as_ref() {
37+
Some(exec_sandbox_input::Payload::Start(request)) => Ok(request),
38+
_ => Err(rpc_error::invalid_argument(
39+
"start",
40+
"first message must be a start payload",
41+
)),
42+
}
43+
}
44+
45+
async fn authorize(
46+
state: &ServerState,
47+
principal: &Principal,
48+
request: &ExecSandboxRequest,
49+
) -> Result<Scope, Status> {
50+
sandbox::validate_exec_start(request)?;
51+
let sandbox =
52+
sandbox::fetch_and_authorize_sandbox(state, principal, &request.sandbox_id).await?;
53+
named_scope(state, sandbox.object_workspace()).await
54+
}
55+
56+
fn stream_unavailable(success: Success) -> Status {
57+
if !matches!(success, Success::StreamTerminal) {
58+
return super::replay_unavailable();
59+
}
60+
rpc_error::failed_precondition(
61+
"REQUEST_STREAM_UNAVAILABLE",
62+
"execution terminated, but its output stream is not stored; this request was not launched again",
63+
)
64+
}
65+
66+
#[tonic::async_trait]
67+
impl Mutation for ExecSandboxRequest {
68+
type Output = ExecStream;
69+
const METHOD: &'static str = "ExecSandbox";
70+
const PROTECTED: bool = true;
71+
const DEFERRED: bool = true;
72+
73+
fn request_id(&self) -> &str {
74+
&self.request_id
75+
}
76+
77+
async fn authorize(&self, state: &ServerState, principal: &Principal) -> Result<Scope, Status> {
78+
authorize(state, principal, self).await
79+
}
80+
81+
async fn execute(
82+
state: &Arc<ServerState>,
83+
request: Request<Self>,
84+
) -> Result<Response<Self::Output>, Status> {
85+
sandbox::handle_exec_sandbox(state, request).await
86+
}
87+
88+
fn capture(_: &Response<Self::Output>) -> Result<Success, Status> {
89+
Err(uncertain())
90+
}
91+
92+
async fn restore(_: &Store, success: Success) -> Result<Self::Output, Status> {
93+
Err(stream_unavailable(success))
94+
}
95+
}
96+
97+
#[tonic::async_trait]
98+
impl Mutation for ExecSandboxInput {
99+
type Output = ExecStream;
100+
const METHOD: &'static str = "ExecSandboxInteractive";
101+
const PROTECTED: bool = true;
102+
const DEFERRED: bool = true;
103+
104+
fn request_id(&self) -> &str {
105+
start(self).map_or("", |request| request.request_id.as_str())
106+
}
107+
108+
fn canonical_message(&self) -> Result<DynamicMessage, Status> {
109+
decode_request_message("ExecSandboxRequest", start(self)?)
110+
}
111+
112+
async fn authorize(&self, state: &ServerState, principal: &Principal) -> Result<Scope, Status> {
113+
authorize(state, principal, start(self)?).await
114+
}
115+
116+
async fn execute(
117+
state: &Arc<ServerState>,
118+
request: Request<Self>,
119+
) -> Result<Response<Self::Output>, Status> {
120+
sandbox::handle_exec_sandbox_interactive_start(state, request).await
121+
}
122+
123+
fn capture(_: &Response<Self::Output>) -> Result<Success, Status> {
124+
Err(uncertain())
125+
}
126+
127+
async fn restore(_: &Store, success: Success) -> Result<Self::Output, Status> {
128+
Err(stream_unavailable(success))
129+
}
130+
}

0 commit comments

Comments
 (0)