Skip to content

Commit 39cf482

Browse files
authored
feat(api): add structured gateway errors and SDK decoding (NVIDIA#3313)
* feat(api): expose structured gateway errors across SDKs Refs NVIDIA#3051. Add standard validation, conflict, and retry details; preserve raw transport status in Rust, Go, TypeScript, and Python; document status and recovery guidance. This is the structured-error foundation only. Mutation result shapes, allow_missing, durable request deduplication, and exec retry semantics remain follow-up work. Signed-off-by: Mrunal Patel <[email protected]> * fix(python): preserve wrapped RPC cleanup handling Inspect the original gRPC call when handling missing sandboxes during deletion waits and managed cleanup. Add intercepted cleanup regressions and clarify the error-wrapper migration contract. Addresses the cleanup review on NVIDIA#3313; part of NVIDIA#3051. Signed-off-by: Mrunal Patel <[email protected]> --------- Signed-off-by: Mrunal Patel <[email protected]>
1 parent b799fcc commit 39cf482

32 files changed

Lines changed: 1355 additions & 179 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ tokio = { version = "1.43", features = ["full"] }
1818

1919
# gRPC/Protobuf
2020
tonic = "0.14"
21+
tonic-types = "0.14"
2122
tonic-prost = "0.14"
2223
tonic-prost-build = "0.14"
2324
prost = "0.14"

‎architecture/gateway.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,15 @@ The gateway validates this requirement before constructing the selected driver.
5353

5454
## Protocol and Auth
5555

56+
Gateway validation and concurrency errors use the standard rich gRPC error
57+
envelope. Shared field validators attach `google.rpc.BadRequest`, and conditional
58+
write conflicts attach `google.rpc.ErrorInfo` with a stable reason and current
59+
version when available. `google.rpc.RetryInfo` expresses a minimum retry delay;
60+
it does not establish that a mutation is safe to repeat. SDKs retain the original
61+
transport status, metadata, and unknown details alongside decoded fields.
62+
Python cleanup inspects the original gRPC call beneath a typed error wrapper,
63+
preserving missing-resource handling without suppressing other failures.
64+
5665
The gateway listens on one service port and multiplexes gRPC and HTTP traffic.
5766
The default local single-user deployment mode is mTLS user authentication:
5867
clients present a certificate signed by the local deployment CA, and the

‎crates/openshell-core/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ prost = { workspace = true }
1818
prost-types = { workspace = true }
1919
tonic = { workspace = true, features = ["channel", "tls-aws-lc"] }
2020
tonic-prost = { workspace = true }
21+
tonic-types = { workspace = true }
2122
tokio = { workspace = true }
2223
tokio-stream = { workspace = true }
2324
thiserror = { workspace = true }

‎crates/openshell-core/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ pub mod proposals;
4444
pub mod proto;
4545
pub mod proto_struct;
4646
pub mod provider_credentials;
47+
pub mod rpc_error;
4748
pub mod sandbox_env;
4849
pub mod secrets;
4950
pub mod settings;
Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Standard gRPC error details shared by gateway handlers and SDK clients.
5+
//!
6+
//! Retry guidance describes recovery from a failure. It never establishes that
7+
//! repeating an arbitrary mutation is safe.
8+
9+
use std::collections::HashMap;
10+
use std::time::Duration;
11+
12+
use prost::Message;
13+
use tonic::{Code, Status};
14+
pub use tonic_types::{ErrorDetails, StatusExt};
15+
16+
/// Domain of gateway-owned machine-readable error reasons.
17+
pub const ERROR_DOMAIN: &str = "openshell.nvidia.com";
18+
19+
/// Decode supported details without trusting inconsistent envelopes or delays.
20+
/// Unknown and malformed details remain available on the original status.
21+
pub fn decode_details(status: &Status) -> Option<ErrorDetails> {
22+
let envelope = tonic_types::pb::Status::decode(status.details()).ok()?;
23+
if envelope.code != status.code() as i32 || envelope.message != status.message() {
24+
return None;
25+
}
26+
let mut details = ErrorDetails::new();
27+
for detail in envelope.details {
28+
match detail.type_url.rsplit('/').next() {
29+
Some("google.rpc.BadRequest") => {
30+
if let Ok(value) = tonic_types::pb::BadRequest::decode(detail.value.as_slice()) {
31+
for violation in value.field_violations {
32+
details.add_bad_request_violation(violation.field, violation.description);
33+
}
34+
}
35+
}
36+
Some("google.rpc.ErrorInfo") => {
37+
if let Ok(value) = tonic_types::pb::ErrorInfo::decode(detail.value.as_slice()) {
38+
details.set_error_info(value.reason, value.domain, value.metadata);
39+
}
40+
}
41+
Some("google.rpc.RetryInfo") => {
42+
if let Ok(value) = tonic_types::pb::RetryInfo::decode(detail.value.as_slice())
43+
&& let Some(delay) = value.retry_delay
44+
&& (0..=315_576_000_000).contains(&delay.seconds)
45+
&& (0..1_000_000_000).contains(&delay.nanos)
46+
{
47+
details.set_retry_info(Some(Duration::new(
48+
delay.seconds.cast_unsigned(),
49+
delay.nanos.cast_unsigned(),
50+
)));
51+
}
52+
}
53+
_ => {}
54+
}
55+
}
56+
Some(details)
57+
}
58+
59+
/// Return a field-level validation failure without reflecting the field value.
60+
pub fn invalid_argument(field: &str, message: impl Into<String>) -> Status {
61+
let message = message.into();
62+
let mut details = ErrorDetails::with_bad_request_violation(field, message.clone());
63+
details.set_error_info("INVALID_ARGUMENT", ERROR_DOMAIN, HashMap::new());
64+
Status::with_error_details(Code::InvalidArgument, message, details)
65+
}
66+
67+
/// Return a precondition failure with a stable reason.
68+
pub fn failed_precondition(reason: &str, message: impl Into<String>) -> Status {
69+
Status::with_error_details(
70+
Code::FailedPrecondition,
71+
message,
72+
ErrorDetails::with_error_info(reason, ERROR_DOMAIN, HashMap::new()),
73+
)
74+
}
75+
76+
/// A conditional write lost a race. Read fresh state before trying a new write.
77+
pub fn resource_version_conflict(message: impl Into<String>, version: Option<u64>) -> Status {
78+
let mut metadata = HashMap::from([("recovery".into(), "REFRESH_STATE".into())]);
79+
if let Some(version) = version {
80+
metadata.insert("current_resource_version".into(), version.to_string());
81+
}
82+
Status::with_error_details(
83+
Code::Aborted,
84+
message,
85+
ErrorDetails::with_error_info("RESOURCE_VERSION_CONFLICT", ERROR_DOMAIN, metadata),
86+
)
87+
}
88+
89+
/// Return a transient failure with a minimum delay for retry-safe operations.
90+
pub fn unavailable(reason: &str, message: impl Into<String>, delay: Duration) -> Status {
91+
let mut details = ErrorDetails::with_error_info(reason, ERROR_DOMAIN, HashMap::new());
92+
details.set_retry_info(Some(delay));
93+
Status::with_error_details(Code::Unavailable, message, details)
94+
}
95+
96+
#[cfg(test)]
97+
mod tests {
98+
use super::*;
99+
100+
#[test]
101+
fn decode_ignores_invalid_delay_and_preserves_other_details() {
102+
let status = invalid_argument("name", "invalid name");
103+
let mut envelope = tonic_types::pb::Status::decode(status.details()).unwrap();
104+
envelope.details.push(prost_types::Any {
105+
type_url: "type.googleapis.com/google.rpc.RetryInfo".into(),
106+
value: tonic_types::pb::RetryInfo {
107+
retry_delay: Some(prost_types::Duration {
108+
seconds: -1,
109+
nanos: 0,
110+
}),
111+
}
112+
.encode_to_vec(),
113+
});
114+
envelope.details.push(prost_types::Any {
115+
type_url: "type.googleapis.com/google.rpc.BadRequest".into(),
116+
value: vec![255],
117+
});
118+
let status = Status::with_details(
119+
status.code(),
120+
status.message(),
121+
envelope.encode_to_vec().into(),
122+
);
123+
let details = decode_details(&status).unwrap();
124+
assert!(details.retry_info().is_none());
125+
assert_eq!(
126+
details.bad_request().unwrap().field_violations[0].field,
127+
"name"
128+
);
129+
let mismatched = Status::with_details(
130+
Code::Internal,
131+
status.message(),
132+
status.details().to_vec().into(),
133+
);
134+
assert!(decode_details(&mismatched).is_none());
135+
}
136+
137+
#[test]
138+
fn validation_identifies_the_field_and_reason() {
139+
let status = invalid_argument("spec.command[0]", "must not be empty");
140+
let details = status.get_error_details();
141+
assert_eq!(status.code(), Code::InvalidArgument);
142+
let violation = &details.bad_request().unwrap().field_violations[0];
143+
assert_eq!(violation.field, "spec.command[0]");
144+
assert_eq!(violation.description, "must not be empty");
145+
assert_eq!(details.error_info().unwrap().domain, ERROR_DOMAIN);
146+
assert!(details.retry_info().is_none());
147+
}
148+
149+
#[test]
150+
fn conflict_requires_fresh_state_without_blind_retry_delay() {
151+
let status = resource_version_conflict("concurrent write", Some(17));
152+
let details = status.get_error_details();
153+
let info = details.error_info().unwrap();
154+
assert_eq!(status.code(), Code::Aborted);
155+
assert_eq!(info.reason, "RESOURCE_VERSION_CONFLICT");
156+
assert_eq!(info.metadata["recovery"], "REFRESH_STATE");
157+
assert_eq!(info.metadata["current_resource_version"], "17");
158+
assert!(details.retry_info().is_none());
159+
}
160+
161+
#[test]
162+
fn temporary_unavailability_has_a_reason_and_delay() {
163+
let status = unavailable("GATEWAY_NOT_READY", "try later", Duration::from_secs(1));
164+
let details = status.get_error_details();
165+
assert_eq!(status.code(), Code::Unavailable);
166+
assert_eq!(details.error_info().unwrap().reason, "GATEWAY_NOT_READY");
167+
assert_eq!(
168+
details.retry_info().unwrap().retry_delay,
169+
Some(Duration::from_secs(1))
170+
);
171+
}
172+
}

‎crates/openshell-sdk/src/client.rs‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1179,17 +1179,7 @@ fn sandbox_template_from_response(
11791179
}
11801180

11811181
fn map_status(status: tonic::Status) -> SdkError {
1182-
let message = status.message().to_string();
1183-
match status.code() {
1184-
tonic::Code::NotFound => SdkError::NotFound { message },
1185-
tonic::Code::AlreadyExists => SdkError::AlreadyExists { message },
1186-
tonic::Code::InvalidArgument => SdkError::invalid_config(message),
1187-
tonic::Code::Unauthenticated | tonic::Code::PermissionDenied => SdkError::auth(message),
1188-
_ => SdkError::Rpc {
1189-
code: status.code() as i32,
1190-
message,
1191-
},
1192-
}
1182+
SdkError::from_status(status)
11931183
}
11941184

11951185
#[cfg(test)]

0 commit comments

Comments
 (0)