Skip to content

Commit 449ef90

Browse files
committed
fix(supervisor): report bounded backend startup failures
Classify Build, Attach and Confirm failures with fixed public diagnostics and a bounded reporting budget while preserving the original startup error. Closes #5107 Depends on corrected PR #4791. Signed-off-by: Shiju <[email protected]>
1 parent 5c45561 commit 449ef90

8 files changed

Lines changed: 310 additions & 80 deletions

File tree

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Closed startup failure codes carried by pre-start configuration refusals.
5+
//!
6+
//! The supervisor reports a code in the existing admission error field. The
7+
//! gateway projects only these fixed diagnostics, never backend error text.
8+
9+
/// The trusted operation that failed before workload execution.
10+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11+
pub enum BackendStartupStage {
12+
/// Constructing the selected backend client.
13+
Build,
14+
/// Attaching the admitted runtime generation.
15+
Attach,
16+
/// Confirming the isolation boundary before starting workload code.
17+
Confirm,
18+
}
19+
20+
impl BackendStartupStage {
21+
/// Stable token used with the existing backend error-kind classification.
22+
#[must_use]
23+
pub const fn code(self) -> &'static str {
24+
match self {
25+
Self::Build => "build",
26+
Self::Attach => "attach",
27+
Self::Confirm => "confirm",
28+
}
29+
}
30+
}
31+
32+
/// Project an exact startup failure code into a bounded, credential-free
33+
/// diagnostic. Unknown and legacy error strings have no projection.
34+
///
35+
/// Only use this projection for an authenticated, generation-fenced
36+
/// `refused_before_start` report; ordinary runtime errors are not startup claims.
37+
#[must_use]
38+
pub fn backend_startup_diagnostic(code: &str) -> Option<String> {
39+
let (stage, kind) = code.strip_prefix("openshell.backend.")?.split_once('.')?;
40+
let stage = match stage {
41+
"build" => "client construction",
42+
"attach" => "attachment",
43+
"confirm" => "isolation confirmation",
44+
_ => return None,
45+
};
46+
let recovery = match kind {
47+
"invalid" => {
48+
"the launch descriptor or admitted identity did not match; check the installation and recreate the sandbox"
49+
}
50+
"denied" => {
51+
"the backend rejected the admitted policy or identity; review the policy and recreate the sandbox"
52+
}
53+
"unavailable" => {
54+
"the backend was unavailable; check its service and host qualification, then retry after cleanup"
55+
}
56+
"unsupported" => {
57+
"the backend does not support the requested operation; choose a compatible backend configuration"
58+
}
59+
"failed" => {
60+
"the backend could not establish isolation; check the installation and host qualification before retrying after cleanup"
61+
}
62+
"terminated" => {
63+
"the isolation boundary was lost before workload execution; check the backend service and recreate the sandbox"
64+
}
65+
_ => return None,
66+
};
67+
Some(format!("Isolation backend {stage} failed: {recovery}"))
68+
}

‎crates/openshell-core/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
1212
pub mod activity;
1313
pub mod auth;
14+
pub mod backend_startup;
1415
pub mod compute;
1516
pub mod config;
1617
pub mod container_paths;

‎crates/openshell-server/src/grpc/policy.rs‎

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5075,6 +5075,9 @@ pub(super) async fn handle_report_sandbox_configuration(
50755075
"refused_before_start requires a rejected admission",
50765076
));
50775077
}
5078+
let startup_diagnostic = refused_before_start
5079+
.then(|| openshell_core::backend_startup::backend_startup_diagnostic(&admission.error))
5080+
.flatten();
50785081
let Some(_mutation_guard) = state
50795082
.compute
50805083
.sandbox_mutation_guard_by_id(&sandbox_id)
@@ -5156,14 +5159,17 @@ pub(super) async fn handle_report_sandbox_configuration(
51565159
Ok(config) if !config.get_ref().configuration_error.is_empty() => config.into_inner().configuration_error,
51575160
_ => "Effective configuration could not be activated; replace the policy or repair attached providers".to_string(),
51585161
};
5159-
if let Some(current) = current
5160-
&& current.state == i32::from(ConfigurationAdmissionState::Accepted)
5161-
{
5162-
if refused_before_start && same_generation(current, &admission) {
5163-
// The runtime refused what it accepted, so the workload never
5164-
// started on it. Runtime text stays out of public status.
5165-
admission.error = "The isolation backend refused this configuration before the workload started; create the sandbox again with a policy the backend accepts".to_string();
5166-
} else {
5162+
if let Some(current) = current {
5163+
let accepted = current.state == i32::from(ConfigurationAdmissionState::Accepted);
5164+
let rejected = current.state == i32::from(ConfigurationAdmissionState::Rejected);
5165+
if refused_before_start
5166+
&& (accepted || rejected)
5167+
&& same_generation(current, &admission)
5168+
{
5169+
// A retry of the same failed generation retains its diagnostic.
5170+
// Only exact known codes are projected; legacy/raw text stays out.
5171+
admission.error = startup_diagnostic.unwrap_or_else(|| "The isolation backend refused this configuration before the workload started; create the sandbox again with a policy the backend accepts".to_string());
5172+
} else if accepted {
51675173
// A rejected desired update does not invalidate an accepted runtime.
51685174
let error = admission.error;
51695175
admission = current.clone();
@@ -9105,7 +9111,8 @@ mod tests {
91059111
assert!(same_generation(&refused, &refused));
91069112
assert!(!same_generation(&refused, &other_generation));
91079113

9108-
handle_report_sandbox_configuration(&state, report(refused, true))
9114+
refused.error = "openshell.backend.confirm.denied".to_string();
9115+
handle_report_sandbox_configuration(&state, report(refused.clone(), true))
91099116
.await
91109117
.unwrap();
91119118
let (admission_state, ready) = readiness().await;
@@ -9115,10 +9122,24 @@ mod tests {
91159122
("False", "ConfigurationInvalid")
91169123
);
91179124
assert!(
9118-
ready.message.contains("isolation backend refused"),
9125+
ready.message.contains("isolation confirmation"),
91199126
"{ready:?}"
91209127
);
91219128
assert!(!ready.message.contains("backend text"), "{ready:?}");
9129+
let first_message = ready.message;
9130+
handle_report_sandbox_configuration(&state, report(refused.clone(), true))
9131+
.await
9132+
.unwrap();
9133+
let (_, repeated) = readiness().await;
9134+
assert_eq!(repeated.message, first_message);
9135+
9136+
refused.error = "openshell.backend.confirm.denied\nBearer secret-value".to_string();
9137+
handle_report_sandbox_configuration(&state, report(refused, true))
9138+
.await
9139+
.unwrap();
9140+
let (_, unknown) = readiness().await;
9141+
assert!(!unknown.message.contains("secret-value"));
9142+
assert!(unknown.message.contains("isolation backend refused"));
91229143
}
91239144

91249145
#[test]

‎crates/openshell-supervisor/src/backend_setup.rs‎

Lines changed: 58 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,13 @@ use std::sync::{Arc, Mutex};
1111

1212
use miette::Result;
1313
use openshell_core::SandboxSessionId;
14+
use openshell_core::backend_startup::BackendStartupStage;
1415
use openshell_core::jwt::{SessionBearerTokenSlot, SupervisorAuthBundle};
1516
use openshell_core::provider_credentials::ProviderCredentialState;
1617
use openshell_isolation_interface::AgentSpec;
1718
use openshell_isolation_interface::contract::{
18-
BackendDescriptor, BackendError, ResolvedWorkloadIdentity, SandboxContext, SandboxPolicy,
19+
BackendDescriptor, BackendError, BackendErrorKind, ResolvedWorkloadIdentity, SandboxContext,
20+
SandboxPolicy,
1921
};
2022
use openshell_sandbox_backend::BoundBoundary;
2123

@@ -85,9 +87,53 @@ pub trait PreparedBackend: Send + Sync {
8587
) -> std::result::Result<BuiltBackend, BackendError>;
8688
}
8789

88-
/// Diagnostic code of an Attach error that is a backend refusal, as opposed to
89-
/// a transient failure the next start can retry.
90-
pub(crate) const ATTACH_REFUSED: &str = "openshell::backend::attach_refused";
90+
/// Keep the original typed error for refusal classification while exposing
91+
/// only an allowlisted stage/kind code to the configuration report.
92+
#[derive(Debug)]
93+
struct StartupError {
94+
stage: BackendStartupStage,
95+
error: BackendError,
96+
}
97+
98+
impl std::fmt::Display for StartupError {
99+
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
100+
std::fmt::Display::fmt(&self.error, formatter)
101+
}
102+
}
103+
104+
impl std::error::Error for StartupError {
105+
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
106+
Some(&self.error)
107+
}
108+
}
109+
110+
impl miette::Diagnostic for StartupError {
111+
fn code<'a>(&'a self) -> Option<Box<dyn std::fmt::Display + 'a>> {
112+
let kind = match self.error.kind() {
113+
BackendErrorKind::Invalid => "invalid",
114+
BackendErrorKind::Denied => "denied",
115+
BackendErrorKind::Unavailable => "unavailable",
116+
BackendErrorKind::Unsupported => "unsupported",
117+
BackendErrorKind::Failed => "failed",
118+
BackendErrorKind::Terminated => "terminated",
119+
};
120+
Some(Box::new(format!(
121+
"openshell.backend.{}.{kind}",
122+
self.stage.code()
123+
)))
124+
}
125+
}
126+
127+
pub(crate) fn startup_error(stage: BackendStartupStage, error: BackendError) -> miette::Report {
128+
miette::Report::new(StartupError { stage, error })
129+
}
130+
131+
pub(crate) fn is_backend_refusal(error: &miette::Report) -> bool {
132+
error.downcast_ref::<StartupError>().is_some_and(|failure| {
133+
failure.stage == BackendStartupStage::Attach
134+
&& matches!(failure.error, BackendError::Denied(_))
135+
})
136+
}
91137

92138
/// Created only after name and launch identity checks. Consuming attachment
93139
/// prevents reusing one prepared startup to build or attach a second client.
@@ -125,7 +171,7 @@ impl SelectedBackend {
125171

126172
let (identity, prepared) = setup
127173
.decode(&descriptor.payload)
128-
.map_err(|error| miette::miette!(error.to_string()))?;
174+
.map_err(|error| startup_error(BackendStartupStage::Build, error))?;
129175
if sandbox_id.is_none_or(|id| id.is_empty() || identity.sandbox_id != id) {
130176
return Err(miette::miette!(
131177
"runtime descriptor does not match admitted sandbox"
@@ -184,11 +230,14 @@ impl SelectedBackend {
184230
let built = self
185231
.prepared
186232
.build(services)
187-
.map_err(|error| miette::miette!(error.to_string()))?;
233+
.map_err(|error| startup_error(BackendStartupStage::Build, error))?;
188234
self.descriptor.payload = built.payload;
189235
if built.backend.backend_name() != self.descriptor.backend_name {
190-
return Err(miette::miette!(
191-
"RPC client backend does not match admitted backend"
236+
return Err(startup_error(
237+
BackendStartupStage::Build,
238+
BackendError::Descriptor(
239+
"RPC client backend does not match admitted backend".into(),
240+
),
192241
));
193242
}
194243
let backend = built.backend;
@@ -204,12 +253,7 @@ impl SelectedBackend {
204253
},
205254
)
206255
.await
207-
.map_err(|error| match error {
208-
BackendError::Denied(_) => {
209-
miette::miette!(code = ATTACH_REFUSED, "{error}")
210-
}
211-
error => miette::miette!("{error}"),
212-
})?;
256+
.map_err(|error| startup_error(BackendStartupStage::Attach, error))?;
213257
Ok((bound, built.proxy_listener))
214258
}
215259
}

‎crates/openshell-supervisor/src/backend_setup/tests.rs‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -594,7 +594,7 @@ async fn attach_refusal_is_marked_as_a_refusal() {
594594
.expect("an already attached boundary refuses attachment");
595595
assert_eq!(
596596
error.code().map(|code| code.to_string()).as_deref(),
597-
Some(ATTACH_REFUSED)
597+
Some("openshell.backend.attach.denied")
598598
);
599599
assert!(error.to_string().contains("already attached"));
600600
}
@@ -613,11 +613,7 @@ async fn attach_transport_failures_are_not_policy_refusals() {
613613
.await
614614
.err()
615615
.expect("backend transport failure must fail startup");
616-
assert_ne!(
617-
error.code().map(|code| code.to_string()).as_deref(),
618-
Some(ATTACH_REFUSED),
619-
"infrastructure failures must not be reported as policy denials"
620-
);
616+
assert!(!is_backend_refusal(&error));
621617
assert_eq!(setup.observed.starts.load(Ordering::SeqCst), 0);
622618
}
623619
}
@@ -944,7 +940,13 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() {
944940
.observed
945941
.deny_confirmation
946942
.store(true, Ordering::SeqCst);
947-
assert!(startup(false).await.is_err());
943+
let error = startup(false)
944+
.await
945+
.expect_err("confirmation must refuse startup");
946+
assert_eq!(
947+
error.code().map(|code| code.to_string()).as_deref(),
948+
Some("openshell.backend.confirm.unsupported")
949+
);
948950
assert_eq!(
949951
setup.observed.events(),
950952
[

0 commit comments

Comments
 (0)