Skip to content

Commit d181609

Browse files
refactor(oci): simplify workspace failure handling and OCI image tests
Report a recorded Docker workspace rejection once, where supervisor readiness fails, and match its fixed message exactly. Read the Podman supervisor exit status once per inspection. Choose the supervisor exit status in one function and derive the workspace error type. The OCI image suite shares step helpers, checks that a custom workspace does not also create /sandbox, and its playbook waits for a connected gateway and collects failure diagnostics through the shared role. Signed-off-by: Matthew Grossman <[email protected]>
1 parent 2ce1cdb commit d181609

8 files changed

Lines changed: 222 additions & 219 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/openshell-driver-docker/src/lib.rs‎

Lines changed: 35 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -5334,41 +5334,46 @@ async fn wait_for_docker_supervisor_ready(
53345334
workload_container_id: &str,
53355335
sandbox_id: &str,
53365336
failures: &Arc<Mutex<HashMap<String, DockerRuntimeFailure>>>,
5337+
) -> Result<(), Status> {
5338+
// The monitor records a workspace rejection before it removes the
5339+
// supervisor, so a later failed inspection must still report it.
5340+
match poll_docker_supervisor_ready(docker, supervisor_id, workload_container_id, sandbox_id)
5341+
.await
5342+
{
5343+
Ok(()) => Ok(()),
5344+
Err(status) => Err(recorded_workspace_validation_failure(failures, sandbox_id)
5345+
.await
5346+
.unwrap_or(status)),
5347+
}
5348+
}
5349+
5350+
async fn poll_docker_supervisor_ready(
5351+
docker: &Docker,
5352+
supervisor_id: &str,
5353+
workload_container_id: &str,
5354+
sandbox_id: &str,
53375355
) -> Result<(), Status> {
53385356
// Gateway provisioning deadlines own startup expiration, including policy
53395357
// repairs that extend the window. Unhealthy means not ready while the
53405358
// supervisor is quarantined; only a stopped process is a startup failure.
53415359
loop {
5342-
if let Some(status) = recorded_workspace_validation_failure(failures, sandbox_id).await {
5343-
return Err(status);
5344-
}
5345-
let sandbox = match docker.inspect_container(workload_container_id, None).await {
5346-
Ok(sandbox) => sandbox,
5347-
Err(error) => {
5348-
return Err(recorded_workspace_validation_failure(failures, sandbox_id)
5349-
.await
5350-
.unwrap_or_else(|| {
5351-
Status::internal(format!("inspect Docker sandbox container: {error}"))
5352-
}));
5353-
}
5354-
};
5360+
let sandbox = docker
5361+
.inspect_container(workload_container_id, None)
5362+
.await
5363+
.map_err(|error| {
5364+
Status::internal(format!("inspect Docker sandbox container: {error}"))
5365+
})?;
53555366
if sandbox.state.unwrap_or_default().running == Some(false) {
5356-
return Err(recorded_workspace_validation_failure(failures, sandbox_id)
5357-
.await
5358-
.unwrap_or_else(|| {
5359-
Status::unavailable("Docker sandbox exited before supervisor became ready")
5360-
}));
5367+
return Err(Status::unavailable(
5368+
"Docker sandbox exited before supervisor became ready",
5369+
));
53615370
}
5362-
let inspected = match docker.inspect_container(supervisor_id, None).await {
5363-
Ok(inspected) => inspected,
5364-
Err(error) => {
5365-
return Err(recorded_workspace_validation_failure(failures, sandbox_id)
5366-
.await
5367-
.unwrap_or_else(|| {
5368-
Status::internal(format!("inspect Docker supervisor container: {error}"))
5369-
}));
5370-
}
5371-
};
5371+
let inspected = docker
5372+
.inspect_container(supervisor_id, None)
5373+
.await
5374+
.map_err(|error| {
5375+
Status::internal(format!("inspect Docker supervisor container: {error}"))
5376+
})?;
53725377
let state = inspected.state.unwrap_or_default();
53735378
match state.health.and_then(|health| health.status) {
53745379
Some(HealthStatusEnum::HEALTHY) => return Ok(()),
@@ -5381,11 +5386,6 @@ async fn wait_for_docker_supervisor_ready(
53815386
return Err(workspace_validation_status());
53825387
}
53835388
_ if state.running == Some(false) => {
5384-
if let Some(status) =
5385-
recorded_workspace_validation_failure(failures, sandbox_id).await
5386-
{
5387-
return Err(status);
5388-
}
53895389
let log_tail = docker_container_log_tail(docker, supervisor_id).await;
53905390
warn!(sandbox_id, supervisor_id, supervisor_logs = %log_tail, "Docker supervisor exited before becoming ready");
53915391
return Err(Status::unavailable(format!(
@@ -5439,7 +5439,7 @@ async fn recorded_workspace_validation_failure(
54395439

54405440
/// Startup failure for a supervisor that exited because the sandbox rejected
54415441
/// the image working directory. The message is fixed text, because supervisor
5442-
/// output may contain secrets; the fixed message identifies the failure for
5442+
/// output may contain secrets, and identifies the failure for
54435443
/// [`supervisor_start_failure_reason`].
54445444
fn workspace_validation_status() -> Status {
54455445
Status::failed_precondition(WORKSPACE_VALIDATION_FAILED_MESSAGE)
@@ -5448,9 +5448,7 @@ fn workspace_validation_status() -> Status {
54485448
/// Condition reason for a supervisor that failed before becoming ready.
54495449
fn supervisor_start_failure_reason(status: &Status, default: &'static str) -> &'static str {
54505450
if status.code() == tonic::Code::FailedPrecondition
5451-
&& status
5452-
.message()
5453-
.starts_with(WORKSPACE_VALIDATION_FAILED_MESSAGE)
5451+
&& status.message() == WORKSPACE_VALIDATION_FAILED_MESSAGE
54545452
{
54555453
CONDITION_WORKSPACE_VALIDATION_FAILED
54565454
} else {

‎crates/openshell-driver-podman/src/watcher.rs‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,10 @@ pub async fn inspect_workload(
375375
let supervisor = client
376376
.inspect_container(&crate::isolation::supervisor_name(sandbox_id))
377377
.await;
378+
let supervisor_exit_code = supervisor
379+
.as_ref()
380+
.ok()
381+
.and_then(|supervisor| exited_exit_code(&supervisor.state));
378382
if workload.state.running {
379383
match supervisor {
380384
Ok(supervisor) if supervisor.state.running => {
@@ -391,12 +395,7 @@ pub async fn inspect_workload(
391395
// Both containers exist before initial start. A missing or exited
392396
// companion therefore requires containment, including after a
393397
// gateway restart that missed the original Podman exit event.
394-
Ok(supervisor) => {
395-
client.stop_container(&workload.id, 0).await?;
396-
workload = client.inspect_container(&workload.id).await?;
397-
workload.state.supervisor_exit_code = exited_exit_code(&supervisor.state);
398-
}
399-
Err(PodmanApiError::NotFound(_)) => {
398+
Ok(_) | Err(PodmanApiError::NotFound(_)) => {
400399
client.stop_container(&workload.id, 0).await?;
401400
workload = client.inspect_container(&workload.id).await?;
402401
}
@@ -408,10 +407,8 @@ pub async fn inspect_workload(
408407
.await
409408
.ok()
410409
.and_then(|logs| boundary_startup_termination_marker(&logs));
411-
workload.state.supervisor_exit_code = supervisor
412-
.ok()
413-
.and_then(|supervisor| exited_exit_code(&supervisor.state));
414410
}
411+
workload.state.supervisor_exit_code = supervisor_exit_code;
415412
Ok(workload)
416413
}
417414

‎crates/openshell-supervisor/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ serde = { workspace = true }
3939
serde_json = { workspace = true }
4040
socket2 = { workspace = true }
4141
tempfile = "3"
42+
thiserror = { workspace = true }
4243
tokio = { workspace = true }
4344
tonic = { workspace = true, features = ["channel", "tls-native-roots"] }
4445
tracing = { workspace = true }

‎crates/openshell-supervisor/src/lib.rs‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -66,19 +66,10 @@ use openshell_ocsf::{
6666
pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx;
6767

6868
/// The sandbox rejected the image working directory while starting the agent.
69-
#[derive(Debug)]
69+
#[derive(Debug, thiserror::Error, miette::Diagnostic)]
70+
#[error("{0}")]
7071
pub struct WorkspaceValidationFailed(String);
7172

72-
impl std::fmt::Display for WorkspaceValidationFailed {
73-
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
74-
f.write_str(&self.0)
75-
}
76-
}
77-
78-
impl std::error::Error for WorkspaceValidationFailed {}
79-
80-
impl miette::Diagnostic for WorkspaceValidationFailed {}
81-
8273
/// Convert an agent start failure into a report.
8374
///
8475
/// The sandbox reports errors across the boundary as text, so a workspace

‎crates/openshell-supervisor/src/main.rs‎

Lines changed: 33 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,7 @@ fn main() -> Result<()> {
423423
};
424424
let admitted_isolation_backend =
425425
std::env::var(openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND).ok();
426-
Box::pin(openshell_supervisor::run_sandbox(
426+
let result = Box::pin(openshell_supervisor::run_sandbox(
427427
command,
428428
workdir,
429429
args.timeout,
@@ -445,9 +445,8 @@ fn main() -> Result<()> {
445445
admitted_isolation_backend,
446446
args.main_exit_marker,
447447
))
448-
.await
449-
.map(supervisor_exit_status)
450-
.or_else(workspace_validation_exit)
448+
.await;
449+
supervisor_exit(result)
451450
}
452451
SupervisorRole::NetworkProxy => {
453452
let listen = args.listen.unwrap_or_else(|| ([127, 0, 0, 1], 3128).into());
@@ -488,31 +487,27 @@ fn otlp_span_filter(log_level: &str) -> EnvFilter {
488487
EnvFilter::new(format!("info,openshell={level}"))
489488
}
490489

491-
/// Exit with the reserved status when the sandbox rejects the image working
492-
/// directory, so the compute driver can report the specific failure.
493-
fn workspace_validation_exit(error: miette::Report) -> Result<i32> {
494-
use openshell_core::driver_utils::SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED;
495-
if error
496-
.downcast_ref::<openshell_supervisor::WorkspaceValidationFailed>()
497-
.is_none()
498-
{
499-
return Err(error);
500-
}
501-
error!("Image workspace validation failed");
502-
eprintln!("{error:?}");
503-
Ok(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)
504-
}
505-
506-
/// The supervisor exits with the workload's status after the workload exits.
507-
/// Keep the reserved workspace-validation status for workspace rejections, so
508-
/// drivers do not misreport a workload that itself exited with it. The gateway
509-
/// receives the workload's exact status separately.
510-
fn supervisor_exit_status(workload_status: i32) -> i32 {
490+
/// Choose the supervisor's exit status for a sandbox run.
491+
///
492+
/// The reserved status tells the compute driver that the sandbox rejected the
493+
/// image working directory. After the workload exits, the supervisor exits
494+
/// with the workload's status, so a workload that itself exits with the
495+
/// reserved status is reported as 1 instead. The gateway receives the
496+
/// workload's exact status separately.
497+
fn supervisor_exit(result: Result<i32>) -> Result<i32> {
511498
use openshell_core::driver_utils::SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED;
512-
if workload_status == SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED {
513-
1
514-
} else {
515-
workload_status
499+
match result {
500+
Ok(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) => Ok(1),
501+
Err(error)
502+
if error
503+
.downcast_ref::<openshell_supervisor::WorkspaceValidationFailed>()
504+
.is_some() =>
505+
{
506+
error!("Image workspace validation failed");
507+
eprintln!("{error:?}");
508+
Ok(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)
509+
}
510+
other => other,
516511
}
517512
}
518513

@@ -530,7 +525,7 @@ mod tests {
530525
}
531526

532527
#[test]
533-
fn workspace_validation_failure_exits_with_reserved_status() {
528+
fn only_workspace_validation_failure_exits_with_reserved_status() {
534529
use miette::WrapErr as _;
535530
use openshell_core::driver_utils::{
536531
SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED, WORKSPACE_VALIDATION_ERROR_CONTEXT,
@@ -546,34 +541,31 @@ mod tests {
546541
"process error: boundary process leaf: start process supervisor leaf: {sandbox_error:?}"
547542
));
548543
assert_eq!(
549-
workspace_validation_exit(error).expect("reserved exit status"),
544+
supervisor_exit(Err(error)).expect("reserved exit status"),
550545
SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED
551546
);
552547
assert!(
553-
workspace_validation_exit(openshell_supervisor::start_agent_error(
548+
supervisor_exit(Err(openshell_supervisor::start_agent_error(
554549
"connect failed".into()
555-
))
550+
)))
556551
.is_err()
557552
);
558553
// Only an agent start failure carries the rejection, even when another
559554
// error's text contains the same words.
560555
assert!(
561-
workspace_validation_exit(miette::miette!(
556+
supervisor_exit(Err(miette::miette!(
562557
"policy sync failed: {WORKSPACE_VALIDATION_ERROR_CONTEXT}"
563-
))
558+
)))
564559
.is_err()
565560
);
566-
}
567561

568-
#[test]
569-
fn workload_exit_does_not_use_reserved_workspace_status() {
570-
use openshell_core::driver_utils::SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED;
562+
// A workload's own exit with the reserved status is not a rejection.
571563
assert_eq!(
572-
supervisor_exit_status(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED),
564+
supervisor_exit(Ok(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED)).unwrap(),
573565
1
574566
);
575-
assert_eq!(supervisor_exit_status(0), 0);
576-
assert_eq!(supervisor_exit_status(143), 143);
567+
assert_eq!(supervisor_exit(Ok(0)).unwrap(), 0);
568+
assert_eq!(supervisor_exit(Ok(143)).unwrap(), 143);
577569
}
578570

579571
#[test]

‎tests/ansible/playbooks/features/oci-image.yaml‎

Lines changed: 27 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
gather_facts: false
88
vars:
99
oci_image_test_root: /var/lib/openshell-oci-image/tests
10+
# The tests are skipped when the gateway never connects.
11+
oci_image_failed: "{{ gateway_status is failed or oci_image_result.rc != 0 }}"
1012
tasks:
1113
- name: Wait for SSH
1214
ansible.builtin.wait_for_connection:
@@ -61,6 +63,20 @@
6163
register: openshell_cli
6264
changed_when: false
6365

66+
# Every test boot restarts the gateway, and some installers take minutes to
67+
# recover. Wait once here; the runner's per-scenario preflight stays short.
68+
- name: Wait for the registered gateway
69+
ansible.builtin.command:
70+
argv: [timeout, 10s, "{{ openshell_cli.stdout }}", status, --output, json]
71+
register: gateway_status
72+
changed_when: false
73+
until: >-
74+
gateway_status.rc == 0 and
75+
(gateway_status.stdout | from_json).status == 'connected'
76+
retries: 60
77+
delay: 5
78+
ignore_errors: true
79+
6480
- name: Run OCI image archive
6581
ansible.builtin.command:
6682
argv:
@@ -84,32 +100,23 @@
84100
register: oci_image_result
85101
changed_when: false
86102
failed_when: false
103+
when: gateway_status is succeeded
87104

88105
- name: Show OCI image diagnostics
89106
ansible.builtin.debug:
90107
var: oci_image_result
91-
when: oci_image_result.rc != 0
108+
when: oci_image_failed
92109

93-
- name: Read OpenShell gateway logs
94-
become: true
95-
ansible.builtin.command:
96-
argv:
97-
- journalctl
98-
- --no-pager
99-
- --lines
100-
- "500"
101-
- _SYSTEMD_UNIT=openshell-gateway.service
102-
- "+"
103-
- _SYSTEMD_USER_UNIT=openshell-gateway.service
104-
register: openshell_gateway_logs
105-
changed_when: false
106-
failed_when: false
107-
when: oci_image_result.rc != 0
110+
- name: Collect OpenShell gateway diagnostics
111+
ansible.builtin.include_role:
112+
name: openshell_diagnostics
113+
when: oci_image_failed
108114

109-
- name: Show OpenShell gateway logs
110-
ansible.builtin.debug:
111-
var: openshell_gateway_logs.stdout_lines
112-
when: oci_image_result.rc != 0
115+
- name: Require a connected gateway
116+
ansible.builtin.assert:
117+
that:
118+
- gateway_status is succeeded
119+
fail_msg: OpenShell gateway did not connect before the OCI image tests; see gateway diagnostics
113120

114121
- name: Require OCI image success
115122
ansible.builtin.assert:

0 commit comments

Comments
 (0)