Skip to content

Commit bc8109d

Browse files
committed
test(conformance): cover startup relay reconnects
Signed-off-by: Evan Lezar <[email protected]>
1 parent 30724e3 commit bc8109d

5 files changed

Lines changed: 169 additions & 9 deletions

File tree

‎architecture/build.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,9 @@ for explicit publication.
264264
CLI conformance runs after target provisioning and operates only through the
265265
configured OpenShell CLI. The smoke scenario verifies the black-box sandbox
266266
lifecycle by creating, inspecting, executing in, and deleting a sandbox. The
267+
dedicated sandbox-lifecycle scenarios verify stop/start persistence and bound a
268+
two-sandbox startup burst that checks relay attachments across the supervisor's
269+
initial reconnect. The guest runs at most two archive tests concurrently. The
267270
file-transfer scenario verifies portable upload and download behavior, Git-aware
268271
filtering, and sandbox workspace path safety.
269272
Feature suites use the same disposable guest but may provision isolated
@@ -314,6 +317,9 @@ with musl, and the gateway and supervisor with GNU. Image assembly stages
314317
the gateway, sandbox, and supervisor as separate binaries for their respective
315318
Dockerfiles. The helpers stage binaries under `artifacts/binaries` so local and
316319
CI builds expose the same inputs to tmachine and image assembly. The Ubuntu
320+
tmachine conformance tests consume the staged nextest archive, so local source
321+
changes to those tests require `nix run .#build-artifacts-test-archives` before
322+
running tmachine. The Ubuntu
317323
Docker and Fedora Podman environments import both local runtime images and
318324
configure the gateway to use them. The Ubuntu `deb` installer consumes
319325
`artifacts/packages/openshell.deb`; the `binaries` installer remains available

‎crates/openshell-conformance/src/lib.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,9 @@ use self::executor::{CliExecutionError, CliExecutor, ProcessCli};
2525

2626
pub use scenarios::{
2727
FILE_TRANSFER_GIT_FILTERING_SCENARIO, FILE_TRANSFER_PATH_SAFETY_SCENARIO,
28-
FILE_TRANSFER_ROUND_TRIP_SCENARIO, FILE_TRANSFER_SCENARIO, SANDBOX_LIFECYCLE_SCENARIO,
29-
SMOKE_SCENARIO,
28+
FILE_TRANSFER_ROUND_TRIP_SCENARIO, FILE_TRANSFER_SCENARIO,
29+
SANDBOX_LIFECYCLE_RELAY_READINESS_SCENARIO, SANDBOX_LIFECYCLE_SCENARIO,
30+
SANDBOX_LIFECYCLE_STATE_TRANSITIONS_SCENARIO, SMOKE_SCENARIO,
3031
};
3132

3233
/// An installed conformance scenario.

‎crates/openshell-conformance/src/scenarios/mod.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,8 @@ pub use file_transfer::{
1111
FILE_TRANSFER_GIT_FILTERING_SCENARIO, FILE_TRANSFER_PATH_SAFETY_SCENARIO,
1212
FILE_TRANSFER_ROUND_TRIP_SCENARIO, FILE_TRANSFER_SCENARIO,
1313
};
14-
pub use sandbox_lifecycle::SANDBOX_LIFECYCLE_SCENARIO;
14+
pub use sandbox_lifecycle::{
15+
SANDBOX_LIFECYCLE_RELAY_READINESS_SCENARIO, SANDBOX_LIFECYCLE_SCENARIO,
16+
SANDBOX_LIFECYCLE_STATE_TRANSITIONS_SCENARIO,
17+
};
1518
pub use smoke::SMOKE_SCENARIO;

‎crates/openshell-conformance/src/scenarios/sandbox_lifecycle.rs‎

Lines changed: 138 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ const CREATE_TIMEOUT: Duration = Duration::from_mins(10);
1313
const COMMAND_TIMEOUT: Duration = Duration::from_mins(2);
1414
const TRANSITION_TIMEOUT: Duration = Duration::from_mins(4);
1515
const TRANSITION_INTERVAL: Duration = Duration::from_secs(2);
16+
const RELAY_ATTACHMENT_TIMEOUT: Duration = Duration::from_secs(30);
17+
const RELAY_SANDBOX_ATTEMPTS: usize = 6;
18+
const RELAY_SCENARIO_CONCURRENCY: usize = 2;
1619

1720
#[derive(Debug, Deserialize)]
1821
struct SandboxState {
@@ -23,17 +26,151 @@ struct SandboxState {
2326
/// Certify sandbox stop, start, and deletion lifecycle behavior.
2427
pub const SANDBOX_LIFECYCLE_SCENARIO: Scenario = Scenario {
2528
name: "sandbox-lifecycle",
26-
description: "Verify sandbox stop, start, and deletion lifecycle behavior.",
29+
description: "Verify sandbox state transitions and relay readiness across reconnects.",
2730
run: run_sandbox_lifecycle,
2831
};
2932

33+
/// Certify sandbox stop, start, workspace preservation, and deletion behavior.
34+
pub const SANDBOX_LIFECYCLE_STATE_TRANSITIONS_SCENARIO: Scenario = Scenario {
35+
name: "sandbox-lifecycle/state-transitions",
36+
description: "Verify sandbox stop, start, workspace preservation, and deletion behavior.",
37+
run: run_state_transitions,
38+
};
39+
40+
/// Certify that relay-backed commands remain available across startup reconnects.
41+
pub const SANDBOX_LIFECYCLE_RELAY_READINESS_SCENARIO: Scenario = Scenario {
42+
name: "sandbox-lifecycle/relay-readiness",
43+
description: "Verify attachments survive supervisor reconnects during sandbox startup.",
44+
run: run_relay_readiness,
45+
};
46+
3047
fn run_sandbox_lifecycle(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
48+
Box::pin(async move {
49+
SANDBOX_LIFECYCLE_STATE_TRANSITIONS_SCENARIO
50+
.run(runner)
51+
.await?;
52+
SANDBOX_LIFECYCLE_RELAY_READINESS_SCENARIO.run(runner).await
53+
})
54+
}
55+
56+
fn run_state_transitions(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
3157
Box::pin(async move {
3258
stop_start_preserves_workspace(runner).await?;
3359
stopped_can_be_deleted(runner).await
3460
})
3561
}
3662

63+
fn run_relay_readiness(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
64+
Box::pin(async move { relay_readiness_survives_reconnects(runner).await })
65+
}
66+
67+
async fn relay_readiness_survives_reconnects(runner: &mut OpenShellRunner) -> Result<(), String> {
68+
for first_attempt in (1..=RELAY_SANDBOX_ATTEMPTS).step_by(RELAY_SCENARIO_CONCURRENCY) {
69+
let second_attempt = first_attempt + 1;
70+
let first_name = relay_sandbox_name(runner, first_attempt);
71+
let second_name = relay_sandbox_name(runner, second_attempt);
72+
runner.track_sandbox(&first_name);
73+
runner.track_sandbox(&second_name);
74+
75+
let (first, second) = tokio::join!(
76+
relay_readiness_attempt(runner, first_attempt, &first_name),
77+
relay_readiness_attempt(runner, second_attempt, &second_name),
78+
);
79+
first?;
80+
second?;
81+
wait_for_absence(
82+
runner,
83+
&first_name,
84+
&format!("relay-readiness/attempt-{first_attempt}/deleted"),
85+
)
86+
.await?;
87+
wait_for_absence(
88+
runner,
89+
&second_name,
90+
&format!("relay-readiness/attempt-{second_attempt}/deleted"),
91+
)
92+
.await?;
93+
runner.forget_sandbox(&first_name);
94+
runner.forget_sandbox(&second_name);
95+
}
96+
Ok(())
97+
}
98+
99+
fn relay_sandbox_name(runner: &OpenShellRunner, attempt: usize) -> String {
100+
format!("ct-{}-rr-{attempt}", runner.id())
101+
}
102+
103+
async fn relay_readiness_attempt(
104+
runner: &OpenShellRunner,
105+
attempt: usize,
106+
sandbox_name: &str,
107+
) -> Result<(), String> {
108+
let marker = format!("relay-ready-{}-{attempt}", runner.id());
109+
// Match the ordering that exposed the original race: detached creation
110+
// returns while the sandbox is still starting, then the attachment opens
111+
// against the first supervisor session before its startup reconnect.
112+
let main = format!("printf '%s\\n' '{marker}'; sleep 5");
113+
let create_command = runner
114+
.step(format!("relay-readiness/attempt-{attempt}/create"))
115+
.description(format!("sandbox '{sandbox_name}' is created"))
116+
.with_timeout(CREATE_TIMEOUT);
117+
let create_args = [
118+
"sandbox",
119+
"create",
120+
"--name",
121+
sandbox_name,
122+
"--detach",
123+
"--",
124+
"sh",
125+
"-c",
126+
&main,
127+
];
128+
create_command
129+
.run(&create_args)
130+
.await
131+
.map_err(|error| error.to_string())?
132+
.require_success()?;
133+
relay_connect(
134+
runner,
135+
sandbox_name,
136+
&format!("attempt-{attempt}/initial"),
137+
&marker,
138+
)
139+
.await?;
140+
141+
run_lifecycle_command(
142+
runner,
143+
"delete",
144+
sandbox_name,
145+
&format!("relay-readiness/attempt-{attempt}/delete"),
146+
)
147+
.await?;
148+
Ok(())
149+
}
150+
151+
async fn relay_connect(
152+
runner: &OpenShellRunner,
153+
sandbox_name: &str,
154+
cycle: &str,
155+
marker: &str,
156+
) -> Result<(), String> {
157+
let result = runner
158+
.step(format!("relay-readiness/{cycle}/connect"))
159+
.description(format!(
160+
"attachment reaches sandbox '{sandbox_name}' during its startup reconnect window"
161+
))
162+
.with_timeout(RELAY_ATTACHMENT_TIMEOUT)
163+
.run(&["sandbox", "connect", sandbox_name])
164+
.await
165+
.map_err(|error| error.to_string())?;
166+
result.require_success()?;
167+
if result.stdout().lines().any(|line| line == marker) {
168+
Ok(())
169+
} else {
170+
Err(result.failure_diagnostic(&format!("stdout contains marker {marker:?}")))
171+
}
172+
}
173+
37174
async fn stop_start_preserves_workspace(runner: &mut OpenShellRunner) -> Result<(), String> {
38175
let sandbox_name = format!("ct-{}-ss", runner.id());
39176
let sentinel = format!("openshell-stop-start-{}", runner.id());

‎tests/suites/conformance/cli/tests/lifecycle.rs‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,33 @@
33

44
//! Driver-agnostic sandbox lifecycle conformance tests.
55
6-
use openshell_conformance::{OpenShellRunner, SANDBOX_LIFECYCLE_SCENARIO};
6+
use openshell_conformance::{
7+
OpenShellRunner, SANDBOX_LIFECYCLE_RELAY_READINESS_SCENARIO,
8+
SANDBOX_LIFECYCLE_STATE_TRANSITIONS_SCENARIO, Scenario,
9+
};
710

811
/// Exercise stop, start, and stopped-deletion behavior through the candidate CLI.
912
#[tokio::test]
10-
async fn sandbox_lifecycle() {
11-
let mut runner = OpenShellRunner::from_env(SANDBOX_LIFECYCLE_SCENARIO.name)
13+
async fn state_transitions() {
14+
run(SANDBOX_LIFECYCLE_STATE_TRANSITIONS_SCENARIO).await;
15+
}
16+
17+
/// Exercise relay readiness during sandbox startup reconnects.
18+
#[tokio::test]
19+
async fn relay_readiness() {
20+
run(SANDBOX_LIFECYCLE_RELAY_READINESS_SCENARIO).await;
21+
}
22+
23+
async fn run(scenario: Scenario) {
24+
let mut runner = OpenShellRunner::from_env(scenario.name)
1225
.expect("candidate openshell CLI is available");
1326

1427
let result = async {
1528
runner.check_gateway_status().await?;
16-
SANDBOX_LIFECYCLE_SCENARIO.run(&mut runner).await
29+
scenario.run(&mut runner).await
1730
}
1831
.await;
1932
if let Err(error) = runner.finish(result).await {
20-
panic!("sandbox lifecycle conformance scenario failed:\n{error}");
33+
panic!("{} conformance scenario failed:\n{error}", scenario.name);
2134
}
2235
}

0 commit comments

Comments
 (0)