Skip to content

Commit 617b8bc

Browse files
committed
test(conformance): migrate sandbox_templates to the conformance suite
Port reusable sandbox workload template CRUD and template-backed sandbox creation coverage from e2e/rust/tests/sandbox_templates.rs into four driver-agnostic scenarios (sandbox-templates/lifecycle, get-after-delete, duplicate-name, missing-template) under tests/suites/conformance/, following the OpenShellRunner/Scenario pattern. Remove the old per-driver test file and its Cargo.toml entry. Verified against an ephemeral Podman gateway. Signed-off-by: politerealism <[email protected]>
1 parent ca17121 commit 617b8bc

6 files changed

Lines changed: 404 additions & 282 deletions

File tree

‎crates/openshell-conformance/src/lib.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,11 @@ pub use scenarios::{
2727
FILE_TRANSFER_CREATE_UPLOAD_SCENARIO, FILE_TRANSFER_GIT_FILTERING_SCENARIO,
2828
FILE_TRANSFER_PATH_SAFETY_SCENARIO, FILE_TRANSFER_ROUND_TRIP_SCENARIO, FILE_TRANSFER_SCENARIO,
2929
MECHANISTIC_PROPOSAL_SCENARIO, NEW_HOSTNAME_PROPOSAL_SCENARIO, POLICY_LOCAL_SCENARIO,
30-
PROVIDER_AUTO_CREATE_SCENARIO, SANDBOX_LIFECYCLE_SCENARIO, SMOKE_SCENARIO,
31-
WORKSPACE_LIFECYCLE_SCENARIO, WORKSPACE_TERMINATING_SCENARIO,
30+
PROVIDER_AUTO_CREATE_SCENARIO, SANDBOX_LIFECYCLE_SCENARIO,
31+
SANDBOX_TEMPLATE_DUPLICATE_NAME_SCENARIO, SANDBOX_TEMPLATE_GET_AFTER_DELETE_SCENARIO,
32+
SANDBOX_TEMPLATE_LIFECYCLE_SCENARIO, SANDBOX_TEMPLATE_MISSING_TEMPLATE_SCENARIO,
33+
SANDBOX_TEMPLATES_SCENARIO, SMOKE_SCENARIO, WORKSPACE_LIFECYCLE_SCENARIO,
34+
WORKSPACE_TERMINATING_SCENARIO,
3235
};
3336

3437
/// An installed conformance scenario.
@@ -55,6 +58,7 @@ const SCENARIOS: &[Scenario] = &[
5558
NEW_HOSTNAME_PROPOSAL_SCENARIO,
5659
POLICY_LOCAL_SCENARIO,
5760
PROVIDER_AUTO_CREATE_SCENARIO,
61+
SANDBOX_TEMPLATES_SCENARIO,
5862
WORKSPACE_LIFECYCLE_SCENARIO,
5963
WORKSPACE_TERMINATING_SCENARIO,
6064
];

‎crates/openshell-conformance/src/scenarios/mod.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ mod file_transfer;
77
mod policy_behavior;
88
mod provider_auto_create;
99
mod sandbox_lifecycle;
10+
mod sandbox_templates;
1011
mod smoke;
1112
mod workspace_lifecycle;
1213

@@ -19,5 +20,10 @@ pub use policy_behavior::{
1920
};
2021
pub use provider_auto_create::PROVIDER_AUTO_CREATE_SCENARIO;
2122
pub use sandbox_lifecycle::SANDBOX_LIFECYCLE_SCENARIO;
23+
pub use sandbox_templates::{
24+
SANDBOX_TEMPLATE_DUPLICATE_NAME_SCENARIO, SANDBOX_TEMPLATE_GET_AFTER_DELETE_SCENARIO,
25+
SANDBOX_TEMPLATE_LIFECYCLE_SCENARIO, SANDBOX_TEMPLATE_MISSING_TEMPLATE_SCENARIO,
26+
SANDBOX_TEMPLATES_SCENARIO,
27+
};
2228
pub use smoke::SMOKE_SCENARIO;
2329
pub use workspace_lifecycle::{WORKSPACE_LIFECYCLE_SCENARIO, WORKSPACE_TERMINATING_SCENARIO};
Lines changed: 345 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,345 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Portable sandbox workload template conformance scenarios.
5+
6+
use std::time::Duration;
7+
8+
use serde_json::Value;
9+
10+
use crate::{CommandResult, OpenShellRunner, Scenario, ScenarioFuture};
11+
12+
const CREATE_TIMEOUT: Duration = Duration::from_mins(10);
13+
const COMMAND_TIMEOUT: Duration = Duration::from_mins(2);
14+
15+
/// Certify reusable sandbox workload template behavior through the public CLI.
16+
pub const SANDBOX_TEMPLATES_SCENARIO: Scenario = Scenario {
17+
name: "sandbox-templates",
18+
description: "Verify sandbox template CRUD and template-backed sandbox creation.",
19+
run: run_sandbox_templates,
20+
};
21+
22+
/// Certify that a sandbox created from a template inherits its resources,
23+
/// labels, and environment.
24+
pub const SANDBOX_TEMPLATE_LIFECYCLE_SCENARIO: Scenario = Scenario {
25+
name: "sandbox-templates/lifecycle",
26+
description: "Verify template create/get/list and template-backed sandbox creation.",
27+
run: run_lifecycle,
28+
};
29+
30+
/// Certify that a deleted template is no longer fetchable.
31+
pub const SANDBOX_TEMPLATE_GET_AFTER_DELETE_SCENARIO: Scenario = Scenario {
32+
name: "sandbox-templates/get-after-delete",
33+
description: "Verify a deleted template returns not-found.",
34+
run: run_get_after_delete,
35+
};
36+
37+
/// Certify that creating a template with a name already in use fails.
38+
pub const SANDBOX_TEMPLATE_DUPLICATE_NAME_SCENARIO: Scenario = Scenario {
39+
name: "sandbox-templates/duplicate-name",
40+
description: "Verify duplicate template creation is rejected.",
41+
run: run_duplicate_name,
42+
};
43+
44+
/// Certify that creating a sandbox from a nonexistent template fails.
45+
pub const SANDBOX_TEMPLATE_MISSING_TEMPLATE_SCENARIO: Scenario = Scenario {
46+
name: "sandbox-templates/missing-template",
47+
description: "Verify sandbox creation from a missing template returns not-found.",
48+
run: run_missing_template,
49+
};
50+
51+
fn run_sandbox_templates(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
52+
Box::pin(async move {
53+
SANDBOX_TEMPLATE_LIFECYCLE_SCENARIO.run(runner).await?;
54+
SANDBOX_TEMPLATE_GET_AFTER_DELETE_SCENARIO
55+
.run(runner)
56+
.await?;
57+
SANDBOX_TEMPLATE_DUPLICATE_NAME_SCENARIO.run(runner).await?;
58+
SANDBOX_TEMPLATE_MISSING_TEMPLATE_SCENARIO.run(runner).await
59+
})
60+
}
61+
62+
fn run_lifecycle(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
63+
Box::pin(async move {
64+
let template_name = format!("ct-{}-tl", runner.id());
65+
let sandbox_name = format!("ct-{}-stl", runner.id());
66+
67+
let result = lifecycle(runner, &template_name, &sandbox_name).await;
68+
cleanup_template(runner, &template_name).await;
69+
result
70+
})
71+
}
72+
73+
async fn lifecycle(
74+
runner: &mut OpenShellRunner,
75+
template_name: &str,
76+
sandbox_name: &str,
77+
) -> Result<(), String> {
78+
runner
79+
.step("lifecycle/create")
80+
.description(format!("template '{template_name}' is created"))
81+
.with_timeout(COMMAND_TIMEOUT)
82+
.run(&[
83+
"sandbox",
84+
"template",
85+
"create",
86+
template_name,
87+
"--cpu",
88+
"500m",
89+
"--memory",
90+
"512Mi",
91+
"--label",
92+
"e2e=sandbox-template",
93+
"--env",
94+
"FEATURE_FLAG=on",
95+
])
96+
.await
97+
.map_err(|error| error.to_string())?
98+
.require_success()?;
99+
100+
let get = runner
101+
.step("lifecycle/get")
102+
.description(format!("template '{template_name}' can be retrieved"))
103+
.with_timeout(COMMAND_TIMEOUT)
104+
.run(&[
105+
"sandbox",
106+
"template",
107+
"get",
108+
template_name,
109+
"--output",
110+
"json",
111+
])
112+
.await
113+
.map_err(|error| error.to_string())?;
114+
get.require_success()?;
115+
let template_json: Value = get.json().map_err(|error| error.to_string())?;
116+
require_eq(&template_json["name"], template_name, "template name")?;
117+
require_eq(
118+
&template_json["labels"]["e2e"],
119+
"sandbox-template",
120+
"template label e2e",
121+
)?;
122+
require_eq(
123+
&template_json["environment"]["FEATURE_FLAG"],
124+
"on",
125+
"template environment FEATURE_FLAG",
126+
)?;
127+
require_eq(&template_json["resources"]["cpu"], "500m", "template cpu")?;
128+
require_eq(
129+
&template_json["resources"]["memory"],
130+
"512Mi",
131+
"template memory",
132+
)?;
133+
134+
let list = runner
135+
.step("lifecycle/list")
136+
.description("template list includes the created template")
137+
.with_timeout(COMMAND_TIMEOUT)
138+
.run(&["sandbox", "template", "list", "--names"])
139+
.await
140+
.map_err(|error| error.to_string())?;
141+
list.require_success()?;
142+
if !list
143+
.stdout()
144+
.lines()
145+
.any(|line| line.trim() == template_name)
146+
{
147+
return Err(list.failure_diagnostic(&format!("template list includes '{template_name}'")));
148+
}
149+
150+
runner.track_sandbox(sandbox_name);
151+
let create_sandbox = runner
152+
.step("lifecycle/sandbox-create")
153+
.description(format!(
154+
"sandbox '{sandbox_name}' is created from template '{template_name}'"
155+
))
156+
.with_timeout(CREATE_TIMEOUT)
157+
.run(&[
158+
"sandbox",
159+
"create",
160+
"--name",
161+
sandbox_name,
162+
"--template",
163+
template_name,
164+
"--",
165+
"sh",
166+
"-lc",
167+
"test \"$FEATURE_FLAG\" = on && echo template-env-ok",
168+
])
169+
.await
170+
.map_err(|error| error.to_string())?;
171+
create_sandbox.require_success()?;
172+
if !output_contains(&create_sandbox, "template-env-ok") {
173+
return Err(create_sandbox.failure_diagnostic("sandbox inherits template environment"));
174+
}
175+
176+
runner
177+
.step("lifecycle/sandbox-delete")
178+
.description(format!("sandbox '{sandbox_name}' is deleted"))
179+
.with_timeout(COMMAND_TIMEOUT)
180+
.run(&["sandbox", "delete", sandbox_name])
181+
.await
182+
.map_err(|error| error.to_string())?
183+
.require_success()?;
184+
runner.forget_sandbox(sandbox_name);
185+
Ok(())
186+
}
187+
188+
fn run_get_after_delete(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
189+
Box::pin(async move {
190+
let template_name = format!("ct-{}-tgd", runner.id());
191+
let result = get_after_delete(runner, &template_name).await;
192+
cleanup_template(runner, &template_name).await;
193+
result
194+
})
195+
}
196+
197+
async fn get_after_delete(runner: &OpenShellRunner, template_name: &str) -> Result<(), String> {
198+
runner
199+
.step("get-after-delete/create")
200+
.description(format!("template '{template_name}' is created"))
201+
.with_timeout(COMMAND_TIMEOUT)
202+
.run(&["sandbox", "template", "create", template_name])
203+
.await
204+
.map_err(|error| error.to_string())?
205+
.require_success()?;
206+
207+
runner
208+
.step("get-after-delete/delete")
209+
.description(format!("template '{template_name}' is deleted"))
210+
.with_timeout(COMMAND_TIMEOUT)
211+
.run(&["sandbox", "template", "delete", template_name])
212+
.await
213+
.map_err(|error| error.to_string())?
214+
.require_success()?;
215+
216+
let get_deleted = runner
217+
.step("get-after-delete/get")
218+
.description(format!(
219+
"deleted template '{template_name}' is not fetchable"
220+
))
221+
.with_timeout(COMMAND_TIMEOUT)
222+
.run(&["sandbox", "template", "get", template_name])
223+
.await
224+
.map_err(|error| error.to_string())?;
225+
if get_deleted.success() {
226+
return Err(get_deleted.failure_diagnostic("deleted template is not fetchable"));
227+
}
228+
if !output_mentions_template_not_found(&get_deleted) {
229+
return Err(get_deleted.failure_diagnostic("deleted template returns not-found"));
230+
}
231+
Ok(())
232+
}
233+
234+
fn run_duplicate_name(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
235+
Box::pin(async move {
236+
let template_name = format!("ct-{}-tdn", runner.id());
237+
let result = duplicate_name(runner, &template_name).await;
238+
cleanup_template(runner, &template_name).await;
239+
result
240+
})
241+
}
242+
243+
async fn duplicate_name(runner: &OpenShellRunner, template_name: &str) -> Result<(), String> {
244+
runner
245+
.step("duplicate-name/create")
246+
.description(format!("template '{template_name}' is created"))
247+
.with_timeout(COMMAND_TIMEOUT)
248+
.run(&["sandbox", "template", "create", template_name])
249+
.await
250+
.map_err(|error| error.to_string())?
251+
.require_success()?;
252+
253+
let duplicate = runner
254+
.step("duplicate-name/duplicate")
255+
.description(format!(
256+
"duplicate template '{template_name}' creation is rejected"
257+
))
258+
.with_timeout(COMMAND_TIMEOUT)
259+
.run(&["sandbox", "template", "create", template_name])
260+
.await
261+
.map_err(|error| error.to_string())?;
262+
if duplicate.success() {
263+
return Err(duplicate.failure_diagnostic("duplicate template creation is rejected"));
264+
}
265+
if !output_contains(&duplicate, "already exists") {
266+
return Err(duplicate.failure_diagnostic("duplicate template error reports already-exists"));
267+
}
268+
Ok(())
269+
}
270+
271+
fn run_missing_template(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
272+
Box::pin(async move {
273+
let template_name = format!("ct-{}-tmt", runner.id());
274+
let sandbox_name = format!("ct-{}-smt", runner.id());
275+
missing_template(runner, &template_name, &sandbox_name).await
276+
})
277+
}
278+
279+
async fn missing_template(
280+
runner: &mut OpenShellRunner,
281+
template_name: &str,
282+
sandbox_name: &str,
283+
) -> Result<(), String> {
284+
runner.track_sandbox(sandbox_name);
285+
let create_sandbox = runner
286+
.step("missing-template/create")
287+
.description(format!(
288+
"sandbox create from missing template '{template_name}' fails"
289+
))
290+
.with_timeout(CREATE_TIMEOUT)
291+
.run(&[
292+
"sandbox",
293+
"create",
294+
"--name",
295+
sandbox_name,
296+
"--template",
297+
template_name,
298+
"--",
299+
"true",
300+
])
301+
.await
302+
.map_err(|error| error.to_string())?;
303+
304+
if create_sandbox.success() {
305+
return Err(create_sandbox.failure_diagnostic("sandbox create from missing template fails"));
306+
}
307+
runner.forget_sandbox(sandbox_name);
308+
if !output_mentions_template_not_found(&create_sandbox) {
309+
return Err(create_sandbox
310+
.failure_diagnostic("sandbox create from missing template reports not-found"));
311+
}
312+
Ok(())
313+
}
314+
315+
/// Best-effort teardown. Mirrors a `Drop`-guard cleanup: ignore failures since
316+
/// the template may already be gone as part of a successful scenario run.
317+
async fn cleanup_template(runner: &OpenShellRunner, template_name: &str) {
318+
let _ = runner
319+
.step("cleanup/template-delete")
320+
.description(format!(
321+
"template '{template_name}' is deleted or already absent"
322+
))
323+
.with_timeout(COMMAND_TIMEOUT)
324+
.run(&["sandbox", "template", "delete", template_name])
325+
.await;
326+
}
327+
328+
fn output_contains(result: &CommandResult, needle: &str) -> bool {
329+
let haystack = format!("{}{}", result.stdout(), result.stderr()).to_lowercase();
330+
haystack.contains(&needle.to_lowercase())
331+
}
332+
333+
fn output_mentions_template_not_found(result: &CommandResult) -> bool {
334+
output_contains(result, "sandbox template") && output_contains(result, "not found")
335+
}
336+
337+
fn require_eq(actual: &Value, expected: &str, label: &str) -> Result<(), String> {
338+
if actual.as_str() == Some(expected) {
339+
Ok(())
340+
} else {
341+
Err(format!(
342+
"{label} mismatch: expected {expected:?}, got {actual:?}"
343+
))
344+
}
345+
}

‎e2e/rust/Cargo.toml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -203,11 +203,6 @@ name = "forward_proxy_jsonrpc_l7"
203203
path = "tests/forward_proxy_jsonrpc_l7.rs"
204204
required-features = ["e2e-host-gateway"]
205205

206-
[[test]]
207-
name = "sandbox_templates"
208-
path = "tests/sandbox_templates.rs"
209-
required-features = ["e2e"]
210-
211206
[[test]]
212207
name = "proxy_egress_pipeline"
213208
path = "tests/proxy_egress_pipeline.rs"

0 commit comments

Comments
 (0)