Skip to content

Commit ca17121

Browse files
committed
test(conformance): migrate provider_auto_create to the conformance suite
Port --provider claude-code --auto-providers credential-placeholder-injection coverage from e2e/rust/tests/provider_auto_create.rs into a new provider-auto-create scenario under tests/suites/conformance/, following the OpenShellRunner/Scenario pattern. Preserve the original's existence-check skip and defensive pre-cleanup, since "claude-code" is a recognized auto-provider type name rather than a value this scenario can scope per run. Remove the old per-driver test file (no Cargo.toml entry; picked up by Cargo's default test auto-discovery) and its tests/artifacts.nix follow-up exclusion entry. Verified against an ephemeral Podman gateway. Signed-off-by: politerealism <[email protected]>
1 parent 398dfe3 commit ca17121

6 files changed

Lines changed: 181 additions & 200 deletions

File tree

‎crates/openshell-conformance/src/lib.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ pub use scenarios::{
2727
FILE_TRANSFER_CREATE_UPLOAD_SCENARIO, FILE_TRANSFER_GIT_FILTERING_SCENARIO,
2828
FILE_TRANSFER_PATH_SAFETY_SCENARIO, FILE_TRANSFER_ROUND_TRIP_SCENARIO, FILE_TRANSFER_SCENARIO,
2929
MECHANISTIC_PROPOSAL_SCENARIO, NEW_HOSTNAME_PROPOSAL_SCENARIO, POLICY_LOCAL_SCENARIO,
30-
SANDBOX_LIFECYCLE_SCENARIO, SMOKE_SCENARIO, WORKSPACE_LIFECYCLE_SCENARIO,
31-
WORKSPACE_TERMINATING_SCENARIO,
30+
PROVIDER_AUTO_CREATE_SCENARIO, SANDBOX_LIFECYCLE_SCENARIO, SMOKE_SCENARIO,
31+
WORKSPACE_LIFECYCLE_SCENARIO, WORKSPACE_TERMINATING_SCENARIO,
3232
};
3333

3434
/// An installed conformance scenario.
@@ -54,6 +54,7 @@ const SCENARIOS: &[Scenario] = &[
5454
MECHANISTIC_PROPOSAL_SCENARIO,
5555
NEW_HOSTNAME_PROPOSAL_SCENARIO,
5656
POLICY_LOCAL_SCENARIO,
57+
PROVIDER_AUTO_CREATE_SCENARIO,
5758
WORKSPACE_LIFECYCLE_SCENARIO,
5859
WORKSPACE_TERMINATING_SCENARIO,
5960
];

‎crates/openshell-conformance/src/scenarios/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
66
mod file_transfer;
77
mod policy_behavior;
8+
mod provider_auto_create;
89
mod sandbox_lifecycle;
910
mod smoke;
1011
mod workspace_lifecycle;
@@ -16,6 +17,7 @@ pub use file_transfer::{
1617
pub use policy_behavior::{
1718
MECHANISTIC_PROPOSAL_SCENARIO, NEW_HOSTNAME_PROPOSAL_SCENARIO, POLICY_LOCAL_SCENARIO,
1819
};
20+
pub use provider_auto_create::PROVIDER_AUTO_CREATE_SCENARIO;
1921
pub use sandbox_lifecycle::SANDBOX_LIFECYCLE_SCENARIO;
2022
pub use smoke::SMOKE_SCENARIO;
2123
pub use workspace_lifecycle::{WORKSPACE_LIFECYCLE_SCENARIO, WORKSPACE_TERMINATING_SCENARIO};
Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Portable provider auto-creation conformance scenario.
5+
6+
use std::time::Duration;
7+
8+
use crate::{CommandResult, OpenShellRunner, Scenario, ScenarioFuture};
9+
10+
const CREATE_TIMEOUT: Duration = Duration::from_mins(10);
11+
const COMMAND_TIMEOUT: Duration = Duration::from_mins(2);
12+
13+
// "claude-code" is a recognized auto-provider type, not an arbitrary name:
14+
// the CLI names the auto-created provider after the `--provider` value, so
15+
// this scenario cannot scope it per run like other tracked resources.
16+
const PROVIDER_NAME: &str = "claude-code";
17+
const CREDENTIAL_ENV_VAR: &str = "ANTHROPIC_API_KEY";
18+
const TEST_API_KEY: &str = "sk-e2e-auto-provider-test-key";
19+
20+
/// Certify that `--provider claude-code --auto-providers` discovers a local
21+
/// credential, auto-creates the provider, and injects a redacted placeholder
22+
/// into the sandbox environment rather than the raw secret.
23+
pub const PROVIDER_AUTO_CREATE_SCENARIO: Scenario = Scenario {
24+
name: "provider-auto-create",
25+
description: "Verify --provider auto-creation injects a redacted credential placeholder.",
26+
run: run_provider_auto_create,
27+
};
28+
29+
fn run_provider_auto_create(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> {
30+
Box::pin(async move {
31+
if provider_exists(runner).await? {
32+
eprintln!(
33+
"skipping provider-auto-create: existing provider '{PROVIDER_NAME}' would make shared state unsafe"
34+
);
35+
return Ok(());
36+
}
37+
38+
// Defensive cleanup of any leftover from a previous crashed run.
39+
delete_provider(runner).await;
40+
41+
let sandbox_name = format!("ct-{}-pa", runner.id());
42+
runner.track_sandbox(&sandbox_name);
43+
runner.track_provider(PROVIDER_NAME);
44+
auto_created_provider_credential_available_in_sandbox(runner, &sandbox_name).await
45+
})
46+
}
47+
48+
async fn provider_exists(runner: &OpenShellRunner) -> Result<bool, String> {
49+
let result = runner
50+
.step("preflight/provider-get")
51+
.description(format!(
52+
"check whether provider '{PROVIDER_NAME}' already exists"
53+
))
54+
.with_timeout(COMMAND_TIMEOUT)
55+
.run(&["provider", "get", PROVIDER_NAME])
56+
.await
57+
.map_err(|error| error.to_string())?;
58+
Ok(result.success())
59+
}
60+
61+
async fn delete_provider(runner: &OpenShellRunner) {
62+
let _ = runner
63+
.step("preflight/provider-delete")
64+
.description(format!(
65+
"provider '{PROVIDER_NAME}' is deleted or already absent"
66+
))
67+
.with_timeout(COMMAND_TIMEOUT)
68+
.run(&["provider", "delete", PROVIDER_NAME])
69+
.await;
70+
}
71+
72+
async fn auto_created_provider_credential_available_in_sandbox(
73+
runner: &OpenShellRunner,
74+
sandbox_name: &str,
75+
) -> Result<(), String> {
76+
let create = runner
77+
.step("create")
78+
.description(format!(
79+
"sandbox '{sandbox_name}' is created with provider auto-creation"
80+
))
81+
.with_timeout(CREATE_TIMEOUT)
82+
.run_with_env(
83+
&[
84+
"sandbox",
85+
"create",
86+
"--name",
87+
sandbox_name,
88+
"--detach",
89+
"--provider",
90+
PROVIDER_NAME,
91+
"--auto-providers",
92+
"--",
93+
"sh",
94+
"-c",
95+
"exec sleep infinity",
96+
],
97+
&[(CREDENTIAL_ENV_VAR, TEST_API_KEY)],
98+
)
99+
.await
100+
.map_err(|error| error.to_string())?;
101+
create.require_success()?;
102+
if !output_contains(&create, "Created provider claude-code") {
103+
return Err(create.failure_diagnostic("output confirms provider auto-creation"));
104+
}
105+
106+
let exec = runner
107+
.step("exec")
108+
.description(format!(
109+
"sandbox '{sandbox_name}' exec reads the injected credential placeholder"
110+
))
111+
.with_timeout(COMMAND_TIMEOUT)
112+
.run(&[
113+
"sandbox",
114+
"exec",
115+
"--name",
116+
sandbox_name,
117+
"--no-tty",
118+
"--",
119+
"printenv",
120+
CREDENTIAL_ENV_VAR,
121+
])
122+
.await
123+
.map_err(|error| error.to_string())?;
124+
exec.require_success()?;
125+
126+
if !contains_placeholder_for_env_key(exec.stdout(), CREDENTIAL_ENV_VAR)
127+
&& !contains_placeholder_for_env_key(exec.stderr(), CREDENTIAL_ENV_VAR)
128+
{
129+
return Err(exec.failure_diagnostic(&format!(
130+
"sandbox environment contains a resolve placeholder for {CREDENTIAL_ENV_VAR}"
131+
)));
132+
}
133+
if output_contains(&exec, TEST_API_KEY) {
134+
return Err(exec.failure_diagnostic(&format!(
135+
"sandbox environment does not expose the raw {CREDENTIAL_ENV_VAR} secret"
136+
)));
137+
}
138+
139+
Ok(())
140+
}
141+
142+
fn output_contains(result: &CommandResult, needle: &str) -> bool {
143+
result.stdout().contains(needle) || result.stderr().contains(needle)
144+
}
145+
146+
/// Matches the supervisor's credential-resolution placeholder token, which is
147+
/// either the legacy fixed form or a versioned `openshell:resolve:env:v<N>_<KEY>`.
148+
fn contains_placeholder_for_env_key(output: &str, key: &str) -> bool {
149+
let legacy = format!("openshell:resolve:env:{key}");
150+
let revision_prefix = "openshell:resolve:env:v";
151+
let revision_suffix = format!("_{key}");
152+
output.split_whitespace().any(|token| {
153+
token == legacy || (token.starts_with(revision_prefix) && token.ends_with(&revision_suffix))
154+
})
155+
}

‎e2e/rust/tests/provider_auto_create.rs‎

Lines changed: 0 additions & 197 deletions
This file was deleted.

‎tests/artifacts.nix‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,6 @@ let
116116
"podman_corporate_proxy"
117117
"podman_gateway_start"
118118
"podman_oci_identity"
119-
"provider_auto_create"
120119
# The provider-refresh feature suite covers revoked Keycloak grants. This
121120
# binary instead covers stable workload handles across repeated rotations
122121
# and explicit refresh reconfiguration in a long-running sandbox.

0 commit comments

Comments
 (0)