Skip to content

Commit 592a748

Browse files
refactor(conformance): make the conformance family one crate
Conformance is one crate per family, with each suite a module, so the family directory is the crate: tests/suites/conformance/sandbox becomes the `conformance` package at tests/suites/conformance. The workspace, its lockfile and the scripts that run it by path are unchanged. Nextest's binary for these tests is now `conformance`; test names are unchanged. The tmachine runner rows, the fmt and clippy lines, the openshell-test dependency and the docs name the new package. Signed-off-by: Matthew Grossman <[email protected]>
1 parent ca412d5 commit 592a748

23 files changed

Lines changed: 78 additions & 83 deletions

File tree

‎.agents/skills/watch-github-actions/SKILL.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ gh run list --json databaseId,status,headBranch,url --jq '.[] | {id: .databaseId
125125

126126
## View Job Logs
127127

128-
For tmachine Kubernetes conformance failures, inspect the `e2e-results-conformance-sandbox-ubuntu-k3s-*` artifact. Its `diagnostics/` holds the K3s report with Agent Sandbox controller state, leader leases, events, and current and previous controller logs. The k3s installers check reconciliation after guest boot with a temporary scheduling-gated Sandbox probe, so a failed probe fails provisioning before any test runs. Distinguish a failed controller probe from a failed conformance scenario; gateway connectivity and controller Deployment availability do not prove reconciliation works.
128+
For tmachine Kubernetes conformance failures, inspect the `e2e-results-conformance-conformance-ubuntu-k3s-*` artifact. Its `diagnostics/` holds the K3s report with Agent Sandbox controller state, leader leases, events, and current and previous controller logs. The k3s installers check reconciliation after guest boot with a temporary scheduling-gated Sandbox probe, so a failed probe fails provisioning before any test runs. Distinguish a failed controller probe from a failed conformance scenario; gateway connectivity and controller Deployment availability do not prove reconciliation works.
129129

130130
For the `e2e-suites` lane (`.github/workflows/e2e-suites.yml`), the last lines of the job log give the runner's result: PASS, FAIL (exit 1) after the end of the nextest log (which names the failed tests), or ERROR (exit 2) with the step that broke. FAIL and ERROR also appear as annotations on the run's summary page. Then inspect the `e2e-results-<family>-<package>-<environment>-<installer>` artifact, which holds `junit.xml`, `nextest.log` and, after a failure, the guest's `diagnostics/`.
131131

‎.github/workflows/branch-checks.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ jobs:
233233
cargo fmt --manifest-path tests/testkit/Cargo.toml --all -- --check
234234
cargo fmt --manifest-path tests/tmachine/Cargo.toml --all -- --check
235235
cargo fmt --manifest-path tests/runner/Cargo.toml --all -- --check
236-
cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -- --check
236+
cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p conformance -- --check
237237
cargo fmt --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh -- --check
238238
cargo fmt --manifest-path tests/support/Cargo.toml --all -- --check
239239
cargo fmt --manifest-path tests/openshell-test/Cargo.toml --all -- --check
@@ -298,7 +298,7 @@ jobs:
298298
cargo clippy --locked --manifest-path tests/testkit/Cargo.toml --all-targets -- -D warnings
299299
cargo clippy --locked --manifest-path tests/tmachine/Cargo.toml --all-targets -- -D warnings
300300
cargo clippy --locked --manifest-path tests/runner/Cargo.toml --all-targets -- -D warnings
301-
cargo clippy --locked --manifest-path tests/suites/conformance/Cargo.toml -p sandbox --all-targets -- -D warnings
301+
cargo clippy --locked --manifest-path tests/suites/conformance/Cargo.toml -p conformance --all-targets -- -D warnings
302302
cargo clippy --locked --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh --all-targets -- -D warnings
303303
cargo clippy --locked --manifest-path tests/support/Cargo.toml --all-targets -- -D warnings
304304
cargo clippy --locked --manifest-path tests/openshell-test/Cargo.toml --all-targets -- -D warnings

‎.github/workflows/e2e-suites.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,11 @@ jobs:
6464
# installer registers with the CLI: `openshell` for the packages,
6565
# `tmachine` for the k3s installers.
6666
rows='[
67-
{"environment":"ubuntu-docker-rootful","installer":"deb","family":"conformance","package":"sandbox","gateway":"openshell","runner":"ubuntu-24.04"},
68-
{"environment":"fedora-podman-rootful","installer":"rpm","family":"conformance","package":"sandbox","gateway":"openshell","runner":"ubuntu-24.04"},
69-
{"environment":"fedora-podman-rootless","installer":"rpm","family":"conformance","package":"sandbox","gateway":"openshell","runner":"ubuntu-24.04"},
70-
{"environment":"ubuntu-k3s","installer":"k3s","family":"conformance","package":"sandbox","gateway":"tmachine","runner":"ubuntu-24.04"},
71-
{"environment":"ubuntu-k3s","installer":"k3s-ha","family":"conformance","package":"sandbox","gateway":"tmachine","runner":"ubuntu-24.04"},
67+
{"environment":"ubuntu-docker-rootful","installer":"deb","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"},
68+
{"environment":"fedora-podman-rootful","installer":"rpm","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"},
69+
{"environment":"fedora-podman-rootless","installer":"rpm","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"},
70+
{"environment":"ubuntu-k3s","installer":"k3s","family":"conformance","package":"conformance","gateway":"tmachine","runner":"ubuntu-24.04"},
71+
{"environment":"ubuntu-k3s","installer":"k3s-ha","family":"conformance","package":"conformance","gateway":"tmachine","runner":"ubuntu-24.04"},
7272
{"environment":"ubuntu-docker-rootful-keycloak","installer":"deb","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}
7373
]'
7474
matrix=$(jq -c --arg family "${FAMILY}" \

‎CI.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ JUnit report, nextest log and, after a failure, the guest's diagnostics as an
204204

205205
### Run a subset of the conformance tests
206206

207-
The conformance lanes run the `sandbox` package of `tests/suites/conformance`
207+
The conformance lanes run the `conformance` crate in `tests/suites/conformance`
208208
through `.github/workflows/e2e-suites.yml`, which keeps the list of
209209
environments. To run one suite in one environment from a checkout, build the
210210
candidate inputs (see [TESTING.md](TESTING.md)) and narrow the tests with a
@@ -213,7 +213,7 @@ nextest filterset:
213213
```shell
214214
nix run .#openshell-test-runner -- \
215215
--environment ubuntu-docker-rootful --installer deb \
216-
--family conformance --package sandbox \
216+
--family conformance --package conformance \
217217
--filter 'test(/^policy_advisor::/)'
218218
```
219219

‎TESTING.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ OpenShell that the VM's installer set up.
229229
```shell
230230
nix run .#openshell-test-runner -- \
231231
--environment ubuntu-docker-rootful --installer deb \
232-
--family conformance --package sandbox
232+
--family conformance --package conformance
233233
```
234234

235235
It needs KVM, and tmachine only boots guests of the host's architecture. The
@@ -418,10 +418,10 @@ HTTP `Host`, TLS SNI, and mTLS handling.
418418
Suites:
419419

420420
- Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests.
421-
- Conformance (`tests/suites/conformance`) - the testkit `sandbox` package:
421+
- Conformance (`tests/suites/conformance`) - the testkit `conformance` crate:
422422
lifecycle, the CLI, persistent workspaces, file transfer, and the policy
423423
advisor. Driver E2E runs it with nextest against the lane's gateway. The
424-
installed-artifact conformance lanes run the same package through
424+
installed-artifact conformance lanes run the same crate through
425425
`openshell-test-runner` (see above).
426426
- Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or
427427
`e2e-vm`) - conformance plus the common and driver-specific coverage for
@@ -509,7 +509,7 @@ rebuilds only the RPMs. Run conformance against the installed RPMs:
509509
nix run .#build-artifacts
510510
nix run .#openshell-test-runner -- \
511511
--environment fedora-podman-rootful --installer rpm \
512-
--family conformance --package sandbox
512+
--family conformance --package conformance
513513
```
514514

515515
Run the portable subset in a disposable rootless Podman guest:

‎e2e/policy-advisor/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,13 +62,13 @@ requests through `policy.local`. See [TESTING.md](../../TESTING.md#rust-cli-e2e)
6262
for the supported source and installed-artifact test paths.
6363

6464
To run only the policy-related conformance tests against an installed
65-
candidate, run the `policy-advisor` suite of the `sandbox` package through the
65+
candidate, run the `policy-advisor` suite of the `conformance` crate through the
6666
test runner:
6767

6868
```bash
6969
nix run .#openshell-test-runner -- \
7070
--environment ubuntu-docker-rootful --installer deb \
71-
--family conformance --package sandbox \
71+
--family conformance --package conformance \
7272
--filter 'test(/^policy_advisor::/)'
7373
```
7474

‎tasks/rust.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ run = [
3939
"cargo clippy --manifest-path tests/testkit/Cargo.toml --all-targets -- -D warnings",
4040
"cargo clippy --manifest-path tests/tmachine/Cargo.toml --all-targets -- -D warnings",
4141
"cargo clippy --manifest-path tests/runner/Cargo.toml --all-targets -- -D warnings",
42-
"cargo clippy --manifest-path tests/suites/conformance/Cargo.toml -p sandbox --all-targets -- -D warnings",
42+
"cargo clippy --manifest-path tests/suites/conformance/Cargo.toml -p conformance --all-targets -- -D warnings",
4343
"cargo clippy --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh --all-targets -- -D warnings",
4444
"cargo clippy --manifest-path tests/support/Cargo.toml --all-targets -- -D warnings",
4545
"cargo clippy --manifest-path tests/openshell-test/Cargo.toml --all-targets -- -D warnings",
@@ -58,7 +58,7 @@ run = [
5858
"cargo fmt --manifest-path tests/testkit/Cargo.toml --all",
5959
"cargo fmt --manifest-path tests/tmachine/Cargo.toml --all",
6060
"cargo fmt --manifest-path tests/runner/Cargo.toml --all",
61-
"cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox",
61+
"cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p conformance",
6262
"cargo fmt --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh",
6363
"cargo fmt --manifest-path tests/support/Cargo.toml --all",
6464
"cargo fmt --manifest-path tests/openshell-test/Cargo.toml --all",
@@ -76,7 +76,7 @@ run = [
7676
"cargo fmt --manifest-path tests/testkit/Cargo.toml --all -- --check",
7777
"cargo fmt --manifest-path tests/tmachine/Cargo.toml --all -- --check",
7878
"cargo fmt --manifest-path tests/runner/Cargo.toml --all -- --check",
79-
"cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -- --check",
79+
"cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p conformance -- --check",
8080
"cargo fmt --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh -- --check",
8181
"cargo fmt --manifest-path tests/support/Cargo.toml --all -- --check",
8282
"cargo fmt --manifest-path tests/openshell-test/Cargo.toml --all -- --check",

‎tests/openshell-test/Cargo.lock‎

Lines changed: 11 additions & 11 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/openshell-test/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ path = "src/main.rs"
1818
[dependencies]
1919
openshell-testkit = { path = "../testkit/testkit" }
2020
# Suite crates. Each must also be named in `src/main.rs`.
21-
sandbox = { path = "../suites/conformance/sandbox" }
21+
conformance = { path = "../suites/conformance" }
2222

2323
# Its own workspace: it links suites from several family workspaces.
2424
[workspace]

‎tests/openshell-test/src/main.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ use std::process::ExitCode;
1212

1313
// A dependency that is never named is not linked, and its tests would be
1414
// missing. Name each suite crate here.
15-
use sandbox as _;
15+
use conformance as _;
1616

1717
fn main() -> ExitCode {
1818
testkit::main()

0 commit comments

Comments
 (0)