Skip to content

Commit 62e1f57

Browse files
test(e2e): run the conformance host lanes with openshell-conformance
The Docker, Podman, VM and Kubernetes host lanes ran the conformance suite with cargo nextest. They now run it with openshell-conformance: ready, then run --junit, so CI exercises the program that gateway owners use. The script uses OPENSHELL_CONFORMANCE_BIN when set and otherwise builds the program from the checkout. The JUnit report goes to OPENSHELL_CONFORMANCE_JUNIT or results/conformance-<gateway>.xml; the Kubernetes runner names it after its report so credential-driver runs keep both. Signed-off-by: Matthew Grossman <[email protected]>
1 parent 1d3ed2d commit 62e1f57

3 files changed

Lines changed: 40 additions & 12 deletions

File tree

‎TESTING.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -420,7 +420,9 @@ Suites:
420420
- Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests.
421421
- Conformance (`tests/suites/conformance`) - the testkit `conformance` crate:
422422
lifecycle, the CLI, persistent workspaces, file transfer, and the policy
423-
advisor. Driver E2E runs it with nextest against the lane's gateway. The
423+
advisor. Driver E2E runs it with `openshell-conformance` against the lane's
424+
gateway and writes a JUnit report under `results/`; set
425+
`OPENSHELL_CONFORMANCE_BIN` to use a prebuilt program. The
424426
installed-artifact conformance lanes run the same crate through
425427
`openshell-test-runner` (see above).
426428
- Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or

‎e2e/rust/e2e-kubernetes.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,8 @@ run_conformance() {
7979
return 0
8080
fi
8181

82-
e2e_run_openshell_conformance "Kubernetes"
82+
OPENSHELL_CONFORMANCE_JUNIT="${OPENSHELL_CONFORMANCE_JUNIT:-${ROOT}/results/${OPENSHELL_E2E_REPORT_NAME:-e2e-kubernetes}-conformance.xml}" \
83+
e2e_run_openshell_conformance "Kubernetes"
8384
}
8485

8586
# `OPENSHELL_E2E_REPORT_NAME` (default `e2e-kubernetes`) names the report

‎e2e/support/conformance.sh‎

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,30 +16,55 @@ e2e_require_openshell_bin() {
1616
return 2
1717
fi
1818

19-
# Cargo runs each test from its package directory, so resolve the candidate
20-
# relative to the caller's working directory before passing it to the tests.
19+
# Tests may run from another directory, so resolve the candidate relative
20+
# to the caller's working directory before passing it to them.
2121
local binary_dir
2222
binary_dir="$(cd "$(dirname "${OPENSHELL_BIN}")" && pwd -P)" || return
2323
export OPENSHELL_BIN="${binary_dir}/$(basename "${OPENSHELL_BIN}")"
2424
}
2525

26+
# Print the path of the openshell-conformance program: OPENSHELL_CONFORMANCE_BIN
27+
# if set, else a debug build from this checkout.
28+
e2e_openshell_conformance_bin() {
29+
local root=$1
30+
31+
if [ -n "${OPENSHELL_CONFORMANCE_BIN:-}" ]; then
32+
if [ ! -x "${OPENSHELL_CONFORMANCE_BIN}" ]; then
33+
echo "ERROR: openshell-conformance is not executable: ${OPENSHELL_CONFORMANCE_BIN}" >&2
34+
return 2
35+
fi
36+
echo "${OPENSHELL_CONFORMANCE_BIN}"
37+
return
38+
fi
39+
40+
cargo build \
41+
--locked \
42+
--manifest-path "${root}/tests/suites/conformance/Cargo.toml" \
43+
--bin openshell-conformance >&2 || return
44+
# Cargo resolves a relative CARGO_TARGET_DIR against the working directory.
45+
local target_dir
46+
target_dir="$(cd "${CARGO_TARGET_DIR:-${root}/tests/suites/conformance/target}" && pwd)" || return
47+
echo "${target_dir}/debug/openshell-conformance"
48+
}
49+
50+
# Run the conformance suite with openshell-conformance. The JUnit report goes
51+
# to OPENSHELL_CONFORMANCE_JUNIT, else results/conformance-<gateway>.xml.
2652
e2e_run_openshell_conformance() {
2753
local gateway_label=${1:-OpenShell}
28-
local root
54+
local root program junit
2955
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
3056

3157
e2e_require_openshell_bin || return
58+
program="$(e2e_openshell_conformance_bin "${root}")" || return
59+
junit="${OPENSHELL_CONFORMANCE_JUNIT:-${root}/results/conformance-$(echo "${gateway_label}" | tr '[:upper:]' '[:lower:]').xml}"
60+
mkdir -p "$(dirname "${junit}")" || return
3261

33-
# The tests find the gateway as the CLI does: OPENSHELL_GATEWAY_ENDPOINT,
62+
# The program finds the gateway as the CLI does: OPENSHELL_GATEWAY_ENDPOINT,
3463
# else OPENSHELL_GATEWAY, else the active gateway. The gateway wrappers set
3564
# these for the gateway they started.
3665
echo "==> Running conformance suites against the ${gateway_label} gateway"
37-
cargo nextest run \
38-
--locked \
39-
--manifest-path "${root}/tests/suites/conformance/Cargo.toml" \
40-
--test-threads 1 \
41-
--no-fail-fast \
42-
--no-tests fail
66+
"${program}" ready || return
67+
"${program}" run --junit "${junit}"
4368
}
4469

4570
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then

0 commit comments

Comments
 (0)