Skip to content

Commit b3313c8

Browse files
committed
ci(windows): run conformance as cargo tests
Signed-off-by: Evan Lezar <[email protected]>
1 parent 32c1b0d commit b3313c8

5 files changed

Lines changed: 40 additions & 39 deletions

File tree

‎.agents/skills/build-openshell-mxc-windows/SKILL.md‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -252,11 +252,12 @@ manual dispatches run the same lint and test commands in a cache-seed job,
252252
followed by a dependent release-binary build and mock-example job. The seed and
253253
PR jobs use the same cache namespaces. Merge queues do not run this workflow.
254254
Main/manual seed and build jobs use job-level `continue-on-error: true`; opt-in
255-
PR jobs report failures normally, except that the full mock-MXC conformance
256-
step is advisory on every hosted lane so known capability gaps are archived
257-
without blocking the job. Applying the label alone does not start a run: re-run
258-
all jobs in the current mirror push run, or push a new mirrored commit. The
259-
binaries are not uploaded or published.
255+
PR jobs report failures normally. The mock-MXC conformance task keeps completed
256+
Cargo test failures advisory so known capability gaps are archived without
257+
blocking the job, while compilation and gateway setup failures remain fatal.
258+
Applying the label alone does not start a run: re-run all jobs in the current
259+
mirror push run, or push a new mirrored commit. The binaries are not uploaded
260+
or published.
260261

261262
The ARM64 check/build steps in this x64-host contract are cross-builds. The
262263
wrapper discovers and adds host-native LLVM and Ninja to `PATH`, requires the
@@ -296,8 +297,8 @@ crypto dependency builds.
296297
| `windows:build:arm64` | Release-builds `openshell-gateway.exe`, `openshell.exe`, and `openshell-supervisor-relay.exe` for ARM64. |
297298
| `windows:test:x64` | Runs native x64 workspace tests with `--no-fail-fast`, excluding unsupported Windows packages as top-level workspace targets. |
298299
| `windows:test:arm64` | Runs native ARM64 workspace tests with `--no-fail-fast` and the same package exclusions. Rejects non-ARM64 hosts. |
299-
| `windows:test:conformance:x64` | Runs every conformance scenario against the mock MXC gateway on native x64, then exits nonzero if any scenario failed. Hosted CI treats that result as advisory and uploads the complete report. |
300-
| `windows:test:conformance:arm64` | Runs the same complete mock-MXC report natively on ARM64. Rejects non-ARM64 hosts. Hosted CI treats scenario failures as advisory. |
300+
| `windows:test:conformance:x64` | Compiles and runs every CLI conformance Cargo test serially against the mock MXC gateway on native x64. Completed scenario failures are advisory; compilation and gateway setup failures remain fatal. |
301+
| `windows:test:conformance:arm64` | Runs the same complete mock-MXC Cargo test report natively on ARM64. Rejects non-ARM64 hosts and preserves the same failure boundary. |
301302
| `windows:test:unsupported:x64` | Re-runs focused `openshell-gateway` tests for unsupported Windows driver behavior. |
302303
| `windows:test:unsupported:arm64` | Re-runs the same focused contracts natively on ARM64. Rejects non-ARM64 hosts. |
303304
| `windows:test:mxc-real:x64` | Runs the serial, ignored real-`wxc-exec` integration suite natively on x64 through the MSVC wrapper. Rejects non-x64 hosts. |
@@ -387,9 +388,9 @@ Useful log files:
387388
| `build-aarch64-pc-windows-msvc-release.log` | ARM64 release build output. |
388389
| `test-x86_64-pc-windows-msvc.log` | Full native x64 workspace test output. |
389390
| `test-aarch64-pc-windows-msvc.log` | Full native ARM64 workspace test output. |
390-
| `build-<target>-conformance.log` | Build output for the CLI, gateway, and standalone conformance runner. |
391-
| `test-<target>-conformance.log` | Complete mock-MXC conformance scenario report. |
392-
| `test-<target>-conformance.err.log` | Conformance runner standard error. |
391+
| `build-<target>-conformance.log` | Build output for the CLI and gateway. |
392+
| `build-<target>-conformance-tests.log` | Compile-only output for the CLI conformance Cargo test package. |
393+
| `test-<target>-conformance.log` | Complete mock-MXC Cargo test report, including standard output and error. |
393394
| `test-<target>-conformance-gateway.log` | Mock-MXC gateway standard output. |
394395
| `test-<target>-conformance-gateway.err.log` | Mock-MXC gateway standard error. |
395396
| `test-x86_64-pc-windows-msvc-unsupported-*.log` | Focused unsupported-driver contract output. |

‎.github/workflows/windows-msvc.yml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,6 @@ jobs:
9999
- name: Test
100100
run: mise run --skip-tools windows:test:${{ matrix.arch }}
101101
- name: Full MXC conformance
102-
continue-on-error: true
103102
run: mise run --skip-tools windows:test:conformance:${{ matrix.arch }}
104103
- name: Upload MXC conformance report
105104
if: always()
@@ -108,8 +107,8 @@ jobs:
108107
name: windows-mxc-conformance-${{ matrix.arch }}
109108
path: |
110109
build-${{ matrix.rust_target }}-conformance.log
110+
build-${{ matrix.rust_target }}-conformance-tests.log
111111
test-${{ matrix.rust_target }}-conformance.log
112-
test-${{ matrix.rust_target }}-conformance.err.log
113112
test-${{ matrix.rust_target }}-conformance-gateway.log
114113
test-${{ matrix.rust_target }}-conformance-gateway.err.log
115114
if-no-files-found: warn
@@ -186,7 +185,6 @@ jobs:
186185
- name: Test
187186
run: mise run --skip-tools windows:test:${{ matrix.arch }}
188187
- name: Full MXC conformance
189-
continue-on-error: true
190188
run: mise run --skip-tools windows:test:conformance:${{ matrix.arch }}
191189
- name: Upload MXC conformance report
192190
if: always()
@@ -195,8 +193,8 @@ jobs:
195193
name: windows-mxc-conformance-${{ matrix.arch }}
196194
path: |
197195
build-${{ matrix.rust_target }}-conformance.log
196+
build-${{ matrix.rust_target }}-conformance-tests.log
198197
test-${{ matrix.rust_target }}-conformance.log
199-
test-${{ matrix.rust_target }}-conformance.err.log
200198
test-${{ matrix.rust_target }}-conformance-gateway.log
201199
test-${{ matrix.rust_target }}-conformance-gateway.err.log
202200
if-no-files-found: warn

‎architecture/windows.md‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -246,14 +246,14 @@ Windows validation separates source correctness from host capability:
246246
fail-closed loss handling without requiring MXC.
247247
- The architecture-specific `windows:*` tasks check, lint, build, and run
248248
workspace and unsupported-driver contract tests for x64 and ARM64.
249-
- Hosted Windows CI runs the complete archive conformance suite against the
250-
in-process MXC mock through the standalone `openshell-conformance` runner.
251-
Every atomic scenario runs so unsupported operations appear as explicit
252-
failures in the archived report instead of being skipped. This report is
253-
advisory while those capability gaps remain: its scenario failures do not
254-
fail the surrounding Windows job, and CI always uploads the runner and
255-
gateway logs. Portable fixtures supply Windows commands, paths, and policies
256-
only to establish each scenario's prerequisites. They keep control-plane
249+
- Hosted Windows CI runs the complete CLI conformance Cargo test package
250+
against the in-process MXC mock. Every atomic scenario runs serially so
251+
unsupported operations appear as explicit failures in the archived report
252+
instead of being skipped. Scenario failures remain advisory while those
253+
capability gaps exist, but compilation, gateway startup, and gateway
254+
selection failures still fail the surrounding Windows job. CI always uploads
255+
the Cargo and gateway logs. Portable fixtures supply Windows commands, paths,
256+
and policies only to establish each scenario's prerequisites. They keep control-plane
257257
lifecycle coverage independent from restart/exec coverage and ensure that
258258
file-transfer and policy failures identify the public unsupported operation
259259
rather than an invalid Unix workload definition.

‎tasks/scripts/windows-msvc.ps1‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -397,7 +397,8 @@ function Invoke-VsCargo {
397397
param(
398398
[Parameter(Mandatory = $true)] [string] $RustTarget,
399399
[Parameter(Mandatory = $true)] [string] $CargoArgs,
400-
[Parameter(Mandatory = $true)] [string] $LogName
400+
[Parameter(Mandatory = $true)] [string] $LogName,
401+
[switch] $AllowNonZeroExit
401402
)
402403

403404
& rustup target add $RustTarget
@@ -464,6 +465,10 @@ function Invoke-VsCargo {
464465
Get-Content $logPath
465466
}
466467
if ($exitCode -ne 0) {
468+
if ($AllowNonZeroExit) {
469+
Write-Warning "Command completed with exit code $exitCode. See $logPath"
470+
return
471+
}
467472
throw "Command failed with exit code $exitCode. See $logPath"
468473
}
469474
} finally {
@@ -564,13 +569,17 @@ function Invoke-ConformanceTest([string] $RustTarget) {
564569

565570
Invoke-VsCargo `
566571
-RustTarget $RustTarget `
567-
-CargoArgs "cargo build --target $RustTarget --bin openshell --bin openshell-gateway --bin openshell-conformance $Z3GatewayFeatures" `
572+
-CargoArgs "cargo build --target $RustTarget --bin openshell --bin openshell-gateway $Z3GatewayFeatures" `
568573
-LogName "build-$RustTarget-conformance.log"
569574

575+
Invoke-VsCargo `
576+
-RustTarget $RustTarget `
577+
-CargoArgs "cargo test --locked --manifest-path tests/suites/conformance/Cargo.toml --package openshell-test-conformance-cli --target $RustTarget --no-run" `
578+
-LogName "build-$RustTarget-conformance-tests.log"
579+
570580
$binaryDir = Join-Path $TargetDir "$RustTarget\debug"
571581
$gateway = Join-Path $binaryDir "openshell-gateway.exe"
572582
$cli = Join-Path $binaryDir "openshell.exe"
573-
$conformance = Join-Path $binaryDir "openshell-conformance.exe"
574583

575584
$buildRoot = Join-Path $binaryDir "build"
576585
$z3Candidates = @(
@@ -593,8 +602,6 @@ function Invoke-ConformanceTest([string] $RustTarget) {
593602
$gatewayConfig = Join-Path $testRoot "gateway.toml"
594603
$gatewayLog = Join-Path $LogDir "test-$RustTarget-conformance-gateway.log"
595604
$gatewayErrorLog = Join-Path $LogDir "test-$RustTarget-conformance-gateway.err.log"
596-
$conformanceLog = Join-Path $LogDir "test-$RustTarget-conformance.log"
597-
$conformanceErrorLog = Join-Path $LogDir "test-$RustTarget-conformance.err.log"
598605
New-Item -ItemType Directory -Force -Path $configRoot, $stateRoot | Out-Null
599606
@"
600607
[openshell]
@@ -641,16 +648,11 @@ backend = "process_container"
641648
& $cli gateway select windows-conformance
642649
if ($LASTEXITCODE -ne 0) { throw "Failed to select the conformance gateway." }
643650

644-
Write-Host "==> $conformance run --openshell-bin $cli"
645-
Write-Host " log: $conformanceLog"
646-
$conformanceProcess = Start-Process -FilePath $conformance `
647-
-ArgumentList @("run", "--openshell-bin", "`"$cli`"") `
648-
-PassThru -Wait -NoNewWindow `
649-
-RedirectStandardOutput $conformanceLog -RedirectStandardError $conformanceErrorLog
650-
Get-Content -LiteralPath $conformanceLog, $conformanceErrorLog -ErrorAction SilentlyContinue
651-
if ($conformanceProcess.ExitCode -ne 0) {
652-
throw "Conformance command failed with exit code $($conformanceProcess.ExitCode). See $conformanceLog"
653-
}
651+
Invoke-VsCargo `
652+
-RustTarget $RustTarget `
653+
-CargoArgs "cargo test --locked --manifest-path tests/suites/conformance/Cargo.toml --package openshell-test-conformance-cli --target $RustTarget --no-fail-fast -- --test-threads=1 --nocapture" `
654+
-LogName "test-$RustTarget-conformance.log" `
655+
-AllowNonZeroExit
654656
} catch {
655657
Get-Content -LiteralPath $gatewayLog, $gatewayErrorLog -ErrorAction SilentlyContinue
656658
throw

‎tasks/windows.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,12 @@ run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1"
5656
run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-precommit aarch64-pc-windows-msvc"
5757

5858
["windows:test:conformance:x64"]
59-
description = "Report all openshell-conformance scenarios against a mock MXC gateway on native Windows x64"
59+
description = "Run all CLI conformance Cargo tests against a mock MXC gateway on native Windows x64"
6060
run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1"
6161
run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-conformance x86_64-pc-windows-msvc"
6262

6363
["windows:test:conformance:arm64"]
64-
description = "Report all openshell-conformance scenarios against a mock MXC gateway on native Windows ARM64"
64+
description = "Run all CLI conformance Cargo tests against a mock MXC gateway on native Windows ARM64"
6565
run = "echo 'windows:* tasks require a Windows MSVC host' && exit 1"
6666
run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-conformance aarch64-pc-windows-msvc"
6767

0 commit comments

Comments
 (0)