Skip to content

Commit ff18e4f

Browse files
committed
test(ci): replace smoke workflow source assertions with parsed YAML checks
1 parent f2629c3 commit ff18e4f

4 files changed

Lines changed: 71 additions & 193 deletions

File tree

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,8 @@
6161
"rimraf": "6.0.1",
6262
"tsx": "4.23.15",
6363
"turbo": "2.11.8",
64-
"typescript": "5.9.3"
64+
"typescript": "5.9.3",
65+
"yaml": "2.9.0"
6566
},
6667
"lint-staged": {
6768
"*.{js,jsx,ts,tsx,json,css,md}": [

‎pnpm-lock.yaml‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎scripts/solheim-smoke-workflow.test.mjs‎

Lines changed: 63 additions & 190 deletions
Original file line numberDiff line numberDiff line change
@@ -5,207 +5,80 @@ import { describe, it } from "node:test"
55
import { spawn, spawnSync } from "node:child_process"
66
import os from "node:os"
77
import path from "node:path"
8-
const workflow = fs.readFileSync(new URL("../.github/workflows/solheim-provider-smoke.yml", import.meta.url), "utf8")
9-
const codeQa = fs.readFileSync(new URL("../.github/workflows/code-qa.yml", import.meta.url), "utf8")
10-
const pkg = JSON.parse(fs.readFileSync(new URL("../package.json", import.meta.url), "utf8"))
11-
const driver = fs.readFileSync(new URL("./solheim-smoke/driver.mts", import.meta.url), "utf8")
12-
const recorder = fs.readFileSync(new URL("./solheim-smoke/record.sh", import.meta.url), "utf8")
13-
const smokePkg = JSON.parse(fs.readFileSync(new URL("./solheim-smoke/package.json", import.meta.url), "utf8"))
14-
const runner = fs.readFileSync(new URL("./agent-evals/runner.ts", import.meta.url), "utf8")
15-
const launcher = fs.readFileSync(new URL("./agent-evals/launcher.ts", import.meta.url), "utf8")
16-
const registry = fs.readFileSync(new URL("./agent-evals/registry.ts", import.meta.url), "utf8")
17-
const fixture = fs.readFileSync(new URL("./solheim-smoke/fixture.ts", import.meta.url), "utf8")
18-
const trace = fs.readFileSync(new URL("./agent-evals/trace.ts", import.meta.url), "utf8")
19-
const evalsPkg = JSON.parse(fs.readFileSync(new URL("./agent-evals/package.json", import.meta.url), "utf8"))
8+
import { parse } from "yaml"
209

21-
describe("Project A provider smoke workflow", () => {
22-
it("runs smoke TypeScript as ESM without requiring compiled workspace packages", () => {
23-
assert.equal(smokePkg.type, "module")
24-
assert.equal(smokePkg.private, true)
25-
})
26-
it("runs only by manual dispatch against the exact trusted main revision", () => {
27-
assert.match(workflow, /workflow_dispatch:/)
28-
assert.match(workflow, /if: github.ref == 'refs\/heads\/main'/)
29-
assert.match(workflow, /ref: \$\{\{ github.sha \}\}/)
30-
assert.ok(!workflow.includes("pull_request_review:"))
31-
assert.ok(!workflow.includes("pull_request_target:"))
32-
assert.match(workflow, /persist-credentials: false/)
33-
assert.match(workflow, /environment: final-vscode-review-smoke/)
34-
})
35-
it("gates the provider secret behind an environment-bound, main-only smoke job", () => {
36-
const jobs = workflow.split("jobs:\n")[1] ?? ""
37-
const smokeJob = jobs.split("\n steps:")[0] ?? ""
38-
assert.ok(smokeJob.length > 0)
39-
// The job runs only on main, so no other ref reaches the secret step.
40-
assert.match(smokeJob, /^ {8}if: github\.ref == 'refs\/heads\/main'$/m)
41-
// The secret-holding environment gates the whole job, so GitHub
42-
// applies its protection rules before the first step runs.
43-
assert.match(smokeJob, /^ {8}environment: final-vscode-review-smoke$/m)
44-
// The environment reference must precede the credential-bearing step.
45-
assert.ok(smokeJob.indexOf("environment: final-vscode-review-smoke") < jobs.indexOf("secrets."))
46-
// The workflow must consume the secret scoped to that environment.
47-
assert.match(workflow, /secrets\.FINAL_SMOKE_OPENAI_API_KEY/)
48-
})
49-
it("documents the required environment protection policy for the secret", () => {
50-
const readme = fs.readFileSync(new URL("./solheim-smoke/README.md", import.meta.url), "utf8")
51-
// The environment requires the same reviewer approval as the deploy
52-
// environment, and the workflow keeps the main-only deployment guard.
53-
assert.match(readme, /same reviewer approval as the `marketplace-production` deploy environment/)
54-
assert.match(readme, /must approve each deployment before GitHub releases the secret/)
55-
assert.match(readme, /runs only on `refs\/heads\/main`/)
10+
// Parse configuration so comments, indentation, and key order are irrelevant.
11+
// GitHub expressions remain strings: these checks protect the configured policy,
12+
// while the recorder tests below exercise process behavior.
13+
const workflow = parse(
14+
fs.readFileSync(new URL("../.github/workflows/solheim-provider-smoke.yml", import.meta.url), "utf8"),
15+
)
16+
const smoke = workflow.jobs.smoke
17+
const steps = smoke.steps
18+
const actionSteps = (action) => steps.filter((step) => step.uses?.startsWith(action + "@"))
19+
20+
describe("Solheim provider smoke workflow policy", () => {
21+
it("allows only manual runs on main behind the protected environment", () => {
22+
assert.deepEqual(Object.keys(workflow.on), ["workflow_dispatch"])
23+
assert.deepEqual(Object.keys(workflow.jobs), ["smoke"])
24+
assert.equal(smoke.if, "github.ref == 'refs/heads/main'")
25+
assert.equal(smoke.environment, "final-vscode-review-smoke")
26+
assert.deepEqual(workflow.permissions, { contents: "read" })
27+
assert.equal(smoke.permissions, undefined)
5628
})
57-
it("has one credential-bearing step, no GitHub write permissions, and no posting", () => {
58-
assert.equal((workflow.match(/secrets\./g) ?? []).length, 1)
59-
const before = workflow.slice(0, workflow.indexOf("- name: Run Solheim provider smoke"))
60-
assert.ok(!before.includes("secrets."))
61-
assert.match(workflow, /permissions:\n\s+contents: read/)
62-
assert.ok(!/\bwrite\b/.test(workflow))
63-
assert.ok(!workflow.includes("github-script"))
64-
assert.ok(!workflow.includes("post-review"))
29+
30+
it("checks out the triggering revision without persisting credentials", () => {
31+
const checkouts = actionSteps("actions/checkout")
32+
assert.equal(checkouts.length, 1)
33+
assert.equal(checkouts[0].with.ref, "${{ github.sha }}")
34+
assert.equal(checkouts[0].with["persist-credentials"], false)
6535
})
66-
it("has one smoke job and no review, routing, shadow, or deterministic lanes", () => {
67-
const jobs = workflow.split("jobs:\n")[1] ?? ""
68-
assert.deepEqual(
69-
[...jobs.matchAll(/^ {4}([a-z-]+):$/gm)].map((match) => match[1]),
70-
["smoke"],
36+
37+
it("exposes the provider credential only to the bounded provider step", () => {
38+
const provider = steps.find((step) => step.id === "provider")
39+
assert.ok(provider, "missing provider step")
40+
assert.equal(provider.env.SOLHEIM_API_KEY, "${{ secrets.FINAL_SMOKE_OPENAI_API_KEY }}")
41+
assert.equal(workflow.env, undefined)
42+
assert.equal(smoke.env, undefined)
43+
// Inspect parsed values, excluding comments and display names.
44+
const credentialSteps = steps.filter((step) =>
45+
JSON.stringify({ env: step.env, with: step.with, run: step.run }).includes("secrets."),
7146
)
72-
for (const name of [
73-
"theme-review",
74-
"probe-dispatch",
75-
"rules-review",
76-
"merge-report",
77-
"bounded-code-review",
78-
"model-qualification",
79-
])
80-
assert.ok(!workflow.includes(name))
81-
assert.ok(!driver.includes("model-qualification"))
82-
assert.ok(!driver.includes("AcknowledgeCompletion"))
83-
})
84-
it("bounds runtime and serializes the single provider instance", () => {
85-
assert.match(workflow, /cancel-in-progress: false/)
86-
assert.match(workflow, /timeout-minutes: 8/)
87-
assert.match(workflow, /^ {8}timeout-minutes: 25$/m)
47+
assert.deepEqual(credentialSteps, [provider])
48+
assert.ok(provider["timeout-minutes"] > 0 && provider["timeout-minutes"] <= 8)
49+
assert.ok(smoke["timeout-minutes"] > provider["timeout-minutes"] && smoke["timeout-minutes"] <= 25)
50+
assert.equal(workflow.concurrency.group, "solheim-provider-smoke")
51+
assert.equal(workflow.concurrency["cancel-in-progress"], false)
8852
})
89-
it("uploads metadata and the recording as separate validated artifacts", () => {
90-
const metadataStep =
91-
workflow.split("- name: Upload provider smoke verdict and trace")[1]?.split("- name:")[0] ?? ""
92-
const recordingStep = workflow.split("- name: Upload provider smoke recording")[1]?.split("- name:")[0] ?? ""
93-
assert.ok(metadataStep.length > 0)
94-
assert.ok(recordingStep.length > 0)
95-
assert.deepEqual(
96-
metadataStep
97-
.split("path: |")[1]
98-
?.split("if-no-files-found:")[0]
53+
54+
it("uploads metadata and the final recording separately, including after provider failure", () => {
55+
const uploads = actionSteps("actions/upload-artifact")
56+
assert.equal(uploads.length, 2)
57+
const output = "${{ runner.temp }}/solheim-provider-smoke"
58+
const paths = uploads.map((step) =>
59+
step.with.path
9960
.trim()
10061
.split("\n")
10162
.map((line) => line.trim()),
102-
[
103-
"${{ runner.temp }}/solheim-provider-smoke/verdict.json",
104-
"${{ runner.temp }}/solheim-provider-smoke/trace.json",
105-
],
10663
)
107-
// The recording artifact carries only the exact validated final path.
108-
assert.equal(
109-
recordingStep.split("path: ")[1]?.split("\n")[0].trim(),
110-
"${{ runner.temp }}/solheim-provider-smoke/smoke.mp4",
111-
)
112-
for (const step of [metadataStep, recordingStep]) {
113-
assert.match(step, /!cancelled\(\)/)
114-
assert.match(step, /if-no-files-found: error/)
115-
assert.match(step, /retention-days: 7/)
116-
assert.match(step, /actions\/upload-artifact@[0-9a-f]{40}/)
117-
assert.ok(!step.includes("*"))
64+
assert.deepEqual(paths, [[output + "/verdict.json", output + "/trace.json"], [output + "/smoke.mp4"]])
65+
for (const step of uploads) {
66+
assert.equal(
67+
step.if,
68+
"${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }}",
69+
)
70+
assert.equal(step.with["if-no-files-found"], "error")
71+
assert.equal(step.with["retention-days"], 7)
72+
assert.ok(steps.indexOf(step) > steps.findIndex((candidate) => candidate.id === "provider"))
11873
}
119-
// A missing recording cannot hide behind a multi-path if-no-files-found.
120-
assert.match(workflow, /name: solheim-provider-smoke-recording-\$\{\{ github\.sha \}\}/)
121-
// Temporary recorder output is never an upload path.
122-
assert.ok(!workflow.includes(".tmp"))
123-
assert.ok(!driver.includes("console.error("))
124-
assert.ok(!driver.includes("stderrLog"))
125-
})
126-
it("reports a missing or failed recording explicitly before the uploads", () => {
127-
const checkStep = workflow.split("- name: Verify recording integrity")[1]?.split("- name:")[0] ?? ""
128-
// The published final is the commit marker; no separate status file exists.
129-
assert.match(checkStep, /test -s "\$SOLHEIM_SMOKE_OUT_DIR\/smoke\.mp4"/)
130-
assert.ok(!workflow.includes("recording-status"))
131-
assert.match(checkStep, /!cancelled\(\)/)
132-
const checkIndex = workflow.indexOf("- name: Verify recording integrity")
133-
const uploadIndex = workflow.indexOf("- name: Upload provider smoke verdict and trace")
134-
assert.ok(checkIndex >= 0 && checkIndex < uploadIndex)
135-
})
136-
it("records a fresh display with credential-free ffmpeg and bounded teardown", () => {
137-
assert.match(workflow, /install -y xvfb ffmpeg/)
138-
assert.match(workflow, /xvfb-run -a -s "-screen 0 1280x720x24 -nolisten tcp"/)
139-
assert.match(recorder, /env -i "PATH=\$PATH" "DISPLAY=\$DISPLAY"/)
140-
assert.match(recorder, /-t 480 -fs 125829120/)
141-
assert.match(recorder, /frag_keyframe\+empty_moov/)
142-
assert.match(recorder, /trap finish EXIT/)
143-
assert.match(recorder, /attempt < 50/)
144-
assert.ok(!recorder.includes("SOLHEIM_API_KEY"))
145-
assert.match(workflow, /retention-days: 7/)
146-
// The recorder writes a unique temporary file inside the output
147-
// directory, forces the mp4 muxer because the name has no mp4 suffix,
148-
// and publishes only validated output with an atomic rename.
149-
assert.match(recorder, /temp_recording="\$SOLHEIM_SMOKE_OUT_DIR\/\.smoke\.mp4\.\$\$\.tmp"/)
150-
assert.match(recorder, /-f mp4 "\$temp_recording"/)
151-
assert.match(recorder, /ffprobe -v error "\$temp_recording"/)
152-
assert.match(recorder, /mv -f "\$temp_recording" "\$final_recording"/)
153-
assert.match(recorder, /rm -f "\$final_recording"/)
154-
// ffmpeg must never write the final path directly.
155-
assert.ok(!/ffmpeg[^\n]*\$final_recording/.test(recorder))
156-
})
157-
it("signals the Node driver directly with a documented stop budget", () => {
158-
// A package-manager wrapper between the recorder and the driver would
159-
// receive the TERM and KILL meant for the driver that performs teardown
160-
// and the atomic verdict and trace writes.
161-
assert.ok(!recorder.includes("pnpm"))
162-
assert.match(recorder, /node --import tsx "\$script_dir\/driver\.mts" &/)
163-
assert.match(recorder, /SOLHEIM_SMOKE_STOP_BUDGET_S:-30/)
164-
assert.match(recorder, /kill -TERM "\$provider_pid"/)
165-
assert.match(recorder, /kill -KILL "\$provider_pid"/)
166-
assert.match(recorder, /wait "\$provider_pid"/)
167-
assert.ok(!recorder.includes("status_file"))
168-
})
169-
it("keeps Code QA and package commands focused on smoke contracts", () => {
170-
for (const command of [
171-
"test:solheim-smoke:unit",
172-
"test:solheim-smoke-ci",
173-
"solheim-smoke:check-types",
174-
"test:agent-evals:unit",
175-
"agent-evals:check-types",
176-
])
177-
assert.ok(codeQa.includes(command))
178-
assert.ok(!codeQa.includes("model-qualification"))
179-
assert.ok(!Object.keys(pkg.scripts).some((name) => name.includes("model-qualification")))
180-
assert.ok(!codeQa.includes("final-smoke"))
181-
assert.equal(pkg.scripts["solheim:smoke"], "node --import tsx scripts/solheim-smoke/driver.mts")
182-
assert.equal(pkg.scripts["test:agent-evals:unit"], "node --import tsx --test scripts/agent-evals/*.test.ts")
183-
assert.equal(pkg.scripts["agent-evals:check-types"], "tsc -p scripts/agent-evals/tsconfig.json")
184-
assert.ok(!fs.existsSync(new URL("../.github/workflows/final-vscode-review-smoke.yml", import.meta.url)))
74+
assert.equal(new Set(uploads.map((step) => step.with.name)).size, 2)
18575
})
186-
it("keeps the shared controller free of smoke, provider, and solution knowledge", () => {
187-
assert.equal(evalsPkg.type, "module")
188-
assert.equal(evalsPkg.private, true)
189-
assert.ok(!/solheim/i.test(runner))
190-
assert.ok(!/smoke/i.test(runner))
191-
assert.ok(!runner.includes("SOLHEIM_API_KEY"))
192-
assert.ok(!/solheim/i.test(trace))
193-
assert.ok(trace.includes("traceSchemaVersion"))
194-
})
195-
it("registers fixtures only through the validated in-memory registry", () => {
196-
assert.ok(!registry.includes("import("))
197-
assert.ok(!registry.includes("fetch("))
198-
assert.ok(!registry.includes("readdir"))
199-
assert.ok(fixture.includes("registerFixture"))
200-
})
201-
it("keeps VS Code launch isolation in one shared launcher with an atomic artifact writer", () => {
202-
assert.ok(driver.includes("createHostRuntime"))
203-
assert.ok(!driver.includes("spawn("))
204-
assert.ok(!driver.includes("downloadAndUnzipVSCode"))
205-
assert.ok(launcher.includes("downloadAndUnzipVSCode"))
206-
assert.ok(launcher.includes("buildChildEnvironment"))
207-
assert.ok(launcher.includes("await fs.rename(pending"))
208-
assert.ok(launcher.includes("process.kill(-child.pid"))
76+
77+
it("pins remote actions to immutable commit revisions", () => {
78+
for (const step of steps.filter((step) => step.uses && !step.uses.startsWith("./"))) {
79+
const revision = step.uses.split("@")[1]
80+
assert.match(revision ?? "", /^[0-9a-f]{40}$/, step.uses)
81+
}
20982
})
21083
})
21184

‎scripts/solheim-smoke/README.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,9 @@ The protected environment `final-vscode-review-smoke` and its secret `FINAL_SMOK
2121
retain their existing names to avoid an unnecessary repository-settings migration. The environment
2222
requires the same reviewer approval as the `marketplace-production` deploy environment: a member of
2323
the `Zoo-Code-Org/admins` team must approve each deployment before GitHub releases the secret. The
24-
main-only deployment restriction stays in the workflow: the smoke job runs only on `refs/heads/main`.
25-
Workflow files cannot declare reviewer rules; repository settings own them.
24+
environment also allows deployments from one selected branch, `main`, and no tag policies. The
25+
workflow keeps its own guard: the smoke job runs only on `refs/heads/main`. Workflow files cannot
26+
declare reviewer rules; repository settings own both protections.
2627

2728
The provider endpoint and model are fixed: `https://api.solheim.ai/v1`, `qwen3.8-27b`. The task has no tool
2829
groups; delegation, follow-ups, writes, shell and MCP are disabled. The driver opens an empty temporary

0 commit comments

Comments
 (0)