@@ -5,207 +5,80 @@ import { describe, it } from "node:test"
55import { spawn , spawnSync } from "node:child_process"
66import os from "node:os"
77import path from "node:path"
8- const workflow = fs . readFileSync ( new URL ( "../.github/workflows/solheim-provider-smoke.yml" , import . meta. url ) , "utf8" )
9- const codeQa = fs . readFileSync ( new URL ( "../.github/workflows/code-qa.yml" , import . meta. url ) , "utf8" )
10- const pkg = JSON . parse ( fs . readFileSync ( new URL ( "../package.json" , import . meta. url ) , "utf8" ) )
11- const driver = fs . readFileSync ( new URL ( "./solheim-smoke/driver.mts" , import . meta. url ) , "utf8" )
12- const recorder = fs . readFileSync ( new URL ( "./solheim-smoke/record.sh" , import . meta. url ) , "utf8" )
13- const smokePkg = JSON . parse ( fs . readFileSync ( new URL ( "./solheim-smoke/package.json" , import . meta. url ) , "utf8" ) )
14- const runner = fs . readFileSync ( new URL ( "./agent-evals/runner.ts" , import . meta. url ) , "utf8" )
15- const launcher = fs . readFileSync ( new URL ( "./agent-evals/launcher.ts" , import . meta. url ) , "utf8" )
16- const registry = fs . readFileSync ( new URL ( "./agent-evals/registry.ts" , import . meta. url ) , "utf8" )
17- const fixture = fs . readFileSync ( new URL ( "./solheim-smoke/fixture.ts" , import . meta. url ) , "utf8" )
18- const trace = fs . readFileSync ( new URL ( "./agent-evals/trace.ts" , import . meta. url ) , "utf8" )
19- const evalsPkg = JSON . parse ( fs . readFileSync ( new URL ( "./agent-evals/package.json" , import . meta. url ) , "utf8" ) )
8+ import { parse } from "yaml"
209
21- describe ( "Project A provider smoke workflow" , ( ) => {
22- it ( "runs smoke TypeScript as ESM without requiring compiled workspace packages" , ( ) => {
23- assert . equal ( smokePkg . type , "module" )
24- assert . equal ( smokePkg . private , true )
25- } )
26- it ( "runs only by manual dispatch against the exact trusted main revision" , ( ) => {
27- assert . match ( workflow , / w o r k f l o w _ d i s p a t c h : / )
28- assert . match ( workflow , / i f : g i t h u b .r e f = = ' r e f s \/ h e a d s \/ m a i n ' / )
29- assert . match ( workflow , / r e f : \$ \{ \{ g i t h u b .s h a \} \} / )
30- assert . ok ( ! workflow . includes ( "pull_request_review:" ) )
31- assert . ok ( ! workflow . includes ( "pull_request_target:" ) )
32- assert . match ( workflow , / p e r s i s t - c r e d e n t i a l s : f a l s e / )
33- assert . match ( workflow , / e n v i r o n m e n t : f i n a l - v s c o d e - r e v i e w - s m o k e / )
34- } )
35- it ( "gates the provider secret behind an environment-bound, main-only smoke job" , ( ) => {
36- const jobs = workflow . split ( "jobs:\n" ) [ 1 ] ?? ""
37- const smokeJob = jobs . split ( "\n steps:" ) [ 0 ] ?? ""
38- assert . ok ( smokeJob . length > 0 )
39- // The job runs only on main, so no other ref reaches the secret step.
40- assert . match ( smokeJob , / ^ { 8 } i f : g i t h u b \. r e f = = ' r e f s \/ h e a d s \/ m a i n ' $ / m)
41- // The secret-holding environment gates the whole job, so GitHub
42- // applies its protection rules before the first step runs.
43- assert . match ( smokeJob , / ^ { 8 } e n v i r o n m e n t : f i n a l - v s c o d e - r e v i e w - s m o k e $ / m)
44- // The environment reference must precede the credential-bearing step.
45- assert . ok ( smokeJob . indexOf ( "environment: final-vscode-review-smoke" ) < jobs . indexOf ( "secrets." ) )
46- // The workflow must consume the secret scoped to that environment.
47- assert . match ( workflow , / s e c r e t s \. F I N A L _ S M O K E _ O P E N A I _ A P I _ K E Y / )
48- } )
49- it ( "documents the required environment protection policy for the secret" , ( ) => {
50- const readme = fs . readFileSync ( new URL ( "./solheim-smoke/README.md" , import . meta. url ) , "utf8" )
51- // The environment requires the same reviewer approval as the deploy
52- // environment, and the workflow keeps the main-only deployment guard.
53- assert . match ( readme , / s a m e r e v i e w e r a p p r o v a l a s t h e ` m a r k e t p l a c e - p r o d u c t i o n ` d e p l o y e n v i r o n m e n t / )
54- assert . match ( readme , / m u s t a p p r o v e e a c h d e p l o y m e n t b e f o r e G i t H u b r e l e a s e s t h e s e c r e t / )
55- assert . match ( readme , / r u n s o n l y o n ` r e f s \/ h e a d s \/ m a i n ` / )
10+ // Parse configuration so comments, indentation, and key order are irrelevant.
11+ // GitHub expressions remain strings: these checks protect the configured policy,
12+ // while the recorder tests below exercise process behavior.
13+ const workflow = parse (
14+ fs . readFileSync ( new URL ( "../.github/workflows/solheim-provider-smoke.yml" , import . meta. url ) , "utf8" ) ,
15+ )
16+ const smoke = workflow . jobs . smoke
17+ const steps = smoke . steps
18+ const actionSteps = ( action ) => steps . filter ( ( step ) => step . uses ?. startsWith ( action + "@" ) )
19+
20+ describe ( "Solheim provider smoke workflow policy" , ( ) => {
21+ it ( "allows only manual runs on main behind the protected environment" , ( ) => {
22+ assert . deepEqual ( Object . keys ( workflow . on ) , [ "workflow_dispatch" ] )
23+ assert . deepEqual ( Object . keys ( workflow . jobs ) , [ "smoke" ] )
24+ assert . equal ( smoke . if , "github.ref == 'refs/heads/main'" )
25+ assert . equal ( smoke . environment , "final-vscode-review-smoke" )
26+ assert . deepEqual ( workflow . permissions , { contents : "read" } )
27+ assert . equal ( smoke . permissions , undefined )
5628 } )
57- it ( "has one credential-bearing step, no GitHub write permissions, and no posting" , ( ) => {
58- assert . equal ( ( workflow . match ( / s e c r e t s \. / g) ?? [ ] ) . length , 1 )
59- const before = workflow . slice ( 0 , workflow . indexOf ( "- name: Run Solheim provider smoke" ) )
60- assert . ok ( ! before . includes ( "secrets." ) )
61- assert . match ( workflow , / p e r m i s s i o n s : \n \s + c o n t e n t s : r e a d / )
62- assert . ok ( ! / \b w r i t e \b / . test ( workflow ) )
63- assert . ok ( ! workflow . includes ( "github-script" ) )
64- assert . ok ( ! workflow . includes ( "post-review" ) )
29+
30+ it ( "checks out the triggering revision without persisting credentials" , ( ) => {
31+ const checkouts = actionSteps ( "actions/checkout" )
32+ assert . equal ( checkouts . length , 1 )
33+ assert . equal ( checkouts [ 0 ] . with . ref , "${{ github.sha }}" )
34+ assert . equal ( checkouts [ 0 ] . with [ "persist-credentials" ] , false )
6535 } )
66- it ( "has one smoke job and no review, routing, shadow, or deterministic lanes" , ( ) => {
67- const jobs = workflow . split ( "jobs:\n" ) [ 1 ] ?? ""
68- assert . deepEqual (
69- [ ...jobs . matchAll ( / ^ { 4 } ( [ a - z - ] + ) : $ / gm) ] . map ( ( match ) => match [ 1 ] ) ,
70- [ "smoke" ] ,
36+
37+ it ( "exposes the provider credential only to the bounded provider step" , ( ) => {
38+ const provider = steps . find ( ( step ) => step . id === "provider" )
39+ assert . ok ( provider , "missing provider step" )
40+ assert . equal ( provider . env . SOLHEIM_API_KEY , "${{ secrets.FINAL_SMOKE_OPENAI_API_KEY }}" )
41+ assert . equal ( workflow . env , undefined )
42+ assert . equal ( smoke . env , undefined )
43+ // Inspect parsed values, excluding comments and display names.
44+ const credentialSteps = steps . filter ( ( step ) =>
45+ JSON . stringify ( { env : step . env , with : step . with , run : step . run } ) . includes ( "secrets." ) ,
7146 )
72- for ( const name of [
73- "theme-review" ,
74- "probe-dispatch" ,
75- "rules-review" ,
76- "merge-report" ,
77- "bounded-code-review" ,
78- "model-qualification" ,
79- ] )
80- assert . ok ( ! workflow . includes ( name ) )
81- assert . ok ( ! driver . includes ( "model-qualification" ) )
82- assert . ok ( ! driver . includes ( "AcknowledgeCompletion" ) )
83- } )
84- it ( "bounds runtime and serializes the single provider instance" , ( ) => {
85- assert . match ( workflow , / c a n c e l - i n - p r o g r e s s : f a l s e / )
86- assert . match ( workflow , / t i m e o u t - m i n u t e s : 8 / )
87- assert . match ( workflow , / ^ { 8 } t i m e o u t - m i n u t e s : 2 5 $ / m)
47+ assert . deepEqual ( credentialSteps , [ provider ] )
48+ assert . ok ( provider [ "timeout-minutes" ] > 0 && provider [ "timeout-minutes" ] <= 8 )
49+ assert . ok ( smoke [ "timeout-minutes" ] > provider [ "timeout-minutes" ] && smoke [ "timeout-minutes" ] <= 25 )
50+ assert . equal ( workflow . concurrency . group , "solheim-provider-smoke" )
51+ assert . equal ( workflow . concurrency [ "cancel-in-progress" ] , false )
8852 } )
89- it ( "uploads metadata and the recording as separate validated artifacts" , ( ) => {
90- const metadataStep =
91- workflow . split ( "- name: Upload provider smoke verdict and trace" ) [ 1 ] ?. split ( "- name:" ) [ 0 ] ?? ""
92- const recordingStep = workflow . split ( "- name: Upload provider smoke recording" ) [ 1 ] ?. split ( "- name:" ) [ 0 ] ?? ""
93- assert . ok ( metadataStep . length > 0 )
94- assert . ok ( recordingStep . length > 0 )
95- assert . deepEqual (
96- metadataStep
97- . split ( "path: |" ) [ 1 ]
98- ?. split ( "if-no-files-found:" ) [ 0 ]
53+
54+ it ( "uploads metadata and the final recording separately, including after provider failure" , ( ) => {
55+ const uploads = actionSteps ( "actions/upload-artifact" )
56+ assert . equal ( uploads . length , 2 )
57+ const output = "${{ runner.temp }}/solheim-provider-smoke"
58+ const paths = uploads . map ( ( step ) =>
59+ step . with . path
9960 . trim ( )
10061 . split ( "\n" )
10162 . map ( ( line ) => line . trim ( ) ) ,
102- [
103- "${{ runner.temp }}/solheim-provider-smoke/verdict.json" ,
104- "${{ runner.temp }}/solheim-provider-smoke/trace.json" ,
105- ] ,
10663 )
107- // The recording artifact carries only the exact validated final path.
108- assert . equal (
109- recordingStep . split ( "path: " ) [ 1 ] ?. split ( "\n" ) [ 0 ] . trim ( ) ,
110- "${{ runner.temp }}/solheim-provider-smoke/smoke.mp4" ,
111- )
112- for ( const step of [ metadataStep , recordingStep ] ) {
113- assert . match ( step , / ! c a n c e l l e d \( \) / )
114- assert . match ( step , / i f - n o - f i l e s - f o u n d : e r r o r / )
115- assert . match ( step , / r e t e n t i o n - d a y s : 7 / )
116- assert . match ( step , / a c t i o n s \/ u p l o a d - a r t i f a c t @ [ 0 - 9 a - f ] { 40 } / )
117- assert . ok ( ! step . includes ( "*" ) )
64+ assert . deepEqual ( paths , [ [ output + "/verdict.json" , output + "/trace.json" ] , [ output + "/smoke.mp4" ] ] )
65+ for ( const step of uploads ) {
66+ assert . equal (
67+ step . if ,
68+ "${{ !cancelled() && (steps.provider.outcome == 'success' || steps.provider.outcome == 'failure') }}" ,
69+ )
70+ assert . equal ( step . with [ "if-no-files-found" ] , "error" )
71+ assert . equal ( step . with [ "retention-days" ] , 7 )
72+ assert . ok ( steps . indexOf ( step ) > steps . findIndex ( ( candidate ) => candidate . id === "provider" ) )
11873 }
119- // A missing recording cannot hide behind a multi-path if-no-files-found.
120- assert . match ( workflow , / n a m e : s o l h e i m - p r o v i d e r - s m o k e - r e c o r d i n g - \$ \{ \{ g i t h u b \. s h a \} \} / )
121- // Temporary recorder output is never an upload path.
122- assert . ok ( ! workflow . includes ( ".tmp" ) )
123- assert . ok ( ! driver . includes ( "console.error(" ) )
124- assert . ok ( ! driver . includes ( "stderrLog" ) )
125- } )
126- it ( "reports a missing or failed recording explicitly before the uploads" , ( ) => {
127- const checkStep = workflow . split ( "- name: Verify recording integrity" ) [ 1 ] ?. split ( "- name:" ) [ 0 ] ?? ""
128- // The published final is the commit marker; no separate status file exists.
129- assert . match ( checkStep , / t e s t - s " \$ S O L H E I M _ S M O K E _ O U T _ D I R \/ s m o k e \. m p 4 " / )
130- assert . ok ( ! workflow . includes ( "recording-status" ) )
131- assert . match ( checkStep , / ! c a n c e l l e d \( \) / )
132- const checkIndex = workflow . indexOf ( "- name: Verify recording integrity" )
133- const uploadIndex = workflow . indexOf ( "- name: Upload provider smoke verdict and trace" )
134- assert . ok ( checkIndex >= 0 && checkIndex < uploadIndex )
135- } )
136- it ( "records a fresh display with credential-free ffmpeg and bounded teardown" , ( ) => {
137- assert . match ( workflow , / i n s t a l l - y x v f b f f m p e g / )
138- assert . match ( workflow , / x v f b - r u n - a - s " - s c r e e n 0 1 2 8 0 x 7 2 0 x 2 4 - n o l i s t e n t c p " / )
139- assert . match ( recorder , / e n v - i " P A T H = \$ P A T H " " D I S P L A Y = \$ D I S P L A Y " / )
140- assert . match ( recorder , / - t 4 8 0 - f s 1 2 5 8 2 9 1 2 0 / )
141- assert . match ( recorder , / f r a g _ k e y f r a m e \+ e m p t y _ m o o v / )
142- assert . match ( recorder , / t r a p f i n i s h E X I T / )
143- assert . match ( recorder , / a t t e m p t < 5 0 / )
144- assert . ok ( ! recorder . includes ( "SOLHEIM_API_KEY" ) )
145- assert . match ( workflow , / r e t e n t i o n - d a y s : 7 / )
146- // The recorder writes a unique temporary file inside the output
147- // directory, forces the mp4 muxer because the name has no mp4 suffix,
148- // and publishes only validated output with an atomic rename.
149- assert . match ( recorder , / t e m p _ r e c o r d i n g = " \$ S O L H E I M _ S M O K E _ O U T _ D I R \/ \. s m o k e \. m p 4 \. \$ \$ \. t m p " / )
150- assert . match ( recorder , / - f m p 4 " \$ t e m p _ r e c o r d i n g " / )
151- assert . match ( recorder , / f f p r o b e - v e r r o r " \$ t e m p _ r e c o r d i n g " / )
152- assert . match ( recorder , / m v - f " \$ t e m p _ r e c o r d i n g " " \$ f i n a l _ r e c o r d i n g " / )
153- assert . match ( recorder , / r m - f " \$ f i n a l _ r e c o r d i n g " / )
154- // ffmpeg must never write the final path directly.
155- assert . ok ( ! / f f m p e g [ ^ \n ] * \$ f i n a l _ r e c o r d i n g / . test ( recorder ) )
156- } )
157- it ( "signals the Node driver directly with a documented stop budget" , ( ) => {
158- // A package-manager wrapper between the recorder and the driver would
159- // receive the TERM and KILL meant for the driver that performs teardown
160- // and the atomic verdict and trace writes.
161- assert . ok ( ! recorder . includes ( "pnpm" ) )
162- assert . match ( recorder , / n o d e - - i m p o r t t s x " \$ s c r i p t _ d i r \/ d r i v e r \. m t s " & / )
163- assert . match ( recorder , / S O L H E I M _ S M O K E _ S T O P _ B U D G E T _ S : - 3 0 / )
164- assert . match ( recorder , / k i l l - T E R M " \$ p r o v i d e r _ p i d " / )
165- assert . match ( recorder , / k i l l - K I L L " \$ p r o v i d e r _ p i d " / )
166- assert . match ( recorder , / w a i t " \$ p r o v i d e r _ p i d " / )
167- assert . ok ( ! recorder . includes ( "status_file" ) )
168- } )
169- it ( "keeps Code QA and package commands focused on smoke contracts" , ( ) => {
170- for ( const command of [
171- "test:solheim-smoke:unit" ,
172- "test:solheim-smoke-ci" ,
173- "solheim-smoke:check-types" ,
174- "test:agent-evals:unit" ,
175- "agent-evals:check-types" ,
176- ] )
177- assert . ok ( codeQa . includes ( command ) )
178- assert . ok ( ! codeQa . includes ( "model-qualification" ) )
179- assert . ok ( ! Object . keys ( pkg . scripts ) . some ( ( name ) => name . includes ( "model-qualification" ) ) )
180- assert . ok ( ! codeQa . includes ( "final-smoke" ) )
181- assert . equal ( pkg . scripts [ "solheim:smoke" ] , "node --import tsx scripts/solheim-smoke/driver.mts" )
182- assert . equal ( pkg . scripts [ "test:agent-evals:unit" ] , "node --import tsx --test scripts/agent-evals/*.test.ts" )
183- assert . equal ( pkg . scripts [ "agent-evals:check-types" ] , "tsc -p scripts/agent-evals/tsconfig.json" )
184- assert . ok ( ! fs . existsSync ( new URL ( "../.github/workflows/final-vscode-review-smoke.yml" , import . meta. url ) ) )
74+ assert . equal ( new Set ( uploads . map ( ( step ) => step . with . name ) ) . size , 2 )
18575 } )
186- it ( "keeps the shared controller free of smoke, provider, and solution knowledge" , ( ) => {
187- assert . equal ( evalsPkg . type , "module" )
188- assert . equal ( evalsPkg . private , true )
189- assert . ok ( ! / s o l h e i m / i. test ( runner ) )
190- assert . ok ( ! / s m o k e / i. test ( runner ) )
191- assert . ok ( ! runner . includes ( "SOLHEIM_API_KEY" ) )
192- assert . ok ( ! / s o l h e i m / i. test ( trace ) )
193- assert . ok ( trace . includes ( "traceSchemaVersion" ) )
194- } )
195- it ( "registers fixtures only through the validated in-memory registry" , ( ) => {
196- assert . ok ( ! registry . includes ( "import(" ) )
197- assert . ok ( ! registry . includes ( "fetch(" ) )
198- assert . ok ( ! registry . includes ( "readdir" ) )
199- assert . ok ( fixture . includes ( "registerFixture" ) )
200- } )
201- it ( "keeps VS Code launch isolation in one shared launcher with an atomic artifact writer" , ( ) => {
202- assert . ok ( driver . includes ( "createHostRuntime" ) )
203- assert . ok ( ! driver . includes ( "spawn(" ) )
204- assert . ok ( ! driver . includes ( "downloadAndUnzipVSCode" ) )
205- assert . ok ( launcher . includes ( "downloadAndUnzipVSCode" ) )
206- assert . ok ( launcher . includes ( "buildChildEnvironment" ) )
207- assert . ok ( launcher . includes ( "await fs.rename(pending" ) )
208- assert . ok ( launcher . includes ( "process.kill(-child.pid" ) )
76+
77+ it ( "pins remote actions to immutable commit revisions" , ( ) => {
78+ for ( const step of steps . filter ( ( step ) => step . uses && ! step . uses . startsWith ( "./" ) ) ) {
79+ const revision = step . uses . split ( "@" ) [ 1 ]
80+ assert . match ( revision ?? "" , / ^ [ 0 - 9 a - f ] { 40 } $ / , step . uses )
81+ }
20982 } )
21083} )
21184
0 commit comments