@@ -53,6 +53,11 @@ export interface SideResult {
5353 readonly status : "passed" | "failed" | "error" ;
5454 readonly title : string ;
5555 readonly message ?: string ;
56+ /**
57+ * For `error`, what stopped: `app` when the app did not start or no test ran, `scenario` when
58+ * the app ran and the scenario stopped before its assertion, such as on a control it did not find.
59+ */
60+ readonly stopped ?: "app" | "scenario" ;
5661 readonly steps : ReadonlyArray < { readonly step : string ; readonly status : string } > ;
5762 /** Capture paths under the run's results directory, in the order the scenario took them. */
5863 readonly captures : ReadonlyArray < string > ;
@@ -75,7 +80,16 @@ export function readSide(
7580 executed . length > 1
7681 ? `the scenario file ran ${ executed . length } tests; a proof needs exactly one`
7782 : "e2e wrote no report: the app did not start, or no test was selected" ;
78- return { sha, status : "error" , title : "" , message, steps : [ ] , captures : [ ] , videos : [ ] } ;
83+ return {
84+ sha,
85+ status : "error" ,
86+ stopped : "app" ,
87+ title : "" ,
88+ message,
89+ steps : [ ] ,
90+ captures : [ ] ,
91+ videos : [ ] ,
92+ } ;
7993 }
8094 const files = ( kind : string ) =>
8195 ( attempt . artifacts ?? [ ] ) . flatMap ( ( artifact ) =>
@@ -84,15 +98,18 @@ export function readSide(
8498 : [ ] ,
8599 ) ;
86100 const error = attempt . error ;
101+ // e2e exits 0 for a skipped test too, and a test that did not run proves nothing.
102+ const status =
103+ attempt . status === "passed"
104+ ? "passed"
105+ : error ?. code === "ASSERTION_FAILED"
106+ ? "failed"
107+ : "error" ;
87108 return {
88109 sha,
89- // e2e exits 0 for a skipped test too, and a test that did not run proves nothing.
90- status :
91- attempt . status === "passed"
92- ? "passed"
93- : error ?. code === "ASSERTION_FAILED"
94- ? "failed"
95- : "error" ,
110+ status,
111+ // A test that ended with an error of its own got as far as the app.
112+ ...( status === "error" ? { stopped : error === undefined ? "app" : "scenario" } : { } ) ,
96113 title : result . titlePath . join ( " > " ) ,
97114 ...( error === undefined
98115 ? attempt . status === "passed"
@@ -123,7 +140,7 @@ export function buildProof(base: SideResult, head: SideResult, link: (path: stri
123140 const broken = base . status === "error" ? base : head . status === "error" ? head : undefined ;
124141 const verdict =
125142 broken !== undefined
126- ? `Not proven: the scenario could not run on ${ broken === base ? `base ${ baseSha } ` : `head ${ headSha } ` } (${ broken . message ?? "unknown error" } ).`
143+ ? `Not proven: the scenario could not run on ${ broken === base ? `base ${ baseSha } ` : `head ${ headSha } ` } (${ broken . message ?? "unknown error" } ).${ broken . stopped === "scenario" ? " The app ran and the scenario stopped before its assertion: fix the scenario and run it again." : "" } `
127144 : base . status === "passed"
128145 ? head . status === "passed"
129146 ? `Not proven: the scenario passes on base ${ baseSha } too, so it does not reproduce the bug.`
@@ -160,6 +177,8 @@ export function buildProof(base: SideResult, head: SideResult, link: (path: stri
160177 scenario : head . title || base . title ,
161178 failsOnBase : base . status === "failed" ,
162179 passesOnHead : head . status === "passed" ,
180+ /** Why there is no verdict on the behavior: the app did not run, or the scenario stopped. */
181+ couldNotRun : broken ?. stopped ?? null ,
163182 baseSha : base . sha ,
164183 headSha : head . sha ,
165184 verdict,
@@ -192,10 +211,18 @@ const withCheckout = Effect.fn("prove.withCheckout")(function* <A, E, R>(
192211 if ( clean && ( yield * git ( [ "rev-parse" , "HEAD" ] ) ) === sha ) return yield * use ( repo ) ;
193212 // Beside this checkout, not in the temp directory, so tooling resolves as it does here.
194213 const worktree = `${ repo } -scenario-${ short ( sha ) } ` ;
214+ // A run that was killed left its worktree behind, and git refuses to add over it.
215+ const discard = git ( [ "worktree" , "remove" , "--force" , worktree ] ) . pipe (
216+ Effect . ignore ,
217+ Effect . andThen ( git ( [ "worktree" , "prune" ] ) ) ,
218+ Effect . ignore ,
219+ ) ;
195220 return yield * Effect . acquireUseRelease (
196- git ( [ "worktree" , "add" , "--quiet" , "--detach" , "--force" , worktree , sha ] ) ,
221+ discard . pipe (
222+ Effect . andThen ( git ( [ "worktree" , "add" , "--quiet" , "--detach" , "--force" , worktree , sha ] ) ) ,
223+ ) ,
197224 ( ) => run ( "vp" , [ "i" ] , { cwd : worktree , echo : true } ) . pipe ( Effect . andThen ( use ( worktree ) ) ) ,
198- ( ) => git ( [ "worktree" , "remove" , "--force" , worktree ] ) . pipe ( Effect . ignore ) ,
225+ ( ) => discard ,
199226 ) ;
200227} ) ;
201228
@@ -204,6 +231,7 @@ const runSide = Effect.fn("prove.runSide")(function* (
204231 side : "base" | "head" ,
205232 rev : string ,
206233 scenarioPath : string ,
234+ target : string ,
207235) {
208236 const fs = yield * FileSystem . FileSystem ;
209237 const path = yield * Path . Path ;
@@ -216,7 +244,7 @@ const runSide = Effect.fn("prove.runSide")(function* (
216244 // The scenario and e2e come from this checkout; only the app comes from `appDir`.
217245 run (
218246 path . join ( scripts , "node_modules/.bin/e2e" ) ,
219- [ "run" , scenarioPath , "--output" , path . relative ( scripts , outDir ) ] ,
247+ [ "run" , scenarioPath , "--target" , target , "-- output", path . relative ( scripts , outDir ) ] ,
220248 {
221249 cwd : scripts ,
222250 env : { ...environment , T3_SCENARIO_APP_DIR : appDir , E2E_TELEMETRY_DISABLED : "1" } ,
@@ -230,7 +258,22 @@ const runSide = Effect.fn("prove.runSide")(function* (
230258 Effect . flatMap ( decodeReport ) ,
231259 Effect . orElseSucceed ( ( ) => undefined ) ,
232260 ) ;
233- return readSide ( sha , path . join ( outDir , "results" ) , report ) ;
261+ const result = readSide ( sha , path . join ( outDir , "results" ) , report ) ;
262+ // Written as each side ends, so a proof whose second side is cut short still has its first.
263+ yield * fs
264+ . writeFileString (
265+ path . join ( scripts , ".e2e/prove" , `${ side } .json` ) ,
266+ `${ yield * encodeProof ( result ) } \n` ,
267+ )
268+ . pipe ( Effect . ignore ) ;
269+ return result ;
270+ } ) ;
271+
272+ /** The one target a scenario runs on: an iOS scenario imports `t3-mobile.ts`, a web one does not. */
273+ const targetOf = Effect . fn ( "prove.targetOf" ) ( function * ( scenarioPath : string ) {
274+ const fs = yield * FileSystem . FileSystem ;
275+ const source = yield * fs . readFileString ( scenarioPath ) . pipe ( Effect . orElseSucceed ( ( ) => "" ) ) ;
276+ return source . includes ( "t3-mobile" ) ? "ios" : "web" ;
234277} ) ;
235278
236279/** Uploads the captures and maps each to its URL; a failed upload leaves the map empty. */
@@ -262,6 +305,7 @@ const prove = Effect.fn("prove")(function* (input: {
262305 readonly base : string ;
263306 readonly head : string ;
264307 readonly uploadCommand : Option . Option < string > ;
308+ readonly target : Option . Option < string > ;
265309} ) {
266310 const fs = yield * FileSystem . FileSystem ;
267311 const path = yield * Path . Path ;
@@ -271,8 +315,10 @@ const prove = Effect.fn("prove")(function* (input: {
271315 // A proof left by an earlier run must not be read as this one's.
272316 yield * fs . remove ( proveDir , { recursive : true , force : true } ) ;
273317 yield * fs . makeDirectory ( proveDir , { recursive : true } ) ;
274- const base = yield * runSide ( "base" , input . base , scenarioPath ) ;
275- const head = yield * runSide ( "head" , input . head , scenarioPath ) ;
318+ // Only that target's app starts: a web scenario needs no Metro and no device.
319+ const target = Option . getOrElse ( input . target , ( ) => "" ) || ( yield * targetOf ( scenarioPath ) ) ;
320+ const base = yield * runSide ( "base" , input . base , scenarioPath , target ) ;
321+ const head = yield * runSide ( "head" , input . head , scenarioPath , target ) ;
276322 const links = Option . isSome ( input . uploadCommand )
277323 ? yield * upload ( input . uploadCommand . value , [ ...base . captures , ...head . captures ] )
278324 : new Map < string , string > ( ) ;
@@ -285,8 +331,13 @@ const prove = Effect.fn("prove")(function* (input: {
285331 yield * fs . writeFileString ( path . join ( proveDir , "proof.md" ) , proof . markdown ) ;
286332 yield * fs . writeFileString ( path . join ( proveDir , "proof.json" ) , `${ json } \n` ) ;
287333 yield * Console . log ( json ) ;
288- const couldNotRun = base . status === "error" || head . status === "error" ;
289- return proof . failsOnBase && proof . passesOnHead ? 0 : couldNotRun ? 2 : 1 ;
334+ return proof . failsOnBase && proof . passesOnHead
335+ ? 0
336+ : proof . couldNotRun === "scenario"
337+ ? 3
338+ : proof . couldNotRun === "app"
339+ ? 2
340+ : 1 ;
290341} ) ;
291342
292343const command = Command . make (
@@ -304,6 +355,12 @@ const command = Command.make(
304355 Flag . withDescription ( "Command that takes capture paths and prints one URL per line." ) ,
305356 Flag . optional ,
306357 ) ,
358+ target : Flag . String ( "target" ) . pipe (
359+ Flag . withDescription (
360+ "Target to run on: web or ios. Default: ios when the scenario imports t3-mobile.ts, else web." ,
361+ ) ,
362+ Flag . optional ,
363+ ) ,
307364 } ,
308365 ( input ) =>
309366 prove ( input ) . pipe (
0 commit comments