Skip to content

Commit 00bd944

Browse files
fix(scripts): a proof needs exactly one test in the scenario file
Reading only the first executed test could report a fix as proven while another test in the same file failed. Co-Authored-By: Claude Fable 5.2 <[email protected]>
1 parent c642ff3 commit 00bd944

3 files changed

Lines changed: 18 additions & 2 deletions

File tree

‎docs/operations/scenarios.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ whose schema differs. New threads default to the replay agent's `showcase` recor
4949
message and send it. To play something else, add a recording to the
5050
[replay agent](development.md#replay-provider).
5151

52+
Write one test per file: a proof runs the file and needs exactly one test in it.
53+
5254
Take an `app.screenshot("label")` where the bug shows. The proof pairs base and head captures by
5355
label. e2e adds a capture at the point of failure itself.
5456

‎scripts/prove-scenario.test.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ const passed = report(0);
3535
const failed = report(1, { code: "ASSERTION_FAILED", message: "expected: focused" });
3636
// The control the scenario clicks is missing: the scenario is broken, the bug is not shown.
3737
const brokenScenario = report(1, { code: "LOCATOR_NOT_FOUND", message: "no node" });
38+
// A file with two tests: the first passes, so reading only it would hide the second.
39+
const twoTests = { run: { results: [...passed.run.results, ...failed.run.results] } };
3840
// e2e exits 0 when the test skips itself at runtime.
3941
const skipped = report(0, undefined, "skipped");
4042

@@ -64,6 +66,13 @@ describe("prove-scenario", () => {
6466
passesOnHead: false,
6567
verdict: /could not run on head/,
6668
},
69+
{
70+
base: failed,
71+
head: twoTests,
72+
failsOnBase: true,
73+
passesOnHead: false,
74+
verdict: /could not run on head .*exactly one/,
75+
},
6776
// e2e writes no report when the app does not start.
6877
{
6978
base: undefined,

‎scripts/prove-scenario.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,15 @@ export function readSide(
6666
report: E2EReport | undefined,
6767
): SideResult {
6868
// The report also lists the tests the run did not select, with no attempts.
69-
const result = report?.run.results.find((candidate) => candidate.attempts.length > 0);
69+
const executed = report?.run.results.filter((candidate) => candidate.attempts.length > 0) ?? [];
70+
// One scenario is one test: with several, a pass in the first could hide a failure in another.
71+
const result = executed.length === 1 ? executed[0] : undefined;
7072
const attempt = result?.attempts.at(-1);
7173
if (result === undefined || attempt === undefined) {
72-
const message = "e2e wrote no report: the app did not start, or no test was selected";
74+
const message =
75+
executed.length > 1
76+
? `the scenario file ran ${executed.length} tests; a proof needs exactly one`
77+
: "e2e wrote no report: the app did not start, or no test was selected";
7378
return { sha, status: "error", title: "", message, steps: [], captures: [], videos: [] };
7479
}
7580
const files = (kind: string) =>

0 commit comments

Comments
 (0)