fix(qa): fail suite on summary scenario failures

vincentkoc · vincentkoc · commit 6d2566682a3f · 2026-06-07T02:49:02.000+02:00
diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts
@@ -503,13 +503,7 @@ describe("qa cli runtime", () => {
       watchUrl: "http://127.0.0.1:43124",
       reportPath: suiteReportPath,
       summaryPath: suiteSummaryPath,
-      scenarios: [
-        {
-          name: "channel chat baseline",
-          status: "fail",
-          steps: [],
-        },
-      ],
+      scenarios: [],
     });
 
     try {
diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts
@@ -706,7 +706,7 @@ export async function runQaSuiteCommand(opts: {
   process.stdout.write(`QA suite watch: ${result.watchUrl}\n`);
   process.stdout.write(`QA suite report: ${result.reportPath}\n`);
   process.stdout.write(`QA suite summary: ${result.summaryPath}\n`);
-  const failedScenarioCount = readQaSuiteFailedScenarioCountFromSummary(result);
+  const failedScenarioCount = await readQaFailedScenarioCountFromSummary(result.summaryPath);
   if (!allowFailures && failedScenarioCount !== null && failedScenarioCount > 0) {
     process.exitCode = 1;
   }
diff --git a/extensions/qa-lab/src/suite-summary.test.ts b/extensions/qa-lab/src/suite-summary.test.ts
@@ -12,7 +12,14 @@ describe("qa suite summary helpers", () => {
     ).toBe(2);
   });
 
-  it("prefers counts.failed when available", () => {
+  it("uses the larger failure signal when counts and scenarios disagree", () => {
+    expect(
+      readQaSuiteFailedScenarioCountFromSummary({
+        counts: { failed: 0 },
+        scenarios: [{ status: "pass" }, { status: "fail" }],
+      }),
+    ).toBe(1);
+
     expect(
       readQaSuiteFailedScenarioCountFromSummary({
         counts: { failed: 3.8 },
diff --git a/extensions/qa-lab/src/suite-summary.ts b/extensions/qa-lab/src/suite-summary.ts
@@ -79,11 +79,18 @@ export function readQaSuiteFailedScenarioCountFromSummary(summary: unknown): num
     };
     scenarios?: Array<QaSuiteScenarioStatus>;
   };
-  if (typeof payload.counts?.failed === "number" && Number.isFinite(payload.counts.failed)) {
-    return Math.max(0, Math.floor(payload.counts.failed));
+  const countedFailures =
+    typeof payload.counts?.failed === "number" && Number.isFinite(payload.counts.failed)
+      ? Math.max(0, Math.floor(payload.counts.failed))
+      : null;
+  const scenarioFailures = Array.isArray(payload.scenarios)
+    ? countQaSuiteFailedScenarios(payload.scenarios)
+    : null;
+  if (countedFailures !== null && scenarioFailures !== null) {
+    return Math.max(countedFailures, scenarioFailures);
   }
-  if (Array.isArray(payload.scenarios)) {
-    return countQaSuiteFailedScenarios(payload.scenarios);
+  if (scenarioFailures !== null) {
+    return scenarioFailures;
   }
-  return null;
+  return countedFailures;
 }

Original file line number	Diff line number	Diff line change
`@@ -706,7 +706,7 @@ export async function runQaSuiteCommand(opts: {`
`706`	`706`	process.stdout.write(`QA suite watch: ${result.watchUrl}\n`);
`707`	`707`	process.stdout.write(`QA suite report: ${result.reportPath}\n`);
`708`	`708`	process.stdout.write(`QA suite summary: ${result.summaryPath}\n`);
`709`		`- const failedScenarioCount = readQaSuiteFailedScenarioCountFromSummary(result);`
	`709`	`+ const failedScenarioCount = await readQaFailedScenarioCountFromSummary(result.summaryPath);`
`710`	`710`	`if (!allowFailures && failedScenarioCount !== null && failedScenarioCount > 0) {`
`711`	`711`	`process.exitCode = 1;`
`712`	`712`	`}`