Skip to content

Commit 3dc6ac3

Browse files
vincentkocvincentkoc
committed
fix(qa): fail closed on skipped suite summaries
1 parent ce015ce commit 3dc6ac3

6 files changed

Lines changed: 200 additions & 13 deletions

File tree

‎extensions/qa-lab/src/cli.runtime.test.ts‎

Lines changed: 83 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,14 @@ describe("qa cli runtime", () => {
450450
it("keeps telegram exit code clear when --allow-failures is set", async () => {
451451
const priorExitCode = process.exitCode;
452452
process.exitCode = undefined;
453+
await fs.writeFile(
454+
telegramSummaryPath,
455+
JSON.stringify({
456+
counts: { total: 1, passed: 0, failed: 1 },
457+
scenarios: [{ status: "fail" }],
458+
}),
459+
"utf8",
460+
);
453461
runTelegramQaLive.mockResolvedValueOnce({
454462
outputDir: telegramArtifactsDir,
455463
reportPath: path.join(telegramArtifactsDir, "report.md"),
@@ -534,6 +542,39 @@ describe("qa cli runtime", () => {
534542
}
535543
});
536544

545+
it("sets a failing exit code when host suite scenarios are skipped", async () => {
546+
const priorExitCode = process.exitCode;
547+
process.exitCode = undefined;
548+
await fs.writeFile(
549+
suiteSummaryPath,
550+
JSON.stringify({
551+
counts: {
552+
total: 1,
553+
passed: 0,
554+
failed: 0,
555+
skipped: 1,
556+
},
557+
scenarios: [{ name: "channel chat baseline", status: "skip" }],
558+
}),
559+
"utf8",
560+
);
561+
runQaSuiteFromRuntime.mockResolvedValueOnce({
562+
watchUrl: "http://127.0.0.1:43124",
563+
reportPath: suiteReportPath,
564+
summaryPath: suiteSummaryPath,
565+
scenarios: [],
566+
});
567+
568+
try {
569+
await runQaSuiteCommand({
570+
repoRoot: "/tmp/openclaw-repo",
571+
});
572+
expect(process.exitCode).toBe(1);
573+
} finally {
574+
process.exitCode = priorExitCode;
575+
}
576+
});
577+
537578
it("keeps host suite exit code clear when --allow-failures is set", async () => {
538579
const priorExitCode = process.exitCode;
539580
process.exitCode = undefined;
@@ -720,7 +761,7 @@ describe("qa cli runtime", () => {
720761
repoRoot: "/tmp/openclaw-repo",
721762
preflight: true,
722763
}),
723-
).rejects.toThrow("QA parity preflight failed with 1 failing scenario.");
764+
).rejects.toThrow("QA parity preflight failed with 1 failing or skipped scenario.");
724765
});
725766

726767
it("keeps parity preflight exit code clear when --allow-failures is set", async () => {
@@ -1503,6 +1544,46 @@ describe("qa cli runtime", () => {
15031544
}
15041545
});
15051546

1547+
it("sets a failing exit code when multipass summary reports skipped scenarios", async () => {
1548+
const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "qa-multipass-summary-"));
1549+
const summaryPath = path.join(repoRoot, "qa-suite-summary.json");
1550+
await fs.writeFile(
1551+
summaryPath,
1552+
JSON.stringify({
1553+
counts: {
1554+
total: 2,
1555+
passed: 1,
1556+
failed: 0,
1557+
skipped: 1,
1558+
},
1559+
}),
1560+
"utf8",
1561+
);
1562+
runQaMultipass.mockResolvedValueOnce({
1563+
outputDir: repoRoot,
1564+
reportPath: path.join(repoRoot, "qa-suite-report.md"),
1565+
summaryPath,
1566+
hostLogPath: path.join(repoRoot, "multipass-host.log"),
1567+
bootstrapLogPath: path.join(repoRoot, "multipass-guest-bootstrap.log"),
1568+
guestScriptPath: path.join(repoRoot, "multipass-guest-run.sh"),
1569+
vmName: "openclaw-qa-test",
1570+
scenarioIds: ["channel-chat-baseline"],
1571+
});
1572+
const priorExitCode = process.exitCode;
1573+
process.exitCode = undefined;
1574+
1575+
try {
1576+
await runQaSuiteCommand({
1577+
repoRoot: "/tmp/openclaw-repo",
1578+
runner: "multipass",
1579+
});
1580+
expect(process.exitCode).toBe(1);
1581+
} finally {
1582+
process.exitCode = priorExitCode;
1583+
await fs.rm(repoRoot, { recursive: true, force: true });
1584+
}
1585+
});
1586+
15061587
it("rejects malformed multipass summary JSON", async () => {
15071588
const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "qa-multipass-summary-"));
15081589
const summaryPath = path.join(repoRoot, "qa-suite-summary.json");
@@ -1577,7 +1658,7 @@ describe("qa cli runtime", () => {
15771658
repoRoot: "/tmp/openclaw-repo",
15781659
runner: "multipass",
15791660
}),
1580-
).rejects.toThrow("did not include counts.failed or scenarios[].status");
1661+
).rejects.toThrow("did not include counts.failed, counts.skipped, or scenarios[].status");
15811662
} finally {
15821663
await fs.rm(repoRoot, { recursive: true, force: true });
15831664
}

‎extensions/qa-lab/src/cli.runtime.ts‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ import {
6868
} from "./scenario-catalog.js";
6969
import { resolveQaScenarioPackScenarioIds } from "./scenario-packs.js";
7070
import { runQaSuiteFromRuntime } from "./suite-launch.runtime.js";
71-
import { readQaSuiteFailedScenarioCountFromFile } from "./suite-summary.js";
71+
import { readQaSuiteFailedOrSkippedScenarioCountFromFile } from "./suite-summary.js";
7272
import {
7373
buildTokenEfficiencyReport,
7474
renderTokenEfficiencyMarkdownReport,
@@ -257,7 +257,7 @@ function isQaSuiteInfraRetryableError(error: unknown) {
257257
message.includes("socket hang up") ||
258258
message.includes("could not read qa summary json") ||
259259
message.includes("could not parse qa summary json") ||
260-
message.includes("did not include counts.failed or scenarios[].status") ||
260+
message.includes("did not include counts.failed, counts.skipped, or scenarios[].status") ||
261261
message.includes("did not produce report artifact")
262262
);
263263
}
@@ -271,7 +271,7 @@ async function assertQaSuiteArtifacts(result: { reportPath: string; summaryPath:
271271
{ cause: error },
272272
);
273273
}
274-
await readQaSuiteFailedScenarioCountFromFile(result.summaryPath);
274+
await readQaSuiteFailedOrSkippedScenarioCountFromFile(result.summaryPath);
275275
}
276276

277277
async function runQaSuiteFromRuntimeWithInfraRetry(
@@ -324,13 +324,15 @@ async function runQaParityPreflight(params: {
324324
process.stdout.write(`QA parity preflight watch: ${result.watchUrl}\n`);
325325
process.stdout.write(`QA parity preflight report: ${result.reportPath}\n`);
326326
process.stdout.write(`QA parity preflight summary: ${result.summaryPath}\n`);
327-
const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile(result.summaryPath);
328-
if (failedScenarioCount > 0) {
327+
const blockingScenarioCount = await readQaSuiteFailedOrSkippedScenarioCountFromFile(
328+
result.summaryPath,
329+
);
330+
if (blockingScenarioCount > 0) {
329331
if (params.allowFailures === true) {
330332
return;
331333
}
332334
throw new Error(
333-
`QA parity preflight failed with ${failedScenarioCount} failing scenario${failedScenarioCount === 1 ? "" : "s"}.`,
335+
`QA parity preflight failed with ${blockingScenarioCount} failing or skipped scenario${blockingScenarioCount === 1 ? "" : "s"}.`,
334336
);
335337
}
336338
}
@@ -639,8 +641,10 @@ export async function runQaSuiteCommand(opts: {
639641
process.stdout.write(`QA Multipass host log: ${result.hostLogPath}\n`);
640642
process.stdout.write(`QA Multipass bootstrap log: ${result.bootstrapLogPath}\n`);
641643
if (!allowFailures) {
642-
const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile(result.summaryPath);
643-
if (failedScenarioCount > 0) {
644+
const blockingScenarioCount = await readQaSuiteFailedOrSkippedScenarioCountFromFile(
645+
result.summaryPath,
646+
);
647+
if (blockingScenarioCount > 0) {
644648
process.exitCode = 1;
645649
}
646650
}
@@ -678,8 +682,10 @@ export async function runQaSuiteCommand(opts: {
678682
process.stdout.write(`QA suite watch: ${result.watchUrl}\n`);
679683
process.stdout.write(`QA suite report: ${result.reportPath}\n`);
680684
process.stdout.write(`QA suite summary: ${result.summaryPath}\n`);
681-
const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile(result.summaryPath);
682-
if (!allowFailures && failedScenarioCount > 0) {
685+
const blockingScenarioCount = await readQaSuiteFailedOrSkippedScenarioCountFromFile(
686+
result.summaryPath,
687+
);
688+
if (!allowFailures && blockingScenarioCount > 0) {
683689
process.exitCode = 1;
684690
}
685691
}

‎extensions/qa-lab/src/confidence-report.test.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,54 @@ describe("qa confidence report", () => {
281281
}
282282
});
283283

284+
it("does not pass suite summaries with unsupported non-pass statuses", async () => {
285+
for (const [artifact, expectedDetail] of [
286+
[
287+
{
288+
counts: { total: 1, passed: 1, failed: 0, skipped: 0 },
289+
scenarios: [{ name: "errored", status: "error" }],
290+
},
291+
"unsupported non-pass status",
292+
],
293+
[
294+
{
295+
scenarios: [{ name: "timed out", status: "timeout" }],
296+
},
297+
"unsupported non-pass status",
298+
],
299+
] as const) {
300+
await writeJson("report-only/qa-suite-summary.json", artifact);
301+
302+
const report = await buildQaConfidenceReport({
303+
manifest: {
304+
version: 1,
305+
profile: "codex-100",
306+
lanes: [
307+
{
308+
id: "report-only",
309+
title: "Report-only",
310+
kind: "qa-suite-summary",
311+
artifact: "report-only/qa-suite-summary.json",
312+
required: true,
313+
},
314+
],
315+
},
316+
artifactRoot: tempRoot,
317+
strictZeroUnknowns: true,
318+
strictGlobalPass: true,
319+
generatedAt: "2026-05-12T00:00:00.000Z",
320+
});
321+
322+
expect(report.pass).toBe(false);
323+
expect(report.globalPass).toBe(false);
324+
expect(report.zeroUnknowns).toBe(false);
325+
expect(report.lanes[0]).toMatchObject({
326+
status: "unknown",
327+
});
328+
expect(report.lanes[0]?.details).toContain(expectedDetail);
329+
}
330+
});
331+
284332
it("rejects skipped token reports when a live usage source is required", async () => {
285333
await writeJson("live-token/qa-runtime-token-efficiency-summary.json", {
286334
status: "skipped",

‎extensions/qa-lab/src/confidence-report.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -383,6 +383,15 @@ function evaluateQaSuiteSummary(payload: unknown): QaConfidenceLaneEvaluation {
383383
(scenario) =>
384384
isRecord(scenario) && (scenario.status === "skip" || scenario.status === "skipped"),
385385
).length ?? 0;
386+
const unknownBlockingScenarioCount =
387+
scenarios?.filter(
388+
(scenario) =>
389+
!isRecord(scenario) ||
390+
(scenario.status !== "pass" &&
391+
scenario.status !== "fail" &&
392+
scenario.status !== "skip" &&
393+
scenario.status !== "skipped"),
394+
).length ?? 0;
386395
const hasScenarioRows = scenarios !== undefined && scenarios.length > 0;
387396
const gatewayLogSentinels = collectGatewayLogSentinels(payload);
388397
if (gatewayLogSentinels.length > 0) {
@@ -430,6 +439,13 @@ function evaluateQaSuiteSummary(payload: unknown): QaConfidenceLaneEvaluation {
430439
)}, failed scenarios=${failedScenarios.length}`,
431440
};
432441
}
442+
if (unknownBlockingScenarioCount > 0) {
443+
return {
444+
passed: false,
445+
status: "unknown",
446+
details: `qa-suite-summary has ${unknownBlockingScenarioCount} scenario row(s) with unsupported non-pass status`,
447+
};
448+
}
433449
const explicitSkippedCount = readNumber(counts?.skipped);
434450
const inferredSkippedCount =
435451
totalCount === undefined || passedCount === undefined
@@ -470,6 +486,21 @@ function evaluateQaSuiteSummary(payload: unknown): QaConfidenceLaneEvaluation {
470486
const fallbackFailedScenarios = payload.scenarios.filter(
471487
(scenario) => isRecord(scenario) && scenario.status === "fail",
472488
);
489+
const fallbackUnknownBlockingScenarios = payload.scenarios.filter(
490+
(scenario) =>
491+
!isRecord(scenario) ||
492+
(scenario.status !== "pass" &&
493+
scenario.status !== "fail" &&
494+
scenario.status !== "skip" &&
495+
scenario.status !== "skipped"),
496+
);
497+
if (fallbackUnknownBlockingScenarios.length > 0) {
498+
return {
499+
passed: false,
500+
status: "unknown",
501+
details: `qa-suite-summary has ${fallbackUnknownBlockingScenarios.length} scenario row(s) with unsupported non-pass status`,
502+
};
503+
}
473504
return {
474505
passed: fallbackFailedScenarios.length === 0,
475506
details: `qa-suite-summary failed scenarios=${fallbackFailedScenarios.length}`,

‎extensions/qa-lab/src/suite-summary.test.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,23 @@ describe("qa suite summary helpers", () => {
3030
).toBe(3);
3131
});
3232

33+
it("counts unknown scenario statuses as blocking for strict gates", () => {
34+
expect(
35+
countQaSuiteFailedOrSkippedScenarios([
36+
{ status: "pass" },
37+
{ status: "timeout" as never },
38+
{ status: "error" as never },
39+
]),
40+
).toBe(2);
41+
42+
expect(
43+
readQaSuiteFailedOrSkippedScenarioCountFromSummary({
44+
counts: { failed: 0, skipped: 0 },
45+
scenarios: [{ status: "timeout" }, { status: "error" }],
46+
}),
47+
).toBe(2);
48+
});
49+
3350
it("uses the larger failure signal when counts and scenarios disagree", () => {
3451
expect(
3552
readQaSuiteFailedScenarioCountFromSummary({

‎extensions/qa-lab/src/suite-summary.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,10 @@ function readNonNegativeCount(value: unknown): number | null {
6565
: null;
6666
}
6767

68+
function isQaSuiteBlockingStatus(status: unknown): boolean {
69+
return status !== "pass";
70+
}
71+
6872
export function countQaSuiteFailedScenarios(
6973
scenarios: ReadonlyArray<QaSuiteScenarioStatus>,
7074
): number {
@@ -82,7 +86,7 @@ export function countQaSuiteFailedOrSkippedScenarios(
8286
): number {
8387
let blocking = 0;
8488
for (const scenario of scenarios) {
85-
if (scenario.status === "fail" || scenario.status === "skip" || scenario.status === "skipped") {
89+
if (isQaSuiteBlockingStatus(scenario.status)) {
8690
blocking += 1;
8791
}
8892
}

0 commit comments

Comments
 (0)