fix(qa): gate character eval on suite summary

vincentkoc · vincentkoc · commit 919befbbb63c · 2026-06-07T03:04:53.000+02:00
diff --git a/extensions/qa-lab/src/character-eval.test.ts b/extensions/qa-lab/src/character-eval.test.ts
@@ -90,17 +90,51 @@ function createConcurrencyGate(expectedActive: number) {
   };
 }
 
-function makeSuiteResult(params: { outputDir: string; model: string; transcript: string }) {
+async function makeSuiteResult(params: {
+  outputDir: string;
+  model: string;
+  transcript: string;
+  resultStatus?: "pass" | "fail";
+  summaryStatus?: "pass" | "fail";
+  summaryFailedCount?: number;
+}) {
+  const resultStatus = params.resultStatus ?? "pass";
+  const summaryStatus = params.summaryStatus ?? resultStatus;
+  const summaryFailedCount = params.summaryFailedCount ?? (summaryStatus === "fail" ? 1 : 0);
+  const summaryPath = path.join(params.outputDir, "qa-suite-summary.json");
+  await fs.mkdir(params.outputDir, { recursive: true });
+  await fs.writeFile(
+    summaryPath,
+    `${JSON.stringify(
+      {
+        counts: {
+          total: 1,
+          passed: summaryFailedCount > 0 ? 0 : 1,
+          failed: summaryFailedCount,
+        },
+        scenarios: [
+          {
+            name: "Character vibes",
+            status: summaryStatus,
+            steps: [],
+          },
+        ],
+      },
+      null,
+      2,
+    )}\n`,
+    "utf8",
+  );
   return {
     outputDir: params.outputDir,
     reportPath: path.join(params.outputDir, "qa-suite-report.md"),
-    summaryPath: path.join(params.outputDir, "qa-suite-summary.json"),
+    summaryPath,
     report: "# report",
     watchUrl: "http://127.0.0.1:43124",
     scenarios: [
       {
         name: "Character vibes",
-        status: "pass",
+        status: resultStatus,
         steps: [
           {
             name: `transcript for ${params.model}`,
@@ -430,6 +464,33 @@ describe("runQaCharacterEval", () => {
     });
   });
 
+  it("marks candidates failed when the suite summary has failed scenarios", async () => {
+    const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
+      makeSuiteResult({
+        outputDir: params.outputDir,
+        model: params.primaryModel,
+        transcript: "USER Alice: hi\n\nASSISTANT openclaw: outwardly fine",
+        summaryStatus: "fail",
+        summaryFailedCount: 1,
+      }),
+    );
+    const runJudge = makeRunJudge([
+      { model: "openai/gpt-5.5", rank: 1, score: 0.5, summary: "failed" },
+    ]);
+
+    const result = await runQaCharacterEval({
+      repoRoot: tempRoot,
+      outputDir: path.join(tempRoot, "character"),
+      models: ["openai/gpt-5.5"],
+      judgeModels: ["openai/gpt-5.5"],
+      runSuite,
+      runJudge,
+    });
+
+    expect(result.runs[0]?.status).toBe("fail");
+    expect(result.runs[0]?.error).toBeUndefined();
+  });
+
   it("marks raw tool failure transcripts as failed output", async () => {
     const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
       makeSuiteResult({
diff --git a/extensions/qa-lab/src/character-eval.ts b/extensions/qa-lab/src/character-eval.ts
@@ -3,7 +3,6 @@ import fs from "node:fs/promises";
 import path from "node:path";
 import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
 import { normalizeStringEntries, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
-import { runQaManualLane } from "./manual-lane.runtime.js";
 import { isQaFastModeModelRef, type QaProviderMode } from "./model-selection.js";
 import {
   QA_FRONTIER_CHARACTER_EVAL_MODELS,
@@ -13,7 +12,7 @@ import {
 } from "./providers/live-frontier/character-eval.js";
 import type { QaThinkingLevel } from "./qa-gateway-config.js";
 import { extractQaVisibleReplyLeakText } from "./reply-failure.js";
-import { runQaSuiteFromRuntime } from "./suite-launch.runtime.js";
+import { readQaSuiteFailedScenarioCountFromFile } from "./suite-summary.js";
 import type { QaSuiteResult } from "./suite.js";
 
 const DEFAULT_CHARACTER_SCENARIO_ID = "character-vibes-gollum";
@@ -412,6 +411,7 @@ async function defaultRunJudge(params: {
   prompt: string;
   timeoutMs: number;
 }) {
+  const { runQaManualLane } = await import("./manual-lane.runtime.js");
   const result = await runQaManualLane({
     repoRoot: params.repoRoot,
     providerMode: "live-frontier",
@@ -425,6 +425,11 @@ async function defaultRunJudge(params: {
   return result.reply;
 }
 
+async function defaultRunSuite(params: Parameters<RunSuiteFn>[0]) {
+  const { runQaSuiteFromRuntime } = await import("./suite-launch.runtime.js");
+  return await runQaSuiteFromRuntime(params);
+}
+
 function renderCharacterEvalReport(params: {
   scenarioId: string;
   startedAt: Date;
@@ -519,7 +524,7 @@ export async function runQaCharacterEval(params: QaCharacterEvalParams) {
   const runsDir = path.join(outputDir, "runs");
   await fs.mkdir(runsDir, { recursive: true });
 
-  const runSuite = params.runSuite ?? runQaSuiteFromRuntime;
+  const runSuite = params.runSuite ?? defaultRunSuite;
   const candidateConcurrency = normalizeConcurrency(
     params.candidateConcurrency,
     DEFAULT_CHARACTER_EVAL_CONCURRENCY,
@@ -560,10 +565,8 @@ export async function runQaCharacterEval(params: QaCharacterEvalParams) {
       });
       const transcript = extractTranscript(result);
       const transcriptFailure = detectTranscriptFailure(transcript);
-      const status =
-        result.scenarios.some((scenario) => scenario.status === "fail") || transcriptFailure
-          ? "fail"
-          : "pass";
+      const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile(result.summaryPath);
+      const status = failedScenarioCount > 0 || transcriptFailure ? "fail" : "pass";
       const run = {
         model,
         status,
