Skip to content

Commit 79eb0b9

Browse files
RomneyDaRomneyDa
committed
refactor(qa): export mixed scenario suite runner
1 parent a90d353 commit 79eb0b9

5 files changed

Lines changed: 44 additions & 45 deletions

File tree

‎extensions/qa-lab/api.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,3 +108,4 @@ export {
108108
type QaSuiteSummaryJsonParams,
109109
runQaFlowSuite,
110110
} from "./src/suite.js";
111+
export { runQaSuite, type QaSuiteRuntimeResult } from "./src/suite-launch.runtime.js";

‎extensions/qa-lab/src/cli.runtime.test.ts‎

Lines changed: 34 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
77
const {
88
runQaManualLane,
99
runQaFlowSuiteFromRuntime,
10-
runQaSuiteFromRuntime,
10+
runQaSuite,
1111
runQaCharacterEval,
1212
runQaMultipass,
1313
listTelegramQaScenarioCatalog,
@@ -20,7 +20,7 @@ const {
2020
} = vi.hoisted(() => ({
2121
runQaManualLane: vi.fn(),
2222
runQaFlowSuiteFromRuntime: vi.fn(),
23-
runQaSuiteFromRuntime: vi.fn(),
23+
runQaSuite: vi.fn(),
2424
runQaCharacterEval: vi.fn(),
2525
runQaMultipass: vi.fn(),
2626
listTelegramQaScenarioCatalog: vi.fn(),
@@ -39,7 +39,7 @@ vi.mock("./manual-lane.runtime.js", () => ({
3939

4040
vi.mock("./suite-launch.runtime.js", () => ({
4141
runQaFlowSuiteFromRuntime,
42-
runQaSuiteFromRuntime,
42+
runQaSuite,
4343
}));
4444

4545
vi.mock("./character-eval.js", () => ({
@@ -204,7 +204,7 @@ describe("qa cli runtime", () => {
204204
stdoutWrite = vi.spyOn(process.stdout, "write").mockReturnValue(true);
205205
stderrWrite = vi.spyOn(process.stderr, "write").mockReturnValue(true);
206206
runQaFlowSuiteFromRuntime.mockReset();
207-
runQaSuiteFromRuntime.mockReset();
207+
runQaSuite.mockReset();
208208
runQaCharacterEval.mockReset();
209209
runQaManualLane.mockReset();
210210
runQaMultipass.mockReset();
@@ -218,7 +218,7 @@ describe("qa cli runtime", () => {
218218
(mode: string, options?: { alternate?: boolean }) =>
219219
defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
220220
);
221-
runQaSuiteFromRuntime.mockResolvedValue(
221+
runQaSuite.mockResolvedValue(
222222
flowSuiteRuntimeResult({
223223
reportPath: suiteReportPath,
224224
summaryPath: suiteSummaryPath,
@@ -300,7 +300,7 @@ describe("qa cli runtime", () => {
300300
it("runs selected Playwright scenarios through the suite command", async () => {
301301
const evidencePath = path.join(suiteArtifactsDir, "qa-evidence.json");
302302
await fs.writeFile(evidencePath, JSON.stringify({ entries: [] }), "utf8");
303-
runQaSuiteFromRuntime.mockResolvedValueOnce(
303+
runQaSuite.mockResolvedValueOnce(
304304
testFileSuiteRuntimeResult({
305305
outputDir: suiteArtifactsDir,
306306
reportPath: suiteReportPath,
@@ -315,7 +315,7 @@ describe("qa cli runtime", () => {
315315
scenarioIds: ["control-ui-chat-flow-playwright"],
316316
});
317317

318-
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
318+
expect(runQaSuite).toHaveBeenCalledWith({
319319
repoRoot: process.cwd(),
320320
outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "scenario-test"),
321321
transportId: "qa-channel",
@@ -336,7 +336,7 @@ describe("qa cli runtime", () => {
336336
}),
337337
).rejects.toThrow("--image, --cpus, --memory, and --disk require --runner multipass");
338338

339-
expect(runQaSuiteFromRuntime).not.toHaveBeenCalled();
339+
expect(runQaSuite).not.toHaveBeenCalled();
340340
});
341341

342342
it("resolves suite repo-root-relative paths before dispatching", async () => {
@@ -351,7 +351,7 @@ describe("qa cli runtime", () => {
351351
scenarioIds: ["approval-turn-tool-followthrough"],
352352
});
353353

354-
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
354+
expect(runQaSuite).toHaveBeenCalledWith({
355355
repoRoot: path.resolve("/tmp/openclaw-repo"),
356356
outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/frontier"),
357357
transportId: "qa-channel",
@@ -372,7 +372,7 @@ describe("qa cli runtime", () => {
372372
enabledPluginIds: ["browser", "memory-core"],
373373
});
374374

375-
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
375+
expect(runQaSuite).toHaveBeenCalledWith({
376376
repoRoot: path.resolve("/tmp/openclaw-repo"),
377377
outputDir: undefined,
378378
transportId: "qa-channel",
@@ -393,7 +393,7 @@ describe("qa cli runtime", () => {
393393
runtimePair: "openclaw,codex",
394394
});
395395

396-
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
396+
expect(runQaSuite).toHaveBeenCalledWith({
397397
repoRoot: path.resolve("/tmp/openclaw-repo"),
398398
outputDir: undefined,
399399
transportId: "qa-channel",
@@ -415,7 +415,7 @@ describe("qa cli runtime", () => {
415415
runtimePair: "legacy-runtime,codex",
416416
}),
417417
).rejects.toThrow('--runtime-pair only supports "openclaw" and "codex".');
418-
expect(runQaSuiteFromRuntime).not.toHaveBeenCalled();
418+
expect(runQaSuite).not.toHaveBeenCalled();
419419
});
420420

421421
it("accepts legacy pi as a runtime-pair suite alias", async () => {
@@ -426,7 +426,7 @@ describe("qa cli runtime", () => {
426426
runtimePair: "pi,codex",
427427
});
428428

429-
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith(
429+
expect(runQaSuite).toHaveBeenCalledWith(
430430
expect.objectContaining({
431431
repoRoot: path.resolve("/tmp/openclaw-repo"),
432432
runtimePair: ["openclaw", "codex"],
@@ -443,7 +443,7 @@ describe("qa cli runtime", () => {
443443
scenarioIds: ["thread-memory-isolation"],
444444
});
445445

446-
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
446+
expect(runQaSuite).toHaveBeenCalledWith({
447447
repoRoot: path.resolve("/tmp/openclaw-repo"),
448448
outputDir: undefined,
449449
transportId: "qa-channel",
@@ -590,7 +590,7 @@ describe("qa cli runtime", () => {
590590
concurrency: 3,
591591
});
592592

593-
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
593+
expectFields(mockFirstObjectArg(runQaSuite), {
594594
repoRoot: path.resolve("/tmp/openclaw-repo"),
595595
transportId: "qa-channel",
596596
scenarioIds: ["channel-chat-baseline", "thread-follow-up"],
@@ -607,7 +607,7 @@ describe("qa cli runtime", () => {
607607
concurrency: 1.5,
608608
}),
609609
).rejects.toThrow("--concurrency must be a positive integer");
610-
expect(runQaSuiteFromRuntime).not.toHaveBeenCalled();
610+
expect(runQaSuite).not.toHaveBeenCalled();
611611
});
612612

613613
it("sets a failing exit code when host suite scenarios fail", async () => {
@@ -625,7 +625,7 @@ describe("qa cli runtime", () => {
625625
}),
626626
"utf8",
627627
);
628-
runQaSuiteFromRuntime.mockResolvedValueOnce(
628+
runQaSuite.mockResolvedValueOnce(
629629
flowSuiteRuntimeResult({
630630
reportPath: suiteReportPath,
631631
summaryPath: suiteSummaryPath,
@@ -658,7 +658,7 @@ describe("qa cli runtime", () => {
658658
}),
659659
"utf8",
660660
);
661-
runQaSuiteFromRuntime.mockResolvedValueOnce(
661+
runQaSuite.mockResolvedValueOnce(
662662
flowSuiteRuntimeResult({
663663
reportPath: suiteReportPath,
664664
summaryPath: suiteSummaryPath,
@@ -690,7 +690,7 @@ describe("qa cli runtime", () => {
690690
}),
691691
"utf8",
692692
);
693-
runQaSuiteFromRuntime.mockResolvedValueOnce(
693+
runQaSuite.mockResolvedValueOnce(
694694
flowSuiteRuntimeResult({
695695
reportPath: suiteReportPath,
696696
summaryPath: suiteSummaryPath,
@@ -716,7 +716,7 @@ describe("qa cli runtime", () => {
716716
});
717717

718718
it("retries host suite runs once for retryable infra failures", async () => {
719-
runQaSuiteFromRuntime
719+
runQaSuite
720720
.mockRejectedValueOnce(
721721
new QaSuiteInfraError("agent_wait_failed", "agent.wait failed: gateway call timed out"),
722722
)
@@ -731,12 +731,12 @@ describe("qa cli runtime", () => {
731731
repoRoot: "/tmp/openclaw-repo",
732732
});
733733

734-
expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(2);
734+
expect(runQaSuite).toHaveBeenCalledTimes(2);
735735
expectWriteContains(stderrWrite, "[qa-suite] infra retry 1/1: agent.wait failed");
736736
});
737737

738738
it("retries host suite runs once for qa-channel readiness timeouts", async () => {
739-
runQaSuiteFromRuntime
739+
runQaSuite
740740
.mockRejectedValueOnce(
741741
new QaSuiteInfraError(
742742
"transport_ready_timeout",
@@ -754,15 +754,15 @@ describe("qa cli runtime", () => {
754754
repoRoot: "/tmp/openclaw-repo",
755755
});
756756

757-
expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(2);
757+
expect(runQaSuite).toHaveBeenCalledTimes(2);
758758
expectWriteContains(
759759
stderrWrite,
760760
"[qa-suite] infra retry 1/1: timed out after 180000ms waiting for qa-channel ready",
761761
);
762762
});
763763

764764
it("does not retry host suite runs for generic timeout wording", async () => {
765-
runQaSuiteFromRuntime.mockRejectedValueOnce(
765+
runQaSuite.mockRejectedValueOnce(
766766
new Error("approval-turn timed out waiting for post-approval read"),
767767
);
768768

@@ -772,7 +772,7 @@ describe("qa cli runtime", () => {
772772
}),
773773
).rejects.toThrow("approval-turn timed out waiting for post-approval read");
774774

775-
expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(1);
775+
expect(runQaSuite).toHaveBeenCalledTimes(1);
776776
});
777777

778778
it("does not retry host suite runs for semantic failures", async () => {
@@ -790,7 +790,7 @@ describe("qa cli runtime", () => {
790790
}),
791791
"utf8",
792792
);
793-
runQaSuiteFromRuntime.mockResolvedValueOnce(
793+
runQaSuite.mockResolvedValueOnce(
794794
flowSuiteRuntimeResult({
795795
reportPath: suiteReportPath,
796796
summaryPath: suiteSummaryPath,
@@ -808,7 +808,7 @@ describe("qa cli runtime", () => {
808808
await runQaSuiteCommand({
809809
repoRoot: "/tmp/openclaw-repo",
810810
});
811-
expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(1);
811+
expect(runQaSuite).toHaveBeenCalledTimes(1);
812812
expect(process.exitCode).toBe(1);
813813
} finally {
814814
process.exitCode = priorExitCode;
@@ -927,7 +927,7 @@ describe("qa cli runtime", () => {
927927
scenarioIds: ["claude-cli-provider-capabilities-subscription"],
928928
});
929929

930-
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
930+
expectFields(mockFirstObjectArg(runQaSuite), {
931931
repoRoot: path.resolve("/tmp/openclaw-repo"),
932932
providerMode: "live-frontier",
933933
primaryModel: "claude-cli/claude-sonnet-4-6",
@@ -944,7 +944,7 @@ describe("qa cli runtime", () => {
944944
scenarioIds: ["channel-chat-baseline"],
945945
});
946946

947-
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
947+
expectFields(mockFirstObjectArg(runQaSuite), {
948948
repoRoot: path.resolve("/tmp/openclaw-repo"),
949949
scenarioIds: [
950950
"channel-chat-baseline",
@@ -971,7 +971,7 @@ describe("qa cli runtime", () => {
971971
scenarioIds: ["channel-chat-baseline"],
972972
});
973973

974-
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
974+
expectFields(mockFirstObjectArg(runQaSuite), {
975975
repoRoot: path.resolve("/tmp/openclaw-repo"),
976976
scenarioIds: [
977977
"channel-chat-baseline",
@@ -996,7 +996,7 @@ describe("qa cli runtime", () => {
996996
scenarioIds: ["channel-chat-baseline", "runtime-tool-bash"],
997997
});
998998

999-
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
999+
expectFields(mockFirstObjectArg(runQaSuite), {
10001000
repoRoot: path.resolve("/tmp/openclaw-repo"),
10011001
scenarioIds: [
10021002
"channel-chat-baseline",
@@ -1029,7 +1029,7 @@ describe("qa cli runtime", () => {
10291029
runtimeParityTier: ["optional,soak"],
10301030
});
10311031

1032-
expectFields(mockFirstObjectArg(runQaSuiteFromRuntime), {
1032+
expectFields(mockFirstObjectArg(runQaSuite), {
10331033
scenarioIds: [
10341034
"runtime-soak-100-turn",
10351035
"runtime-tool-image-generate",
@@ -1569,7 +1569,7 @@ describe("qa cli runtime", () => {
15691569
memory: "4G",
15701570
disk: "24G",
15711571
});
1572-
expect(runQaSuiteFromRuntime).not.toHaveBeenCalled();
1572+
expect(runQaSuite).not.toHaveBeenCalled();
15731573
});
15741574

15751575
it("rejects Vitest and Playwright scenarios on the multipass runner", async () => {
@@ -1838,7 +1838,7 @@ describe("qa cli runtime", () => {
18381838
alternateModel: "anthropic/claude-opus-4-8",
18391839
});
18401840

1841-
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
1841+
expect(runQaSuite).toHaveBeenCalledWith({
18421842
repoRoot: path.resolve("/tmp/openclaw-repo"),
18431843
outputDir: undefined,
18441844
transportId: "qa-channel",

‎extensions/qa-lab/src/cli.runtime.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ import {
6868
type QaRuntimeParityTier,
6969
} from "./scenario-catalog.js";
7070
import { resolveQaScenarioPackScenarioIds } from "./scenario-packs.js";
71-
import { runQaFlowSuiteFromRuntime, runQaSuiteFromRuntime } from "./suite-launch.runtime.js";
71+
import { runQaFlowSuiteFromRuntime, runQaSuite } from "./suite-launch.runtime.js";
7272
import { readQaSuiteFailedOrSkippedScenarioCountFromFile } from "./suite-summary.js";
7373
import {
7474
buildTokenEfficiencyReport,
@@ -686,7 +686,7 @@ export async function runQaSuiteCommand(opts: {
686686
}
687687
const thinkingDefault = parseQaThinkingLevel("--thinking", opts.thinking);
688688
const runtimeResult = await runQaSuiteWithInfraRetry(() =>
689-
runQaSuiteFromRuntime({
689+
runQaSuite({
690690
repoRoot,
691691
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
692692
transportId,

‎extensions/qa-lab/src/suite-launch.runtime.test.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ vi.mock("./test-file-scenario-runner.js", async (importOriginal) => ({
1717
runQaTestFileScenarios,
1818
}));
1919

20-
import { runQaSuiteFromRuntime } from "./suite-launch.runtime.js";
20+
import { runQaSuite } from "./suite-launch.runtime.js";
2121

2222
const tempRoots: string[] = [];
2323

@@ -56,7 +56,7 @@ describe("qa suite runtime launcher", () => {
5656
});
5757

5858
it("routes selected flow scenarios to the flow suite engine", async () => {
59-
const result = await runQaSuiteFromRuntime({
59+
const result = await runQaSuite({
6060
repoRoot: process.cwd(),
6161
providerMode: "mock-openai",
6262
scenarioIds: ["channel-chat-baseline"],
@@ -81,7 +81,7 @@ describe("qa suite runtime launcher", () => {
8181

8282
it("routes selected Playwright scenarios to the Playwright scenario runner", async () => {
8383
const repoRoot = await makeTempRepo("qa-suite-launch-");
84-
const result = await runQaSuiteFromRuntime({
84+
const result = await runQaSuite({
8585
repoRoot,
8686
outputDir: ".artifacts/qa-e2e/scenario-test",
8787
scenarioIds: ["control-ui-chat-flow-playwright"],
@@ -112,7 +112,7 @@ describe("qa suite runtime launcher", () => {
112112

113113
it("rejects mixed flow and Vitest/Playwright scenarios", async () => {
114114
await expect(
115-
runQaSuiteFromRuntime({
115+
runQaSuite({
116116
repoRoot: process.cwd(),
117117
scenarioIds: ["channel-chat-baseline", "control-ui-chat-flow-playwright"],
118118
}),
@@ -124,7 +124,7 @@ describe("qa suite runtime launcher", () => {
124124

125125
it("rejects runtime-pair requests for Vitest/Playwright scenarios", async () => {
126126
await expect(
127-
runQaSuiteFromRuntime({
127+
runQaSuite({
128128
repoRoot: process.cwd(),
129129
runtimePair: ["openclaw", "codex"],
130130
scenarioIds: ["control-ui-chat-flow-playwright"],
@@ -141,7 +141,7 @@ describe("qa suite runtime launcher", () => {
141141
await fs.symlink(outsideRoot, path.join(repoRoot, "artifacts-link"));
142142

143143
await expect(
144-
runQaSuiteFromRuntime({
144+
runQaSuite({
145145
repoRoot,
146146
outputDir: "artifacts-link/qa-out",
147147
scenarioIds: ["control-ui-chat-flow-playwright"],

‎extensions/qa-lab/src/suite-launch.runtime.ts‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,7 @@ async function runQaTestFileSuiteFromRuntime(params: {
9090
});
9191
}
9292

93-
export async function runQaSuiteFromRuntime(
94-
...args: [QaSuiteRunParams?]
95-
): Promise<QaSuiteRuntimeResult> {
93+
export async function runQaSuite(...args: [QaSuiteRunParams?]): Promise<QaSuiteRuntimeResult> {
9694
const runParams = args[0];
9795
const testFileScenarios = resolveTestFileScenariosForSuiteDispatch(runParams);
9896
if (testFileScenarios) {

0 commit comments

Comments
 (0)