Skip to content

Commit d8f099e

Browse files
steipetesteipete
authored
fix(qa): honor fail-fast across unified suite runners (#113843)
1 parent f2d2cc3 commit d8f099e

9 files changed

Lines changed: 717 additions & 33 deletions

‎extensions/qa-lab/src/cli.runtime.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1080,6 +1080,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
10801080
fastMode: opts.fastMode,
10811081
...(thinkingDefault ? { thinkingDefault } : {}),
10821082
allowFailures: true,
1083+
...(opts.failFast ? { failFast: true } : {}),
10831084
scenarioIds,
10841085
...(opts.concurrency !== undefined
10851086
? { concurrency: parseQaPositiveIntegerOption("--concurrency", opts.concurrency) }

‎extensions/qa-lab/src/cli.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,23 @@ describe("qa cli registration", () => {
255255
expect(runQaLabSelfCheckCommand).not.toHaveBeenCalled();
256256
});
257257

258+
it("forwards fail-fast to taxonomy-backed QA profile runs", async () => {
259+
await program.parseAsync([
260+
"node",
261+
"openclaw",
262+
"qa",
263+
"run",
264+
"--qa-profile",
265+
"smoke-ci",
266+
"--fail-fast",
267+
]);
268+
269+
expect(runQaProfileCommand).toHaveBeenCalledWith(
270+
expect.objectContaining({ failFast: true, profile: "smoke-ci" }),
271+
);
272+
expect(runQaLabSelfCheckCommand).not.toHaveBeenCalled();
273+
});
274+
258275
it.each([
259276
["--output-dir", [".artifacts/qa-e2e/smoke-ci"]],
260277
["--surface", ["agents"]],
@@ -268,6 +285,7 @@ describe("qa cli registration", () => {
268285
["--alt-model", ["anthropic/claude-sonnet-4-6"]],
269286
["--concurrency", ["2"]],
270287
["--allow-failures", []],
288+
["--fail-fast", []],
271289
["--fast", []],
272290
])("rejects qa run profile-only flag %s without --qa-profile", async (flag, values) => {
273291
await expect(
@@ -894,6 +912,20 @@ describe("qa cli registration", () => {
894912
expect(options.providerMode).toBeUndefined();
895913
});
896914

915+
it.each(["host", "multipass"])("forwards --fail-fast to the %s suite runner", async (runner) => {
916+
await program.parseAsync([
917+
"node",
918+
"openclaw",
919+
"qa",
920+
"suite",
921+
"--runner",
922+
runner,
923+
"--fail-fast",
924+
]);
925+
926+
expect(requireQaSuiteOptions()).toEqual(expect.objectContaining({ failFast: true, runner }));
927+
});
928+
897929
it("forwards --pack for suite runs", async () => {
898930
await program.parseAsync(["node", "openclaw", "qa", "suite", "--pack", "personal-agent"]);
899931

‎extensions/qa-lab/src/cli.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ type QaScenarioRunCliOptions = {
3131
altModel?: QaSuiteCommandOptions["alternateModel"];
3232
concurrency?: QaSuiteCommandOptions["concurrency"];
3333
allowFailures?: QaSuiteCommandOptions["allowFailures"];
34+
failFast?: QaSuiteCommandOptions["failFast"];
3435
fast?: QaSuiteCommandOptions["fastMode"];
3536
};
3637

@@ -57,6 +58,7 @@ const QA_RUN_PROFILE_ONLY_OPTIONS = [
5758
{ optionName: "altModel", flag: "--alt-model" },
5859
{ optionName: "concurrency", flag: "--concurrency" },
5960
{ optionName: "allowFailures", flag: "--allow-failures" },
61+
{ optionName: "failFast", flag: "--fail-fast" },
6062
{ optionName: "fast", flag: "--fast" },
6163
] as const;
6264

@@ -433,6 +435,7 @@ export function registerQaLabCli(program: Command) {
433435
"Write artifacts without setting a failing exit code when scenarios fail",
434436
false,
435437
)
438+
.option("--fail-fast", "Stop after the first failed QA scenario")
436439
.option("--fast", "Enable provider fast mode where supported", false);
437440
qaRun.action(async (opts: QaRunCliOptions, command: Command) => {
438441
validateQaRunMode(opts, command);
@@ -451,6 +454,7 @@ export function registerQaLabCli(program: Command) {
451454
alternateModel: opts.altModel,
452455
concurrency: opts.concurrency,
453456
allowFailures: opts.allowFailures,
457+
...(opts.failFast ? { failFast: true } : {}),
454458
fastMode: opts.fast,
455459
});
456460
return;
@@ -497,6 +501,7 @@ export function registerQaLabCli(program: Command) {
497501
"Write artifacts without setting a failing exit code when scenarios fail",
498502
false,
499503
)
504+
.option("--fail-fast", "Stop after the first failed QA scenario")
500505
.option("--fast", "Enable provider fast mode where supported", false)
501506
.option(
502507
"--thinking <level>",
@@ -535,6 +540,7 @@ export function registerQaLabCli(program: Command) {
535540
enabledPluginIds: opts.enablePlugin,
536541
concurrency: opts.concurrency,
537542
allowFailures: opts.allowFailures,
543+
...(opts.failFast ? { failFast: true } : {}),
538544
image: opts.image,
539545
cpus: opts.cpus,
540546
memory: opts.memory,

‎extensions/qa-lab/src/multipass.runtime.test.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,16 @@ describe("qa multipass runtime", () => {
154154
expect(script).toContain("/workspace/openclaw-host/.artifacts/qa-e2e/multipass-default-test");
155155
});
156156

157+
it("forwards fail-fast to the real guest QA suite command", async () => {
158+
const script = await renderPersistedGuestScript({
159+
outputDirName: "multipass-fail-fast-test",
160+
failFast: true,
161+
scenarioIds: ["channel-chat-baseline"],
162+
});
163+
164+
expect(script).toContain("'--fail-fast'");
165+
});
166+
157167
it("redacts persisted credentials while forwarding them to the executable script", async () => {
158168
vi.stubEnv("OPENAI_API_KEY", TEST_ENV_VALUE);
159169
const { executableScript, persistedScript } = await captureGuestScriptsAtTransfer({

‎extensions/qa-lab/src/multipass.runtime.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,7 @@ function createQaMultipassPlan(params: {
242242
fastMode?: boolean;
243243
thinkingDefault?: string;
244244
allowFailures?: boolean;
245+
failFast?: boolean;
245246
scenarioIds?: string[];
246247
concurrency?: number;
247248
runtimePair?: [RuntimeId, RuntimeId];
@@ -288,6 +289,7 @@ function createQaMultipassPlan(params: {
288289
...(params.fastMode ? ["--fast"] : []),
289290
...(params.thinkingDefault ? ["--thinking", params.thinkingDefault] : []),
290291
...(params.allowFailures ? ["--allow-failures"] : []),
292+
...(params.failFast ? ["--fail-fast"] : []),
291293
...(params.concurrency ? ["--concurrency", String(params.concurrency)] : []),
292294
...(params.runtimePair ? ["--runtime-pair", params.runtimePair.join(",")] : []),
293295
...(params.channelDriverSelection
@@ -572,6 +574,7 @@ export async function runQaMultipass(params: {
572574
alternateModel?: string;
573575
fastMode?: boolean;
574576
allowFailures?: boolean;
577+
failFast?: boolean;
575578
scenarioIds?: string[];
576579
concurrency?: number;
577580
runtimePair?: [RuntimeId, RuntimeId];

0 commit comments

Comments
 (0)