Skip to content

Commit ab9e233

Browse files
RomneyDaRomneyDa
authored
improve(qa): execute runtime scenarios through Docker (#99705)
* test(qa): run Docker runtime scenarios end to end * test(qa): keep plugin lifecycle probe runtime-only
1 parent bc802ce commit ab9e233

8 files changed

Lines changed: 98 additions & 28 deletions

‎extensions/qa-lab/src/scenario-catalog.test.ts‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -242,16 +242,38 @@ describe("qa scenario catalog", () => {
242242
expect(readQaScenarioById("openai-web-search-minimal").coverage?.secondary).toContain(
243243
"runtime.reasoning-and-cache-controls",
244244
);
245-
expect(
246-
readQaScenarioById("openai-web-search-native-assertions").coverage?.secondary,
247-
).toEqual(
248-
expect.arrayContaining(["web-search.openai-native-web-search", "plugins.web-search-and-fetch"]),
245+
expect(readQaScenarioById("openai-web-search-native-assertions").coverage?.secondary).toEqual(
246+
expect.arrayContaining([
247+
"web-search.openai-native-web-search",
248+
"plugins.web-search-and-fetch",
249+
]),
249250
);
250251
expect(readQaScenarioById("openwebui-openai-compatible").coverage?.secondary).toEqual(
251252
expect.arrayContaining(["gateway.openai-compatible-apis", "runtime.hosted-provider-turns"]),
252253
);
253254
});
254255

256+
it("routes Docker runtime scenarios through the shared lane adapter", () => {
257+
const scenarioLanes = [
258+
["openai-compatible-chat-tools", "openai-chat-tools"],
259+
["openai-web-search-minimal", "openai-web-search-minimal"],
260+
["openai-web-search-native-assertions", "openai-web-search-minimal"],
261+
["openwebui-openai-compatible", "openwebui"],
262+
["plugin-lifecycle-probe", "plugin-lifecycle-matrix"],
263+
["packaged-bundled-plugin-install-uninstall", "bundled-plugin-install-uninstall"],
264+
] as const;
265+
266+
for (const [scenarioId, lane] of scenarioLanes) {
267+
const execution = readQaScenarioById(scenarioId).execution;
268+
expect(execution.kind).toBe("script");
269+
if (execution.kind !== "script") {
270+
throw new Error(`expected script scenario, got ${execution.kind}`);
271+
}
272+
expect(execution.path).toBe("test/e2e/qa-lab/runtime/docker-e2e-lane.ts");
273+
expect(execution.args).toStrictEqual(["--lane", lane]);
274+
}
275+
});
276+
255277
it("loads runtime parity tier metadata for first-hour and soak lanes", () => {
256278
const firstHour = readQaScenarioById("runtime-first-hour-20-turn");
257279
const soak = readQaScenarioById("runtime-soak-100-turn");

‎qa/scenarios/plugins/packaged-bundled-plugin-install-uninstall.yaml‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,17 @@ scenario:
2020
- docs/cli/plugins.md
2121
- docs/help/testing.md
2222
codeRefs:
23+
- scripts/e2e/bundled-plugin-install-uninstall-docker.sh
24+
- scripts/e2e/lib/bundled-plugin-install-uninstall/sweep.sh
2325
- scripts/e2e/lib/bundled-plugin-install-uninstall/probe.mjs
26+
- scripts/e2e/lib/bundled-plugin-install-uninstall/runtime-smoke.mjs
27+
- test/e2e/qa-lab/runtime/docker-e2e-lane.ts
2428
- test/scripts/bundled-plugin-install-uninstall-probe.test.ts
2529
execution:
26-
kind: vitest
27-
path: test/scripts/bundled-plugin-install-uninstall-probe.test.ts
28-
summary: Vitest coverage for packaged bundled plugin root discovery and install/uninstall record assertions.
30+
kind: script
31+
path: test/e2e/qa-lab/runtime/docker-e2e-lane.ts
32+
summary: Runs the packaged bundled-plugin Docker sweep and asserts install, runtime load, and uninstall evidence for each selected plugin.
33+
timeoutMs: 7200000
34+
args:
35+
- --lane
36+
- bundled-plugin-install-uninstall

‎qa/scenarios/plugins/plugin-lifecycle-probe.yaml‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,26 @@ scenario:
88
- plugins.lifecycle
99
- cli.plugin-validation-repair
1010
- plugins.setup-flows
11-
objective: Exercise strict plugin load/uninstall proof parsing through QA Lab evidence.
11+
objective: Exercise packaged plugin install, inspect, disable, enable, update, downgrade, and uninstall behavior in a clean Docker runtime.
1212
successCriteria:
13-
- Enabled loaded plugin inspect JSON is accepted as proof.
14-
- Pending or missing inspect JSON is rejected instead of treated as loaded.
15-
- Malformed config during uninstall proof fails with a bounded diagnostic.
13+
- A clean container installs the candidate OpenClaw package and fixture plugin package.
14+
- Runtime inspect reports the installed plugin as enabled and loaded.
15+
- Disable and enable commands persist the expected plugin state.
16+
- Update and downgrade commands replace the installed plugin version while preserving the npm project root.
17+
- Forced uninstall succeeds after installed plugin code is removed and emits a bounded resource summary.
1618
docsRefs:
1719
- docs/plugins/manifest.md
1820
- docs/cli/plugins.md
1921
- docs/concepts/qa-e2e-automation.md
2022
codeRefs:
23+
- scripts/e2e/plugin-lifecycle-matrix-docker.sh
24+
- test/e2e/qa-lab/plugins/plugin-lifecycle-probe-runtime.ts
25+
- test/e2e/qa-lab/runtime/docker-e2e-lane.ts
2126
- test/e2e/qa-lab/plugins/plugin-lifecycle-probe.e2e.test.ts
2227
execution:
23-
kind: vitest
24-
path: test/e2e/qa-lab/plugins/plugin-lifecycle-probe.e2e.test.ts
25-
summary: Vitest coverage for plugin lifecycle proof parsing.
28+
kind: script
29+
path: test/e2e/qa-lab/runtime/docker-e2e-lane.ts
30+
summary: Runs the packaged plugin lifecycle matrix in a clean Docker runtime and asserts each emitted lifecycle phase.
31+
args:
32+
- --lane
33+
- plugin-lifecycle-matrix

‎qa/scenarios/runtime/openai-compatible-chat-tools.yaml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,12 @@ scenario:
2121
- scripts/e2e/lib/openai-chat-tools/client.mjs
2222
- scripts/e2e/lib/openai-chat-tools/write-config.mjs
2323
- scripts/e2e/openai-chat-tools-docker.sh
24+
- test/e2e/qa-lab/runtime/docker-e2e-lane.ts
2425
- test/e2e/qa-lab/runtime/openai-compatible-chat-tools.e2e.test.ts
2526
execution:
26-
kind: vitest
27-
path: test/e2e/qa-lab/runtime/openai-compatible-chat-tools.e2e.test.ts
28-
summary: Vitest coverage for OpenAI-compatible chat-completions tool-call API behavior.
27+
kind: script
28+
path: test/e2e/qa-lab/runtime/docker-e2e-lane.ts
29+
summary: Runs the OpenAI chat-tools Docker E2E lane against a real Gateway and the OpenAI-compatible chat-completions API.
30+
args:
31+
- --lane
32+
- openai-chat-tools

‎qa/scenarios/runtime/openai-web-search-minimal.yaml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,12 @@ scenario:
2121
codeRefs:
2222
- scripts/e2e/lib/openai-web-search-minimal/client.mjs
2323
- scripts/e2e/openai-web-search-minimal-docker.sh
24+
- test/e2e/qa-lab/runtime/docker-e2e-lane.ts
2425
- test/e2e/qa-lab/runtime/openai-web-search-minimal.e2e.test.ts
2526
execution:
26-
kind: vitest
27-
path: test/e2e/qa-lab/runtime/openai-web-search-minimal.e2e.test.ts
28-
summary: Vitest coverage for OpenAI web_search minimal-reasoning success and rejection validation.
27+
kind: script
28+
path: test/e2e/qa-lab/runtime/docker-e2e-lane.ts
29+
summary: Runs the OpenAI web_search Docker E2E lane through a real Gateway with a container-local OpenAI Responses mock.
30+
args:
31+
- --lane
32+
- openai-web-search-minimal

‎qa/scenarios/runtime/openai-web-search-native-assertions.yaml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,13 @@ scenario:
2222
codeRefs:
2323
- scripts/e2e/lib/openai-web-search-minimal/assertions.mjs
2424
- scripts/e2e/lib/openai-web-search-minimal/mock-server.mjs
25+
- scripts/e2e/openai-web-search-minimal-docker.sh
26+
- test/e2e/qa-lab/runtime/docker-e2e-lane.ts
2527
- test/e2e/qa-lab/runtime/openai-web-search-minimal-assertions.e2e.test.ts
2628
execution:
27-
kind: vitest
28-
path: test/e2e/qa-lab/runtime/openai-web-search-minimal-assertions.e2e.test.ts
29-
summary: Vitest coverage for native OpenAI web_search request-log assertions.
29+
kind: script
30+
path: test/e2e/qa-lab/runtime/docker-e2e-lane.ts
31+
summary: Runs the OpenAI web_search Docker E2E lane and asserts the emitted native Responses request log.
32+
args:
33+
- --lane
34+
- openai-web-search-minimal

‎qa/scenarios/runtime/openwebui-openai-compatible.yaml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,12 @@ scenario:
2020
codeRefs:
2121
- scripts/e2e/openwebui-probe.mjs
2222
- scripts/e2e/openwebui-docker.sh
23+
- test/e2e/qa-lab/runtime/docker-e2e-lane.ts
2324
- test/e2e/qa-lab/runtime/openwebui-probe.e2e.test.ts
2425
execution:
25-
kind: vitest
26-
path: test/e2e/qa-lab/runtime/openwebui-probe.e2e.test.ts
27-
summary: Vitest coverage for OpenWebUI model and chat-completions probe behavior.
26+
kind: script
27+
path: test/e2e/qa-lab/runtime/docker-e2e-lane.ts
28+
summary: Runs the OpenWebUI Docker topology and probes its OpenAI-compatible model and chat APIs against OpenClaw.
29+
args:
30+
- --lane
31+
- openwebui

‎test/e2e/qa-lab/plugins/plugin-lifecycle-probe-runtime.ts‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,24 @@ import os from "node:os";
66
import path from "node:path";
77
import { readPluginInstallRecords } from "../../../../scripts/e2e/lib/plugin-index-sqlite.mjs";
88
import { resolveWindowsTaskkillPath } from "../../../../scripts/lib/windows-taskkill.mjs";
9-
import { createTempDirTracker } from "../../../helpers/temp-dir.js";
109

11-
const tempDirs = createTempDirTracker();
10+
// The Docker entrypoint runs without Vitest installed, so keep cleanup local to this runtime probe.
11+
const tempDirs = (() => {
12+
const dirs = new Set<string>();
13+
return {
14+
make(prefix: string): string {
15+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
16+
dirs.add(dir);
17+
return dir;
18+
},
19+
cleanup(): void {
20+
for (const dir of dirs) {
21+
fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 });
22+
}
23+
dirs.clear();
24+
},
25+
};
26+
})();
1227

1328
type ProbeEnv = Pick<NodeJS.ProcessEnv, "HOME" | "OPENCLAW_CONFIG_PATH" | "OPENCLAW_STATE_DIR">;
1429

0 commit comments

Comments
 (0)