Skip to content

Commit 02d017f

Browse files
RomneyDaRomneyDa
committed
test(qa): run gateway and MCP scenarios over real transports
1 parent e779b8f commit 02d017f

8 files changed

Lines changed: 850 additions & 25 deletions

File tree

‎extensions/qa-lab/api.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ export {
101101
testing as __testing,
102102
buildQaRuntimeEnv,
103103
type QaCliBackendAuthMode,
104+
type QaGatewayChildListeningContext,
104105
type QaGatewayChildCommand,
105106
type QaGatewayChildStateMutationContext,
106107
resolveQaControlUiRoot,

‎extensions/qa-lab/src/bundled-plugin-staging.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -194,12 +194,13 @@ function collectQaBundledPluginIds(params: {
194194
repoRoot: string;
195195
allowedPluginIds: readonly string[];
196196
}) {
197-
const pluginIds = new Set(
198-
params.allowedPluginIds.map((pluginId) => {
199-
assertSafeQaBundledPluginId(pluginId);
200-
return pluginId;
201-
}),
202-
);
197+
const pluginIds = new Set<string>();
198+
for (const pluginId of params.allowedPluginIds) {
199+
assertSafeQaBundledPluginId(pluginId);
200+
if (resolveQaBundledPluginSourceDir({ repoRoot: params.repoRoot, pluginId })) {
201+
pluginIds.add(pluginId);
202+
}
203+
}
203204
for (const pluginId of QA_ALWAYS_STAGE_RUNTIME_PLUGIN_IDS) {
204205
if (
205206
resolveQaBundledPluginSourceDir({

‎extensions/qa-lab/src/gateway-child.test.ts‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import {
1111
resolveQaControlUiRoot,
1212
startQaGatewayChild,
1313
} from "./gateway-child.js";
14+
import { createTempDirHarness } from "./temp-dir.test-helper.js";
1415

1516
const fetchWithSsrFGuardMock = vi.hoisted(() => vi.fn());
1617
const resolveQaNodeExecPathMock = vi.hoisted(() => vi.fn(async () => process.execPath));
@@ -22,7 +23,8 @@ vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
2223
fetchWithSsrFGuard: fetchWithSsrFGuardMock,
2324
}));
2425

25-
vi.mock("openclaw/plugin-sdk/temp-path", () => ({
26+
vi.mock("openclaw/plugin-sdk/temp-path", async (importOriginal) => ({
27+
...(await importOriginal<typeof import("openclaw/plugin-sdk/temp-path")>()),
2628
resolvePreferredOpenClawTmpDir: () => qaTempPathState.preferredTmpDir,
2729
}));
2830

@@ -31,6 +33,7 @@ vi.mock("./node-exec.js", () => ({
3133
}));
3234

3335
const cleanups: Array<() => Promise<void>> = [];
36+
const tempDirs = createTempDirHarness();
3437

3538
afterEach(async () => {
3639
fetchWithSsrFGuardMock.mockReset();
@@ -39,6 +42,7 @@ afterEach(async () => {
3942
while (cleanups.length > 0) {
4043
await cleanups.pop()?.();
4144
}
45+
await tempDirs.cleanup();
4246
});
4347

4448
function createParams(baseEnv?: NodeJS.ProcessEnv) {
@@ -1621,6 +1625,24 @@ describe("qa bundled plugin dir", () => {
16211625
).rejects.toThrow("invalid QA bundled plugin id: ../escape");
16221626
});
16231627

1628+
it("leaves external allowed plugins to configured load paths", async () => {
1629+
const repoRoot = await tempDirs.makeTempDir("qa-bundled-external-id-");
1630+
await writeFile(
1631+
path.join(repoRoot, "package.json"),
1632+
JSON.stringify({ name: "openclaw", type: "module" }, null, 2),
1633+
"utf8",
1634+
);
1635+
const tempRoot = await tempDirs.makeTempDir("qa-bundled-external-target-");
1636+
1637+
const { bundledPluginsDir } = await testing.createQaBundledPluginsDir({
1638+
repoRoot,
1639+
tempRoot,
1640+
allowedPluginIds: ["external-fixture"],
1641+
});
1642+
1643+
await expect(readdir(bundledPluginsDir)).resolves.not.toContain("external-fixture");
1644+
});
1645+
16241646
it("stages source-only bundled plugins into a repo-like runtime root with node_modules", async () => {
16251647
const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-bundled-source-stage-"));
16261648
cleanups.push(async () => {

‎extensions/qa-lab/src/gateway-child.ts‎

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,19 @@ export type QaGatewayChildCommand = {
7777
usePackagedPlugins?: boolean;
7878
};
7979

80+
export type QaGatewayChildListeningContext = {
81+
attempt: number;
82+
baseUrl: string;
83+
wsUrl: string;
84+
token: string;
85+
configPath: string;
86+
runtimeEnv: NodeJS.ProcessEnv;
87+
};
88+
8089
async function getFreePort() {
8190
return await new Promise<number>((resolve, reject) => {
8291
const server = net.createServer();
83-
server.once("error", reject);
92+
server.once("error", (error) => reject(error));
8493
server.listen(0, "127.0.0.1", () => {
8594
const address = server.address();
8695
if (!address || typeof address === "string") {
@@ -317,6 +326,23 @@ async function fetchLocalGatewayHealth(params: {
317326
}
318327
}
319328

329+
async function fetchLocalGatewayListening(baseUrl: string): Promise<boolean> {
330+
const { release } = await fetchWithSsrFGuard({
331+
url: `${baseUrl}/healthz`,
332+
init: {
333+
method: "HEAD",
334+
headers: {
335+
connection: "close",
336+
},
337+
signal: AbortSignal.timeout(2_000),
338+
},
339+
policy: { allowPrivateNetwork: true },
340+
auditContext: "qa-lab-gateway-child-listening",
341+
});
342+
await release();
343+
return true;
344+
}
345+
320346
async function waitForQaGatewayRestartBoundary(params: {
321347
logs: () => string;
322348
offset: number;
@@ -575,6 +601,47 @@ async function waitForGatewayReady(params: {
575601
);
576602
}
577603

604+
async function waitForGatewayListening(params: {
605+
baseUrl: string;
606+
logs: () => string;
607+
child: {
608+
exitCode: number | null;
609+
signalCode: NodeJS.Signals | null;
610+
};
611+
getSpawnError?: () => unknown;
612+
timeoutMs?: number;
613+
}) {
614+
const startedAt = Date.now();
615+
while (Date.now() - startedAt < (params.timeoutMs ?? 60_000)) {
616+
const spawnError = params.getSpawnError?.();
617+
if (spawnError) {
618+
throw new QaSuiteInfraError(
619+
"gateway_startup_unhealthy",
620+
`gateway failed to spawn: ${formatErrorMessage(spawnError)}\n${params.logs()}`,
621+
{ cause: spawnError },
622+
);
623+
}
624+
if (params.child.exitCode !== null || params.child.signalCode !== null) {
625+
throw new QaSuiteInfraError(
626+
"gateway_startup_unhealthy",
627+
`gateway exited before listening (exitCode=${String(params.child.exitCode)}, signal=${String(params.child.signalCode)}):\n${params.logs()}`,
628+
);
629+
}
630+
try {
631+
if (await fetchLocalGatewayListening(params.baseUrl)) {
632+
return;
633+
}
634+
} catch {
635+
// retry until the HTTP listener accepts requests
636+
}
637+
await sleep(100);
638+
}
639+
throw new QaSuiteInfraError(
640+
"gateway_startup_unhealthy",
641+
`gateway failed to listen before timeout:\n${params.logs()}`,
642+
);
643+
}
644+
578645
function isRetryableRpcStartupError(error: unknown) {
579646
const details = formatErrorMessage(error);
580647
return (
@@ -614,6 +681,7 @@ export async function startQaGatewayChild(params: {
614681
enabledPluginIds?: string[];
615682
forwardHostHome?: boolean;
616683
mockAuthAgentIds?: readonly string[];
684+
onListening?: (context: QaGatewayChildListeningContext) => Promise<void> | void;
617685
mutateConfig?: (cfg: OpenClawConfig) => OpenClawConfig;
618686
runtimeEnvPatch?: NodeJS.ProcessEnv;
619687
}) {
@@ -834,6 +902,21 @@ export async function startQaGatewayChild(params: {
834902
const getAttemptSpawnError = monitorQaGatewayChildSpawnError(attemptChild, output);
835903

836904
try {
905+
await waitForGatewayListening({
906+
baseUrl,
907+
logs,
908+
child: attemptChild,
909+
getSpawnError: getAttemptSpawnError,
910+
timeoutMs: 120_000,
911+
});
912+
await params.onListening?.({
913+
attempt,
914+
baseUrl,
915+
wsUrl,
916+
token: gatewayToken,
917+
configPath,
918+
runtimeEnv: env,
919+
});
837920
await waitForGatewayReady({
838921
baseUrl,
839922
logs,

‎qa/scenarios/plugins/mcp-plugin-tools-call.yaml‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ scenario:
44
id: mcp-plugin-tools-call
55
surface: mcp
66
coverage:
7-
secondary:
7+
primary:
88
- plugins.mcp-tools
99
- tools.invocation
1010
objective: Verify OpenClaw can expose plugin tools over MCP and a real MCP client can call one successfully.
@@ -16,9 +16,15 @@ scenario:
1616
- docs/cli/mcp.md
1717
- docs/gateway/protocol.md
1818
codeRefs:
19+
- test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
1920
- src/mcp/plugin-tools-serve.ts
20-
- src/mcp/plugin-tools-mcp-client.test.ts
21+
- src/mcp/plugin-tools-handlers.ts
2122
execution:
22-
kind: vitest
23-
path: src/mcp/plugin-tools-mcp-client.test.ts
24-
summary: Verify OpenClaw can expose plugin tools over MCP and a real MCP client can call one successfully.
23+
kind: script
24+
path: test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
25+
summary: Registers a fixture plugin, starts the real plugin-tools stdio server, and calls its tool with a real MCP client.
26+
args:
27+
- --scenario
28+
- mcp-plugin-tools-call
29+
- --artifact-base
30+
- ${outputDir}

‎qa/scenarios/runtime/gateway-smoke.yaml‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ scenario:
44
id: gateway-smoke
55
surface: runtime
66
coverage:
7-
secondary:
7+
primary:
88
- gateway.websocket-transport
99
- gateway.health-apis
1010
- gateway.hello-ok-snapshot
@@ -18,8 +18,15 @@ scenario:
1818
- docs/gateway/index.md
1919
- docs/concepts/qa-e2e-automation.md
2020
codeRefs:
21+
- test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
22+
- extensions/qa-lab/src/gateway-child.ts
2123
- test/e2e/qa-lab/runtime/gateway-smoke.e2e.test.ts
2224
execution:
23-
kind: vitest
24-
path: test/e2e/qa-lab/runtime/gateway-smoke.e2e.test.ts
25-
summary: Vitest coverage for gateway health and WebSocket smoke checks.
25+
kind: script
26+
path: test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
27+
summary: Starts a real Gateway child and exercises the Gateway smoke client against its WebSocket and health RPC surfaces.
28+
args:
29+
- --scenario
30+
- gateway-smoke
31+
- --artifact-base
32+
- ${outputDir}

‎qa/scenarios/runtime/mcp-gateway-connect-startup-retry.yaml‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ scenario:
44
id: mcp-gateway-connect-startup-retry
55
surface: runtime
66
coverage:
7-
secondary:
7+
primary:
88
- gateway.connect-request
99
- gateway.protocol-version-negotiation
1010
- gateway.startup-retry
@@ -18,11 +18,15 @@ scenario:
1818
- docs/cli/mcp.md
1919
- docs/concepts/qa-e2e-automation.md
2020
codeRefs:
21-
- src/gateway/client.test.ts
22-
- test/e2e/qa-lab/runtime/mcp-channels.fixture.ts
23-
- test/e2e/qa-lab/runtime/mcp-client-temp-state.fixture.ts
24-
- test/e2e/qa-lab/runtime/mcp-gateway-transport.e2e.test.ts
21+
- test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
22+
- extensions/qa-lab/src/gateway-child.ts
23+
- src/mcp/channel-bridge.ts
2524
execution:
26-
kind: vitest
27-
path: src/gateway/client.test.ts
28-
summary: Vitest coverage for Gateway connect request, protocol negotiation, and startup retry behavior used by MCP bridge clients.
25+
kind: script
26+
path: test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
27+
summary: Starts the real MCP client before a delayed real Gateway becomes ready and captures retry, connect-frame, and negotiated-protocol evidence.
28+
args:
29+
- --scenario
30+
- mcp-gateway-connect-startup-retry
31+
- --artifact-base
32+
- ${outputDir}

0 commit comments

Comments
 (0)