Skip to content

Commit 9d68f87

Browse files
RomneyDaRomneyDa
authored
test(qa): run gateway and MCP scenarios over real transports (#99735)
1 parent 3b4092d commit 9d68f87

8 files changed

Lines changed: 850 additions & 25 deletions

File tree

‎extensions/qa-lab/api.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ export {
101101
testing as __testing,
102102
buildQaRuntimeEnv,
103103
type QaCliBackendAuthMode,
104+
type QaGatewayChildListeningContext,
104105
type QaGatewayChildCommand,
105106
type QaGatewayChildStateMutationContext,
106107
resolveQaControlUiRoot,

‎extensions/qa-lab/src/bundled-plugin-staging.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -194,12 +194,13 @@ function collectQaBundledPluginIds(params: {
194194
repoRoot: string;
195195
allowedPluginIds: readonly string[];
196196
}) {
197-
const pluginIds = new Set(
198-
params.allowedPluginIds.map((pluginId) => {
199-
assertSafeQaBundledPluginId(pluginId);
200-
return pluginId;
201-
}),
202-
);
197+
const pluginIds = new Set<string>();
198+
for (const pluginId of params.allowedPluginIds) {
199+
assertSafeQaBundledPluginId(pluginId);
200+
if (resolveQaBundledPluginSourceDir({ repoRoot: params.repoRoot, pluginId })) {
201+
pluginIds.add(pluginId);
202+
}
203+
}
203204
for (const pluginId of QA_ALWAYS_STAGE_RUNTIME_PLUGIN_IDS) {
204205
if (
205206
resolveQaBundledPluginSourceDir({

‎extensions/qa-lab/src/gateway-child.test.ts‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import {
1111
resolveQaControlUiRoot,
1212
startQaGatewayChild,
1313
} from "./gateway-child.js";
14+
import { createTempDirHarness } from "./temp-dir.test-helper.js";
1415

1516
const fetchWithSsrFGuardMock = vi.hoisted(() => vi.fn());
1617
const resolveQaNodeExecPathMock = vi.hoisted(() => vi.fn(async () => process.execPath));
@@ -22,7 +23,8 @@ vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
2223
fetchWithSsrFGuard: fetchWithSsrFGuardMock,
2324
}));
2425

25-
vi.mock("openclaw/plugin-sdk/temp-path", () => ({
26+
vi.mock("openclaw/plugin-sdk/temp-path", async (importOriginal) => ({
27+
...(await importOriginal<typeof import("openclaw/plugin-sdk/temp-path")>()),
2628
resolvePreferredOpenClawTmpDir: () => qaTempPathState.preferredTmpDir,
2729
}));
2830

@@ -31,6 +33,7 @@ vi.mock("./node-exec.js", () => ({
3133
}));
3234

3335
const cleanups: Array<() => Promise<void>> = [];
36+
const tempDirs = createTempDirHarness();
3437

3538
afterEach(async () => {
3639
fetchWithSsrFGuardMock.mockReset();
@@ -39,6 +42,7 @@ afterEach(async () => {
3942
while (cleanups.length > 0) {
4043
await cleanups.pop()?.();
4144
}
45+
await tempDirs.cleanup();
4246
});
4347

4448
function createParams(baseEnv?: NodeJS.ProcessEnv) {
@@ -1623,6 +1627,24 @@ describe("qa bundled plugin dir", () => {
16231627
).rejects.toThrow("invalid QA bundled plugin id: ../escape");
16241628
});
16251629

1630+
it("leaves external allowed plugins to configured load paths", async () => {
1631+
const repoRoot = await tempDirs.makeTempDir("qa-bundled-external-id-");
1632+
await writeFile(
1633+
path.join(repoRoot, "package.json"),
1634+
JSON.stringify({ name: "openclaw", type: "module" }, null, 2),
1635+
"utf8",
1636+
);
1637+
const tempRoot = await tempDirs.makeTempDir("qa-bundled-external-target-");
1638+
1639+
const { bundledPluginsDir } = await testing.createQaBundledPluginsDir({
1640+
repoRoot,
1641+
tempRoot,
1642+
allowedPluginIds: ["external-fixture"],
1643+
});
1644+
1645+
await expect(readdir(bundledPluginsDir)).resolves.not.toContain("external-fixture");
1646+
});
1647+
16261648
it("stages source-only bundled plugins into a repo-like runtime root with node_modules", async () => {
16271649
const repoRoot = await mkdtemp(path.join(os.tmpdir(), "qa-bundled-source-stage-"));
16281650
cleanups.push(async () => {

‎extensions/qa-lab/src/gateway-child.ts‎

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,19 @@ export type QaGatewayChildCommand = {
7777
usePackagedPlugins?: boolean;
7878
};
7979

80+
export type QaGatewayChildListeningContext = {
81+
attempt: number;
82+
baseUrl: string;
83+
wsUrl: string;
84+
token: string;
85+
configPath: string;
86+
runtimeEnv: NodeJS.ProcessEnv;
87+
};
88+
8089
async function getFreePort() {
8190
return await new Promise<number>((resolve, reject) => {
8291
const server = net.createServer();
83-
server.once("error", reject);
92+
server.once("error", (error) => reject(error));
8493
server.listen(0, "127.0.0.1", () => {
8594
const address = server.address();
8695
if (!address || typeof address === "string") {
@@ -319,6 +328,23 @@ async function fetchLocalGatewayHealth(params: {
319328
}
320329
}
321330

331+
async function fetchLocalGatewayListening(baseUrl: string): Promise<boolean> {
332+
const { release } = await fetchWithSsrFGuard({
333+
url: `${baseUrl}/healthz`,
334+
init: {
335+
method: "HEAD",
336+
headers: {
337+
connection: "close",
338+
},
339+
signal: AbortSignal.timeout(2_000),
340+
},
341+
policy: { allowPrivateNetwork: true },
342+
auditContext: "qa-lab-gateway-child-listening",
343+
});
344+
await release();
345+
return true;
346+
}
347+
322348
async function waitForQaGatewayRestartBoundary(params: {
323349
logs: () => string;
324350
offset: number;
@@ -577,6 +603,47 @@ async function waitForGatewayReady(params: {
577603
);
578604
}
579605

606+
async function waitForGatewayListening(params: {
607+
baseUrl: string;
608+
logs: () => string;
609+
child: {
610+
exitCode: number | null;
611+
signalCode: NodeJS.Signals | null;
612+
};
613+
getSpawnError?: () => unknown;
614+
timeoutMs?: number;
615+
}) {
616+
const startedAt = Date.now();
617+
while (Date.now() - startedAt < (params.timeoutMs ?? 60_000)) {
618+
const spawnError = params.getSpawnError?.();
619+
if (spawnError) {
620+
throw new QaSuiteInfraError(
621+
"gateway_startup_unhealthy",
622+
`gateway failed to spawn: ${formatErrorMessage(spawnError)}\n${params.logs()}`,
623+
{ cause: spawnError },
624+
);
625+
}
626+
if (params.child.exitCode !== null || params.child.signalCode !== null) {
627+
throw new QaSuiteInfraError(
628+
"gateway_startup_unhealthy",
629+
`gateway exited before listening (exitCode=${String(params.child.exitCode)}, signal=${String(params.child.signalCode)}):\n${params.logs()}`,
630+
);
631+
}
632+
try {
633+
if (await fetchLocalGatewayListening(params.baseUrl)) {
634+
return;
635+
}
636+
} catch {
637+
// retry until the HTTP listener accepts requests
638+
}
639+
await sleep(100);
640+
}
641+
throw new QaSuiteInfraError(
642+
"gateway_startup_unhealthy",
643+
`gateway failed to listen before timeout:\n${params.logs()}`,
644+
);
645+
}
646+
580647
function isRetryableRpcStartupError(error: unknown) {
581648
const details = formatErrorMessage(error);
582649
return (
@@ -616,6 +683,7 @@ export async function startQaGatewayChild(params: {
616683
enabledPluginIds?: string[];
617684
forwardHostHome?: boolean;
618685
mockAuthAgentIds?: readonly string[];
686+
onListening?: (context: QaGatewayChildListeningContext) => Promise<void> | void;
619687
mutateConfig?: (cfg: OpenClawConfig) => OpenClawConfig;
620688
runtimeEnvPatch?: NodeJS.ProcessEnv;
621689
}) {
@@ -836,6 +904,21 @@ export async function startQaGatewayChild(params: {
836904
const getAttemptSpawnError = monitorQaGatewayChildSpawnError(attemptChild, output);
837905

838906
try {
907+
await waitForGatewayListening({
908+
baseUrl,
909+
logs,
910+
child: attemptChild,
911+
getSpawnError: getAttemptSpawnError,
912+
timeoutMs: 120_000,
913+
});
914+
await params.onListening?.({
915+
attempt,
916+
baseUrl,
917+
wsUrl,
918+
token: gatewayToken,
919+
configPath,
920+
runtimeEnv: env,
921+
});
839922
await waitForGatewayReady({
840923
baseUrl,
841924
logs,

‎qa/scenarios/plugins/mcp-plugin-tools-call.yaml‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ scenario:
44
id: mcp-plugin-tools-call
55
surface: mcp
66
coverage:
7-
secondary:
7+
primary:
88
- plugins.mcp-tools
99
- tools.invocation
1010
objective: Verify OpenClaw can expose plugin tools over MCP and a real MCP client can call one successfully.
@@ -16,9 +16,15 @@ scenario:
1616
- docs/cli/mcp.md
1717
- docs/gateway/protocol.md
1818
codeRefs:
19+
- test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
1920
- src/mcp/plugin-tools-serve.ts
20-
- src/mcp/plugin-tools-mcp-client.test.ts
21+
- src/mcp/plugin-tools-handlers.ts
2122
execution:
22-
kind: vitest
23-
path: src/mcp/plugin-tools-mcp-client.test.ts
24-
summary: Verify OpenClaw can expose plugin tools over MCP and a real MCP client can call one successfully.
23+
kind: script
24+
path: test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
25+
summary: Registers a fixture plugin, starts the real plugin-tools stdio server, and calls its tool with a real MCP client.
26+
args:
27+
- --scenario
28+
- mcp-plugin-tools-call
29+
- --artifact-base
30+
- ${outputDir}

‎qa/scenarios/runtime/gateway-smoke.yaml‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ scenario:
44
id: gateway-smoke
55
surface: runtime
66
coverage:
7-
secondary:
7+
primary:
88
- gateway.websocket-transport
99
- gateway.health-apis
1010
- gateway.hello-ok-snapshot
@@ -18,8 +18,15 @@ scenario:
1818
- docs/gateway/index.md
1919
- docs/concepts/qa-e2e-automation.md
2020
codeRefs:
21+
- test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
22+
- extensions/qa-lab/src/gateway-child.ts
2123
- test/e2e/qa-lab/runtime/gateway-smoke.e2e.test.ts
2224
execution:
23-
kind: vitest
24-
path: test/e2e/qa-lab/runtime/gateway-smoke.e2e.test.ts
25-
summary: Vitest coverage for gateway health and WebSocket smoke checks.
25+
kind: script
26+
path: test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
27+
summary: Starts a real Gateway child and exercises the Gateway smoke client against its WebSocket and health RPC surfaces.
28+
args:
29+
- --scenario
30+
- gateway-smoke
31+
- --artifact-base
32+
- ${outputDir}

‎qa/scenarios/runtime/mcp-gateway-connect-startup-retry.yaml‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ scenario:
44
id: mcp-gateway-connect-startup-retry
55
surface: runtime
66
coverage:
7-
secondary:
7+
primary:
88
- gateway.connect-request
99
- gateway.protocol-version-negotiation
1010
- gateway.startup-retry
@@ -18,11 +18,15 @@ scenario:
1818
- docs/cli/mcp.md
1919
- docs/concepts/qa-e2e-automation.md
2020
codeRefs:
21-
- src/gateway/client.test.ts
22-
- test/e2e/qa-lab/runtime/mcp-channels.fixture.ts
23-
- test/e2e/qa-lab/runtime/mcp-client-temp-state.fixture.ts
24-
- test/e2e/qa-lab/runtime/mcp-gateway-transport.e2e.test.ts
21+
- test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
22+
- extensions/qa-lab/src/gateway-child.ts
23+
- src/mcp/channel-bridge.ts
2524
execution:
26-
kind: vitest
27-
path: src/gateway/client.test.ts
28-
summary: Vitest coverage for Gateway connect request, protocol negotiation, and startup retry behavior used by MCP bridge clients.
25+
kind: script
26+
path: test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts
27+
summary: Starts the real MCP client before a delayed real Gateway becomes ready and captures retry, connect-frame, and negotiated-protocol evidence.
28+
args:
29+
- --scenario
30+
- mcp-gateway-connect-startup-retry
31+
- --artifact-base
32+
- ${outputDir}

0 commit comments

Comments
 (0)