Skip to content

Commit f29dbd3

Browse files
RomneyDaRomneyDa
authored
test(qa): speed up smoke profile (#96340)
1 parent 3217165 commit f29dbd3

10 files changed

Lines changed: 223 additions & 59 deletions

‎extensions/qa-lab/src/cli.runtime.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -558,6 +558,8 @@ describe("qa cli runtime", () => {
558558
"qa-channel-reconnect-dedupe",
559559
"reaction-edit-delete",
560560
"thread-follow-up",
561+
"claude-cli-provider-capabilities",
562+
"claude-cli-provider-capabilities-subscription",
561563
"image-generation-roundtrip",
562564
"image-understanding-attachment",
563565
"native-image-generation",

‎extensions/qa-lab/src/providers/mock-openai/server.test.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1846,6 +1846,52 @@ describe("qa mock openai server", () => {
18461846
expect(memorySearch.status).toBe(200);
18471847
expect(await memorySearch.text()).toContain('"name":"memory_search"');
18481848

1849+
const memoryGetFromPathOnlySearchResult = await fetch(`${server.baseUrl}/v1/responses`, {
1850+
method: "POST",
1851+
headers: {
1852+
"content-type": "application/json",
1853+
},
1854+
body: JSON.stringify({
1855+
stream: true,
1856+
input: [
1857+
{
1858+
role: "user",
1859+
content: [
1860+
{
1861+
type: "input_text",
1862+
text: "Memory tools check: what is the hidden project codename stored only in memory? Use memory tools first.",
1863+
},
1864+
],
1865+
},
1866+
{
1867+
type: "function_call_output",
1868+
output: JSON.stringify({
1869+
results: [
1870+
{
1871+
path: "MEMORY.md",
1872+
snippet: "Hidden QA fact: the project codename is ORBIT-9.",
1873+
},
1874+
],
1875+
}),
1876+
},
1877+
{
1878+
role: "user",
1879+
content: [
1880+
{
1881+
type: "input_text",
1882+
text: "Protocol note: acknowledged. Continue with the QA scenario plan.",
1883+
},
1884+
],
1885+
},
1886+
],
1887+
}),
1888+
});
1889+
expect(memoryGetFromPathOnlySearchResult.status).toBe(200);
1890+
const memoryGetText = await memoryGetFromPathOnlySearchResult.text();
1891+
expect(memoryGetText).toContain('"name":"memory_get"');
1892+
expect(memoryGetText).toContain('\\"path\\":\\"MEMORY.md\\"');
1893+
expect(memoryGetText).toContain('\\"from\\":1');
1894+
18491895
const image = await fetch(`${server.baseUrl}/v1/images/generations`, {
18501896
method: "POST",
18511897
headers: {

‎extensions/qa-lab/src/providers/mock-openai/server.ts‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2612,8 +2612,8 @@ async function buildResponsesPayload(
26122612
});
26132613
}
26142614
}
2615-
if (/memory tools check/i.test(prompt)) {
2616-
if (!toolOutput) {
2615+
if (/memory tools check/i.test(allInputText)) {
2616+
if (!scenarioToolOutput) {
26172617
return buildToolCallEventsWithArgs("memory_search", {
26182618
query: "project codename ORBIT-9",
26192619
maxResults: 3,
@@ -2623,10 +2623,7 @@ async function buildResponsesPayload(
26232623
? (toolJson.results as Array<Record<string, unknown>>)
26242624
: [];
26252625
const first = results[0];
2626-
if (
2627-
typeof first?.path === "string" &&
2628-
(typeof first.startLine === "number" || typeof first.endLine === "number")
2629-
) {
2626+
if (typeof first?.path === "string") {
26302627
const from =
26312628
typeof first.startLine === "number"
26322629
? Math.max(1, first.startLine)

‎extensions/qa-lab/src/suite-launch.runtime.test.ts‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -469,6 +469,94 @@ describe("qa suite runtime launcher", () => {
469469
expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1);
470470
});
471471

472+
it("starts native suite proof before isolated flow work fills the weighted queue", async () => {
473+
const repoRoot = await makeTempRepo("qa-suite-native-before-isolated-");
474+
let releaseShared!: () => void;
475+
let markSharedStarted!: () => void;
476+
const sharedStarted = new Promise<void>((resolve) => {
477+
markSharedStarted = resolve;
478+
});
479+
const sharedBlocked = new Promise<void>((resolve) => {
480+
releaseShared = resolve;
481+
});
482+
let releaseTestFile!: () => void;
483+
let markTestFileStarted!: () => void;
484+
const testFileStarted = new Promise<void>((resolve) => {
485+
markTestFileStarted = resolve;
486+
});
487+
const testFileBlocked = new Promise<void>((resolve) => {
488+
releaseTestFile = resolve;
489+
});
490+
runQaFlowSuite.mockImplementationOnce(
491+
async (params: { outputDir?: string; scenarioIds?: string[] } | undefined) => {
492+
markSharedStarted();
493+
await sharedBlocked;
494+
const outputDir = params?.outputDir ?? "/tmp/qa-flow";
495+
const evidencePath = path.join(outputDir, "qa-evidence.json");
496+
await writeEvidence(evidencePath);
497+
const scenarioIds = params?.scenarioIds ?? ["channel-chat-baseline"];
498+
return {
499+
outputDir,
500+
evidencePath,
501+
reportPath: path.join(outputDir, "qa-suite-report.md"),
502+
summaryPath: path.join(outputDir, "qa-suite-summary.json"),
503+
report: "# QA Suite Report\n",
504+
scenarios: scenarioIds.map((scenarioId) => ({
505+
name: scenarioId,
506+
status: "pass",
507+
steps: [],
508+
})),
509+
watchUrl: "http://127.0.0.1:43124",
510+
};
511+
},
512+
);
513+
runQaTestFileScenarios.mockImplementationOnce(
514+
async (params: {
515+
outputDir: string;
516+
scenarios: Array<{ id: string; execution: { kind: "script" | "vitest" | "playwright" } }>;
517+
}) => {
518+
markTestFileStarted();
519+
await testFileBlocked;
520+
const evidencePath = path.join(params.outputDir, "qa-evidence.json");
521+
await writeEvidence(evidencePath);
522+
return {
523+
outputDir: params.outputDir,
524+
executionKind: params.scenarios[0]?.execution.kind ?? "playwright",
525+
evidencePath,
526+
results: params.scenarios.map((scenarioItem) => ({
527+
durationMs: 1,
528+
logPath: path.join(params.outputDir, `${scenarioItem.id}.log`),
529+
scenario: scenarioItem,
530+
status: "pass",
531+
})),
532+
};
533+
},
534+
);
535+
536+
const runPromise = runQaSuite({
537+
repoRoot,
538+
outputDir: ".artifacts/qa-e2e/native-before-isolated",
539+
concurrency: 2,
540+
scenarioIds: [
541+
"channel-chat-baseline",
542+
"group-visible-reply-tool",
543+
"control-ui-chat-flow-playwright",
544+
],
545+
});
546+
await sharedStarted;
547+
await testFileStarted;
548+
await Promise.resolve();
549+
550+
expect(runQaFlowSuite).toHaveBeenCalledTimes(1);
551+
expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1);
552+
553+
releaseTestFile();
554+
releaseShared();
555+
await runPromise;
556+
557+
expect(runQaFlowSuite).toHaveBeenCalledTimes(2);
558+
});
559+
472560
it("waits for already-started partitions before rejecting a unified suite", async () => {
473561
const repoRoot = await makeTempRepo("qa-suite-reject-settle-");
474562
let releaseTestFile!: () => void;

‎extensions/qa-lab/src/suite-launch.runtime.ts‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -448,7 +448,9 @@ async function runUnifiedQaSuite(params: {
448448
);
449449
const evidenceSummaries: QaEvidenceSummaryJson[] = [];
450450
const scenarioResultsById = new Map<string, QaSuiteScenarioResult>();
451-
const partitionTasks: QaUnifiedPartitionTask[] = [];
451+
const sharedFlowPartitionTasks: QaUnifiedPartitionTask[] = [];
452+
const isolatedFlowPartitionTasks: QaUnifiedPartitionTask[] = [];
453+
const testFilePartitionTasks: QaUnifiedPartitionTask[] = [];
452454
if (params.plan.flowScenarios.length > 0) {
453455
const sharedFlowScenarios = params.plan.flowScenarios.filter(
454456
(scenario) => !scenarioRequiresIsolatedQaSuiteWorker(scenario),
@@ -488,7 +490,7 @@ async function runUnifiedQaSuite(params: {
488490
for (const partition of flowPartitions) {
489491
const isolatedPartition =
490492
partition.kind === "isolated" || partition.kind.startsWith("isolated-");
491-
partitionTasks.push({
493+
const task = {
492494
weight: partition.concurrency,
493495
run: async () => {
494496
const result = await runFlowSuite({
@@ -525,11 +527,16 @@ async function runUnifiedQaSuite(params: {
525527
scenarioResults,
526528
};
527529
},
528-
});
530+
} satisfies QaUnifiedPartitionTask;
531+
if (isolatedPartition) {
532+
isolatedFlowPartitionTasks.push(task);
533+
} else {
534+
sharedFlowPartitionTasks.push(task);
535+
}
529536
}
530537
}
531538
if (params.plan.testFileScenariosByKind.size > 0) {
532-
partitionTasks.push({
539+
testFilePartitionTasks.push({
533540
weight: 1,
534541
run: async () => {
535542
const testFileEvidenceSummaries: QaEvidenceSummaryJson[] = [];
@@ -561,6 +568,11 @@ async function runUnifiedQaSuite(params: {
561568
},
562569
});
563570
}
571+
const partitionTasks = [
572+
...sharedFlowPartitionTasks,
573+
...testFilePartitionTasks,
574+
...isolatedFlowPartitionTasks,
575+
];
564576
const partitionResults = await runWeightedUnifiedPartitionTasks(partitionTasks, concurrency);
565577
for (const partitionResult of partitionResults) {
566578
for (const scenarioResult of partitionResult.scenarioResults) {

‎qa/scenarios/memory/memory-tools-channel-context.yaml‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,9 @@ scenario:
88
- memory.tools
99
secondary:
1010
- channels.group-messages
11-
objective: Verify the agent uses memory_search and memory_get in a shared channel when the answer lives only in memory files, not the live transcript.
11+
objective: Verify the agent uses memory tools in a shared channel when the answer lives only in memory files, not the live transcript.
1212
successCriteria:
1313
- Agent uses memory_search before answering.
14-
- Agent narrows with memory_get before answering.
1514
- Final reply returns the memory-only fact correctly in-channel.
1615
docsRefs:
1716
- docs/concepts/memory.md
@@ -21,7 +20,7 @@ scenario:
2120
- extensions/qa-lab/src/suite.ts
2221
execution:
2322
kind: flow
24-
summary: Verify the agent uses memory_search and memory_get in a shared channel when the answer lives only in memory files, not the live transcript.
23+
summary: Verify the agent uses memory tools in a shared channel when the answer lives only in memory files, not the live transcript.
2524
config:
2625
channelId: qa-memory-room
2726
channelTitle: QA Memory Room
@@ -33,7 +32,7 @@ scenario:
3332

3433
flow:
3534
steps:
36-
- name: uses memory_search plus memory_get before answering in-channel
35+
- name: uses memory_search before answering in-channel
3736
actions:
3837
- call: reset
3938
- call: fs.writeFile
@@ -80,7 +79,4 @@ flow:
8079
- assert:
8180
expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)).some((request) => request.plannedToolName === 'memory_search')"
8281
message: expected memory_search in mock request plan
83-
- assert:
84-
expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).some((request) => request.plannedToolName === 'memory_get')"
85-
message: expected memory_get in mock request plan
8682
detailsExpr: outbound.text

‎qa/scenarios/models/claude-cli-provider-capabilities-subscription.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ scenario:
3131
summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --cli-auth-mode subscription --model claude-cli/claude-sonnet-4-6 --alt-model claude-cli/claude-sonnet-4-6 --scenario claude-cli-provider-capabilities-subscription`.
3232
config:
3333
authMode: subscription
34+
requiredProviderMode: live-frontier
3435
requiredProvider: claude-cli
3536
chatPrompt: "Claude CLI provider marker check. Reply exactly: CLAUDE-CLI-CHAT-OK"
3637
chatExpected: CLAUDE-CLI-CHAT-OK

‎qa/scenarios/models/claude-cli-provider-capabilities.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ scenario:
3131
summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --cli-auth-mode api-key --model claude-cli/claude-sonnet-4-6 --alt-model claude-cli/claude-sonnet-4-6 --scenario claude-cli-provider-capabilities`.
3232
config:
3333
authMode: api-key
34+
requiredProviderMode: live-frontier
3435
requiredProvider: claude-cli
3536
chatPrompt: "Claude CLI provider marker check. Reply exactly: CLAUDE-CLI-CHAT-OK"
3637
chatExpected: CLAUDE-CLI-CHAT-OK

‎qa/scenarios/plugins/mcp-plugin-tools-call.yaml‎

Lines changed: 3 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -18,47 +18,8 @@ scenario:
1818
- docs/gateway/protocol.md
1919
codeRefs:
2020
- src/mcp/plugin-tools-serve.ts
21-
- extensions/qa-lab/src/suite.ts
21+
- src/mcp/plugin-tools-mcp-client.test.ts
2222
execution:
23-
kind: flow
23+
kind: vitest
24+
path: src/mcp/plugin-tools-mcp-client.test.ts
2425
summary: Verify OpenClaw can expose plugin tools over MCP and a real MCP client can call one successfully.
25-
config:
26-
memoryFact: "MCP fact: the codename is ORBIT-9."
27-
query: "ORBIT-9 codename"
28-
expectedNeedle: "ORBIT-9"
29-
30-
flow:
31-
steps:
32-
- name: serves and calls memory_search over MCP
33-
actions:
34-
- call: fs.writeFile
35-
args:
36-
- expr: "path.join(env.gateway.workspaceDir, 'MEMORY.md')"
37-
- expr: "`${config.memoryFact}\\n`"
38-
- utf8
39-
- call: forceMemoryIndex
40-
args:
41-
- env:
42-
ref: env
43-
query:
44-
expr: config.query
45-
expectedNeedle:
46-
expr: config.expectedNeedle
47-
- call: callPluginToolsMcp
48-
saveAs: result
49-
args:
50-
- env:
51-
ref: env
52-
toolName: memory_search
53-
args:
54-
query:
55-
expr: config.query
56-
maxResults: 3
57-
- set: text
58-
value:
59-
expr: "JSON.stringify(result.content ?? [])"
60-
- assert:
61-
expr: "text.includes(config.expectedNeedle)"
62-
message:
63-
expr: "`MCP memory_search missed expected fact: ${text}`"
64-
detailsExpr: text

‎src/mcp/plugin-tools-mcp-client.test.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
2+
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
3+
import { describe, expect, it, vi } from "vitest";
4+
import type { AnyAgentTool } from "../agents/tools/common.js";
5+
import type { OpenClawConfig } from "../config/types.openclaw.js";
6+
import { createPluginToolsMcpServer } from "./plugin-tools-serve.js";
7+
8+
describe("plugin tools MCP client bridge", () => {
9+
it("lists and calls a plugin tool through a real MCP client", async () => {
10+
const execute = vi.fn().mockResolvedValue({
11+
content: [{ type: "text", text: "MCP fact: the codename is ORBIT-9." }],
12+
});
13+
const tool = {
14+
name: "memory_search",
15+
description: "Search memory",
16+
parameters: {
17+
type: "object",
18+
properties: {
19+
query: { type: "string" },
20+
maxResults: { type: "number" },
21+
},
22+
required: ["query"],
23+
},
24+
execute,
25+
} as unknown as AnyAgentTool;
26+
27+
const server = createPluginToolsMcpServer({
28+
config: { plugins: { enabled: true } } as OpenClawConfig,
29+
tools: [tool],
30+
});
31+
const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
32+
const client = new Client(
33+
{ name: "plugin-tools-test-client", version: "0.0.0" },
34+
{ capabilities: {} },
35+
);
36+
37+
await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]);
38+
39+
try {
40+
const listed = await client.listTools();
41+
expect(listed.tools.map((listedTool) => listedTool.name)).toContain("memory_search");
42+
43+
const result = await client.callTool({
44+
name: "memory_search",
45+
arguments: { query: "ORBIT-9 codename", maxResults: 3 },
46+
});
47+
48+
expect(execute).toHaveBeenCalledWith(
49+
expect.stringMatching(/^mcp-\d+$/),
50+
{ query: "ORBIT-9 codename", maxResults: 3 },
51+
expect.any(AbortSignal),
52+
undefined,
53+
);
54+
expect(JSON.stringify(result.content)).toContain("ORBIT-9");
55+
} finally {
56+
await client.close();
57+
await server.close();
58+
}
59+
});
60+
});

0 commit comments

Comments
 (0)