Skip to content

Commit 20266c1

Browse files
steipetesteipete
committed
feat(qa-lab): add control ui qa-channel roundtrip scenario
1 parent f682413 commit 20266c1

12 files changed

Lines changed: 472 additions & 5 deletions

‎docs/concepts/qa-e2e-automation.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,11 @@ the source of truth for one test run and should define:
131131
- optional gateway config patch
132132
- the executable `qa-flow`
133133

134+
The reusable runtime surface that backs `qa-flow` is allowed to stay generic
135+
and cross-cutting. For example, markdown scenarios can combine transport-side
136+
helpers with browser-side helpers that drive the embedded Control UI through the
137+
Gateway `browser.request` seam without adding a special-case runner.
138+
134139
The baseline list should stay broad enough to cover:
135140

136141
- DM and channel chat

‎extensions/qa-lab/src/gateway-child.test.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,25 @@ describe("buildQaRuntimeEnv", () => {
115115
expect(env.OPENCLAW_STATE_DIR).toBe("/tmp/openclaw-qa/state");
116116
});
117117

118+
it("can forward host HOME for browser-backed QA runs while keeping OpenClaw home sandboxed", async () => {
119+
const hostHome = await mkdtemp(path.join(os.tmpdir(), "qa-host-home-"));
120+
cleanups.push(async () => {
121+
await rm(hostHome, { recursive: true, force: true });
122+
});
123+
124+
const env = buildQaRuntimeEnv({
125+
...createParams({
126+
HOME: hostHome,
127+
}),
128+
providerMode: "mock-openai",
129+
forwardHostHome: true,
130+
});
131+
132+
expect(env.HOME).toBe(hostHome);
133+
expect(env.OPENCLAW_HOME).toBe("/tmp/openclaw-qa/home");
134+
expect(env.OPENCLAW_STATE_DIR).toBe("/tmp/openclaw-qa/state");
135+
});
136+
118137
it("preserves the live Anthropic key for live Claude CLI runs without writing it into config", async () => {
119138
const hostHome = await mkdtemp(path.join(os.tmpdir(), "qa-host-home-"));
120139
cleanups.push(async () => {

‎extensions/qa-lab/src/gateway-child.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,7 @@ export function buildQaRuntimeEnv(params: {
295295
configPath: string;
296296
gatewayToken: string;
297297
homeDir: string;
298+
forwardHostHome?: boolean;
298299
stateDir: string;
299300
xdgConfigHome: string;
300301
xdgDataHome: string;
@@ -307,9 +308,12 @@ export function buildQaRuntimeEnv(params: {
307308
claudeCliAuthMode?: QaCliBackendAuthMode;
308309
}) {
309310
const baseEnv = params.baseEnv ?? process.env;
311+
const forwardedHostHome = params.forwardHostHome
312+
? baseEnv.HOME?.trim() || os.homedir()
313+
: undefined;
310314
const env: NodeJS.ProcessEnv = {
311315
...baseEnv,
312-
HOME: params.homeDir,
316+
HOME: forwardedHostHome ?? params.homeDir,
313317
...(params.providerMode === "live-frontier"
314318
? resolveQaLiveCliAuthEnv(baseEnv, {
315319
forwardHostHomeForClaudeCli: params.forwardHostHomeForClaudeCli,
@@ -837,6 +841,7 @@ export async function startQaGatewayChild(params: {
837841
claudeCliAuthMode?: QaCliBackendAuthMode;
838842
controlUiEnabled?: boolean;
839843
enabledPluginIds?: string[];
844+
forwardHostHome?: boolean;
840845
mutateConfig?: (cfg: OpenClawConfig) => OpenClawConfig;
841846
}) {
842847
const tempRoot = await fs.mkdtemp(
@@ -969,6 +974,7 @@ export async function startQaGatewayChild(params: {
969974
configPath,
970975
gatewayToken,
971976
homeDir,
977+
forwardHostHome: params.forwardHostHome,
972978
stateDir,
973979
xdgConfigHome,
974980
xdgDataHome,

‎extensions/qa-lab/src/scenario-catalog.test.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,12 @@ describe("qa scenario catalog", () => {
8181
expect(fanoutConfig?.expectedReplyGroups?.flat()).toContain("subagent-2: ok");
8282
});
8383

84+
it("loads scenario-declared gateway runtime options from markdown", () => {
85+
const scenario = readQaScenarioById("control-ui-qa-channel-image-roundtrip");
86+
87+
expect(scenario.gatewayRuntime?.forwardHostHome).toBe(true);
88+
});
89+
8490
it("keeps the character eval scenario natural and task-shaped", () => {
8591
const characterConfig = readQaScenarioExecutionConfig("character-vibes-gollum") as
8692
| {

‎extensions/qa-lab/src/scenario-catalog.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,10 @@ const qaScenarioExecutionSchema = z.object({
5151
config: qaScenarioConfigSchema.optional(),
5252
});
5353

54+
const qaScenarioGatewayRuntimeSchema = z.object({
55+
forwardHostHome: z.boolean().optional(),
56+
});
57+
5458
const qaFlowCallActionSchema = z.object({
5559
call: z.string().trim().min(1),
5660
args: z.array(z.unknown()).optional(),
@@ -137,6 +141,7 @@ const qaSeedScenarioSchema = z.object({
137141
successCriteria: z.array(z.string().trim().min(1)).min(1),
138142
plugins: z.array(z.string().trim().min(1)).optional(),
139143
gatewayConfigPatch: z.record(z.string(), z.unknown()).optional(),
144+
gatewayRuntime: qaScenarioGatewayRuntimeSchema.optional(),
140145
docsRefs: z.array(z.string().trim().min(1)).optional(),
141146
codeRefs: z.array(z.string().trim().min(1)).optional(),
142147
execution: qaScenarioExecutionSchema.optional(),

‎extensions/qa-lab/src/scenario-runtime-api.test.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,16 @@ function createDeps(overrides?: Partial<QaScenarioRuntimeDeps>): QaScenarioRunti
3030
waitForGatewayHealthy: fn,
3131
waitForTransportReady: fn,
3232
waitForQaChannelReady: fn,
33+
browserRequest: fn,
34+
waitForBrowserReady: fn,
35+
browserOpenTab: fn,
36+
browserSnapshot: fn,
37+
browserAct: fn,
38+
webOpenPage: fn,
39+
webWait: fn,
40+
webType: fn,
41+
webSnapshot: fn,
42+
webEvaluate: fn,
3343
waitForConfigRestartSettle: fn,
3444
patchConfig: fn,
3545
applyConfig: fn,
@@ -130,6 +140,16 @@ describe("createQaScenarioRuntimeApi", () => {
130140
expect(api.config).toEqual({ expected: "value" });
131141
expect(api.waitForCondition).toBe(waitForCondition);
132142
expect(api.waitForChannelReady).toBe(api.waitForTransportReady);
143+
expect(api.browserRequest).toBeDefined();
144+
expect(api.waitForBrowserReady).toBeDefined();
145+
expect(api.browserOpenTab).toBeDefined();
146+
expect(api.browserSnapshot).toBeDefined();
147+
expect(api.browserAct).toBeDefined();
148+
expect(api.webOpenPage).toBeDefined();
149+
expect(api.webWait).toBeDefined();
150+
expect(api.webType).toBeDefined();
151+
expect(api.webSnapshot).toBeDefined();
152+
expect(api.webEvaluate).toBeDefined();
133153
expect(api.getTransportSnapshot()).toEqual(state.getSnapshot());
134154
expect(api.imageUnderstandingPngBase64).toBe("png-small");
135155

‎extensions/qa-lab/src/scenario-runtime-api.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,16 @@ export type QaScenarioRuntimeDeps = {
4242
waitForGatewayHealthy: QaScenarioRuntimeFunction;
4343
waitForTransportReady: QaScenarioRuntimeFunction;
4444
waitForQaChannelReady: QaScenarioRuntimeFunction;
45+
browserRequest: QaScenarioRuntimeFunction;
46+
waitForBrowserReady: QaScenarioRuntimeFunction;
47+
browserOpenTab: QaScenarioRuntimeFunction;
48+
browserSnapshot: QaScenarioRuntimeFunction;
49+
browserAct: QaScenarioRuntimeFunction;
50+
webOpenPage: QaScenarioRuntimeFunction;
51+
webWait: QaScenarioRuntimeFunction;
52+
webType: QaScenarioRuntimeFunction;
53+
webSnapshot: QaScenarioRuntimeFunction;
54+
webEvaluate: QaScenarioRuntimeFunction;
4555
waitForConfigRestartSettle: QaScenarioRuntimeFunction;
4656
patchConfig: QaScenarioRuntimeFunction;
4757
applyConfig: QaScenarioRuntimeFunction;
@@ -116,6 +126,16 @@ export type QaScenarioRuntimeApi<
116126
waitForTransportReady: TDeps["waitForTransportReady"];
117127
waitForChannelReady: TDeps["waitForTransportReady"];
118128
waitForQaChannelReady: TDeps["waitForQaChannelReady"];
129+
browserRequest: TDeps["browserRequest"];
130+
waitForBrowserReady: TDeps["waitForBrowserReady"];
131+
browserOpenTab: TDeps["browserOpenTab"];
132+
browserSnapshot: TDeps["browserSnapshot"];
133+
browserAct: TDeps["browserAct"];
134+
webOpenPage: TDeps["webOpenPage"];
135+
webWait: TDeps["webWait"];
136+
webType: TDeps["webType"];
137+
webSnapshot: TDeps["webSnapshot"];
138+
webEvaluate: TDeps["webEvaluate"];
119139
waitForConfigRestartSettle: TDeps["waitForConfigRestartSettle"];
120140
patchConfig: TDeps["patchConfig"];
121141
applyConfig: TDeps["applyConfig"];
@@ -205,6 +225,16 @@ export function createQaScenarioRuntimeApi<
205225
waitForTransportReady: params.deps.waitForTransportReady,
206226
waitForChannelReady: params.deps.waitForTransportReady,
207227
waitForQaChannelReady: params.deps.waitForQaChannelReady,
228+
browserRequest: params.deps.browserRequest,
229+
waitForBrowserReady: params.deps.waitForBrowserReady,
230+
browserOpenTab: params.deps.browserOpenTab,
231+
browserSnapshot: params.deps.browserSnapshot,
232+
browserAct: params.deps.browserAct,
233+
webOpenPage: params.deps.webOpenPage,
234+
webWait: params.deps.webWait,
235+
webType: params.deps.webType,
236+
webSnapshot: params.deps.webSnapshot,
237+
webEvaluate: params.deps.webEvaluate,
208238
waitForConfigRestartSettle: params.deps.waitForConfigRestartSettle,
209239
patchConfig: params.deps.patchConfig,
210240
applyConfig: params.deps.applyConfig,

‎extensions/qa-lab/src/suite.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ describe("qa suite failure reply handling", () => {
1111
config?: Record<string, unknown>,
1212
plugins?: string[],
1313
gatewayConfigPatch?: Record<string, unknown>,
14+
gatewayRuntime?: { forwardHostHome?: boolean },
1415
): Parameters<typeof qaSuiteTesting.selectQaSuiteScenarios>[0]["scenarios"][number] =>
1516
({
1617
id,
@@ -20,6 +21,7 @@ describe("qa suite failure reply handling", () => {
2021
successCriteria: ["test"],
2122
plugins,
2223
gatewayConfigPatch,
24+
gatewayRuntime,
2325
sourcePath: `qa/scenarios/${id}.md`,
2426
execution: {
2527
kind: "flow",
@@ -199,6 +201,19 @@ describe("qa suite failure reply handling", () => {
199201
});
200202
});
201203

204+
it("collects gateway runtime options across selected scenarios", () => {
205+
const scenarios = [
206+
makeScenario("plain"),
207+
makeScenario("browser-ui", undefined, ["browser"], undefined, {
208+
forwardHostHome: true,
209+
}),
210+
];
211+
212+
expect(qaSuiteTesting.collectQaSuiteGatewayRuntimeOptions(scenarios)).toEqual({
213+
forwardHostHome: true,
214+
});
215+
});
216+
202217
it("filters provider-specific scenarios from an implicit live lane", () => {
203218
const scenarios = [
204219
makeScenario("generic"),

‎extensions/qa-lab/src/suite.ts‎

Lines changed: 44 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,13 @@ import {
1515
import { buildAgentSessionKey } from "openclaw/plugin-sdk/routing";
1616
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
1717
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
18+
import {
19+
callQaBrowserRequest,
20+
qaBrowserAct,
21+
qaBrowserOpenTab,
22+
qaBrowserSnapshot,
23+
waitForQaBrowserReady,
24+
} from "./browser-runtime.js";
1825
import { ensureRepoBoundDirectory, resolveRepoRelativeOutputDir } from "./cli-paths.js";
1926
import { waitForCronRunCompletion } from "./cron-run-wait.js";
2027
import {
@@ -60,6 +67,14 @@ import { qaChannelPlugin, type QaBusMessage } from "./runtime-api.js";
6067
import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js";
6168
import { runScenarioFlow } from "./scenario-flow-runner.js";
6269
import { createQaScenarioRuntimeApi } from "./scenario-runtime-api.js";
70+
import {
71+
closeAllQaWebSessions,
72+
qaWebEvaluate,
73+
qaWebOpenPage,
74+
qaWebSnapshot,
75+
qaWebType,
76+
qaWebWait,
77+
} from "./web-runtime.js";
6378

6479
type QaSuiteStep = {
6580
name: string;
@@ -313,6 +328,18 @@ function collectQaSuiteGatewayConfigPatch(
313328
return merged;
314329
}
315330

331+
function collectQaSuiteGatewayRuntimeOptions(
332+
scenarios: ReturnType<typeof readQaBootstrapScenarioCatalog>["scenarios"],
333+
) {
334+
let forwardHostHome = false;
335+
for (const scenario of scenarios) {
336+
if (scenario.gatewayRuntime?.forwardHostHome === true) {
337+
forwardHostHome = true;
338+
}
339+
}
340+
return forwardHostHome ? { forwardHostHome: true } : undefined;
341+
}
342+
316343
function liveTurnTimeoutMs(env: QaSuiteEnvironment, fallbackMs: number) {
317344
return resolveQaLiveTurnTimeoutMs(env, fallbackMs);
318345
}
@@ -1236,6 +1263,16 @@ function createScenarioFlowApi(
12361263
waitForGatewayHealthy,
12371264
waitForTransportReady,
12381265
waitForQaChannelReady,
1266+
browserRequest: callQaBrowserRequest,
1267+
waitForBrowserReady: waitForQaBrowserReady,
1268+
browserOpenTab: qaBrowserOpenTab,
1269+
browserSnapshot: qaBrowserSnapshot,
1270+
browserAct: qaBrowserAct,
1271+
webOpenPage: qaWebOpenPage,
1272+
webWait: qaWebWait,
1273+
webType: qaWebType,
1274+
webSnapshot: qaWebSnapshot,
1275+
webEvaluate: qaWebEvaluate,
12391276
waitForConfigRestartSettle,
12401277
patchConfig,
12411278
applyConfig,
@@ -1284,6 +1321,7 @@ function createScenarioFlowApi(
12841321

12851322
export const qaSuiteTesting = {
12861323
collectQaSuiteGatewayConfigPatch,
1324+
collectQaSuiteGatewayRuntimeOptions,
12871325
collectQaSuitePluginIds,
12881326
createScenarioWaitForCondition,
12891327
findFailureOutboundMessage,
@@ -1397,6 +1435,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
13971435
});
13981436
const enabledPluginIds = collectQaSuitePluginIds(selectedCatalogScenarios);
13991437
const gatewayConfigPatch = collectQaSuiteGatewayConfigPatch(selectedCatalogScenarios);
1438+
const gatewayRuntimeOptions = collectQaSuiteGatewayRuntimeOptions(selectedCatalogScenarios);
14001439
const concurrency = normalizeQaSuiteConcurrency(
14011440
params?.concurrency,
14021441
selectedCatalogScenarios.length,
@@ -1594,6 +1633,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
15941633
claudeCliAuthMode: params?.claudeCliAuthMode,
15951634
controlUiEnabled: params?.controlUiEnabled ?? true,
15961635
enabledPluginIds,
1636+
forwardHostHome: gatewayRuntimeOptions?.forwardHostHome,
15971637
mutateConfig: gatewayConfigPatch
15981638
? (cfg) => applyQaMergePatch(cfg, gatewayConfigPatch) as OpenClawConfig
15991639
: undefined,
@@ -1606,9 +1646,9 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
16061646
lab,
16071647
mock,
16081648
gateway,
1609-
cfg: transport.createGatewayConfig({
1610-
baseUrl: lab.listenUrl,
1611-
}),
1649+
// Markdown scenarios should see the full staged gateway config, not just
1650+
// the transport fragment. Routing/session/plugin assertions depend on it.
1651+
cfg: gateway.cfg,
16121652
transport,
16131653
repoRoot,
16141654
providerMode,
@@ -1717,6 +1757,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
17171757
preserveGatewayRuntimeDir = path.join(outputDir, "artifacts", "gateway-runtime");
17181758
throw error;
17191759
} finally {
1760+
await closeAllQaWebSessions();
17201761
const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1" || false;
17211762
await gateway.stop({
17221763
keepTemp,

0 commit comments

Comments
 (0)