Skip to content

Commit 843ad14

Browse files
RomneyDaRomneyDa
authored
feat(qa): add transport scenario requirements (#97915)
1 parent 15de9d8 commit 843ad14

8 files changed

Lines changed: 334 additions & 13 deletions

‎extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,19 @@ export {
1515
type LiveTransportScenarioDefinition,
1616
type LiveTransportStandardScenarioId,
1717
} from "openclaw/plugin-sdk/qa-live-transport-scenarios";
18+
export {
19+
assertQaTransportSupportsScenario,
20+
defineQaTransportScenario,
21+
filterQaTransportScenariosForTransport,
22+
findUnsupportedQaTransportScenarioRequirements,
23+
mergeQaTransportScenarioRequirements,
24+
qaTransportRequirementsForStandardScenario,
25+
type QaTransportCapabilityName,
26+
type QaTransportScenarioDefinition,
27+
type QaTransportScenarioDefinitionInput,
28+
type QaTransportScenarioRequirements,
29+
type QaTransportScenarioUnsupportedRequirements,
30+
} from "../../qa-transport-scenarios.js";
1831

1932
export type LiveTransportCoverageMember = {
2033
scenarioId?: string;

‎extensions/qa-lab/src/qa-channel-transport.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ class QaChannelTransport extends QaStateBackedTransportAdapter {
135135
label: "qa-channel + qa-lab bus",
136136
accountId: QA_CHANNEL_ACCOUNT_ID,
137137
requiredPluginIds: QA_CHANNEL_REQUIRED_PLUGIN_IDS,
138+
supportedActions: ["delete", "edit", "react", "thread-create"],
138139
state,
139140
});
140141
}

‎extensions/qa-lab/src/qa-transport-scenarios.test.ts‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
// Qa Lab tests cover transport-backed scenario metadata.
2+
import { describe, expect, it } from "vitest";
3+
import { createQaBusState } from "./bus-state.js";
4+
import { createQaChannelTransport } from "./qa-channel-transport.js";
5+
import {
6+
assertQaTransportSupportsScenario,
7+
defineQaTransportScenario,
8+
filterQaTransportScenariosForTransport,
9+
findUnsupportedQaTransportScenarioRequirements,
10+
mergeQaTransportScenarioRequirements,
11+
qaTransportRequirementsForStandardScenario,
12+
} from "./qa-transport-scenarios.js";
13+
import type { QaTransportAdapter } from "./qa-transport.js";
14+
15+
type ScenarioTransportFixture = Pick<
16+
QaTransportAdapter,
17+
"capabilities" | "id" | "supportedActions"
18+
>;
19+
20+
function createScenarioTransportFixture(
21+
supportedActions: QaTransportAdapter["supportedActions"],
22+
): ScenarioTransportFixture {
23+
const transport = createQaChannelTransport(createQaBusState());
24+
return {
25+
capabilities: transport.capabilities,
26+
id: "crabline",
27+
supportedActions,
28+
};
29+
}
30+
31+
describe("qa transport scenarios", () => {
32+
it("maps standard live transport buckets onto existing transport capabilities", () => {
33+
expect(qaTransportRequirementsForStandardScenario("canary")).toEqual({
34+
capabilities: ["assertNoFailureReplies", "sendInboundMessage", "waitForOutboundMessage"],
35+
});
36+
expect(qaTransportRequirementsForStandardScenario("restart-resume")).toEqual({
37+
capabilities: [
38+
"assertNoFailureReplies",
39+
"sendInboundMessage",
40+
"waitForOutboundMessage",
41+
"waitForReady",
42+
],
43+
});
44+
});
45+
46+
it("merges standard and explicit requirements without duplicating names", () => {
47+
const scenario = defineQaTransportScenario({
48+
id: "slack-thread-follow-up",
49+
standardId: "thread-follow-up",
50+
timeoutMs: 30_000,
51+
title: "Thread follow-up",
52+
transportRequirements: {
53+
actions: ["thread-create", "thread-create"],
54+
capabilities: ["waitForOutboundMessage", "waitForCondition"],
55+
},
56+
});
57+
58+
expect(scenario.transportRequirements).toEqual({
59+
actions: ["thread-create"],
60+
capabilities: [
61+
"assertNoFailureReplies",
62+
"sendInboundMessage",
63+
"waitForOutboundMessage",
64+
"waitForCondition",
65+
],
66+
});
67+
});
68+
69+
it("can merge requirements independently for shared scenario builders", () => {
70+
expect(
71+
mergeQaTransportScenarioRequirements([
72+
{ actions: ["react"], capabilities: ["sendInboundMessage"] },
73+
{ actions: ["react", "delete"], capabilities: ["sendInboundMessage", "waitForReady"] },
74+
]),
75+
).toEqual({
76+
actions: ["react", "delete"],
77+
capabilities: ["sendInboundMessage", "waitForReady"],
78+
});
79+
});
80+
81+
it("validates a scenario against a real qa-channel transport", () => {
82+
const transport = createQaChannelTransport(createQaBusState());
83+
const scenario = defineQaTransportScenario({
84+
id: "qa-channel-reaction",
85+
timeoutMs: 30_000,
86+
title: "Reaction action",
87+
transportRequirements: {
88+
actions: ["react"],
89+
capabilities: ["executeGenericAction", "waitForCondition"],
90+
},
91+
});
92+
93+
expect(() => assertQaTransportSupportsScenario({ scenario, transport })).not.toThrow();
94+
});
95+
96+
it("reports unsupported transport actions without executing the action", () => {
97+
const transport = createScenarioTransportFixture([]);
98+
const scenario = defineQaTransportScenario({
99+
id: "reaction-action",
100+
timeoutMs: 30_000,
101+
title: "Reaction action",
102+
transportRequirements: {
103+
actions: ["react"],
104+
capabilities: ["executeGenericAction"],
105+
},
106+
});
107+
108+
expect(findUnsupportedQaTransportScenarioRequirements({ scenario, transport })).toEqual({
109+
actions: ["react"],
110+
capabilities: [],
111+
});
112+
expect(() => assertQaTransportSupportsScenario({ scenario, transport })).toThrow(
113+
"QA transport crabline cannot run scenario reaction-action; unsupported actions: react",
114+
);
115+
});
116+
117+
it("filters scenarios by the supplied transport contract", () => {
118+
const transport = createScenarioTransportFixture([]);
119+
const canary = defineQaTransportScenario({
120+
id: "canary",
121+
standardId: "canary",
122+
timeoutMs: 30_000,
123+
title: "Canary",
124+
});
125+
const reaction = defineQaTransportScenario({
126+
id: "reaction",
127+
timeoutMs: 30_000,
128+
title: "Reaction",
129+
transportRequirements: { actions: ["react"] },
130+
});
131+
132+
expect(
133+
filterQaTransportScenariosForTransport({
134+
scenarios: [canary, reaction],
135+
transport,
136+
}),
137+
).toEqual([canary]);
138+
});
139+
});

‎extensions/qa-lab/src/qa-transport-scenarios.ts‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
// Qa Lab plugin module defines scenario metadata for transport-backed QA runs.
2+
import type {
3+
LiveTransportScenarioDefinition,
4+
LiveTransportStandardScenarioId,
5+
} from "openclaw/plugin-sdk/qa-live-transport-scenarios";
6+
import type {
7+
QaTransportActionName,
8+
QaTransportAdapter,
9+
QaTransportCapabilities,
10+
} from "./qa-transport.js";
11+
12+
export type QaTransportCapabilityName = keyof QaTransportCapabilities;
13+
14+
export type QaTransportScenarioRequirements = {
15+
actions?: readonly QaTransportActionName[];
16+
capabilities?: readonly QaTransportCapabilityName[];
17+
};
18+
19+
export type QaTransportScenarioDefinition<TId extends string = string> =
20+
LiveTransportScenarioDefinition<TId> & {
21+
transportRequirements: QaTransportScenarioRequirements;
22+
};
23+
24+
export type QaTransportScenarioDefinitionInput<TId extends string = string> =
25+
LiveTransportScenarioDefinition<TId> & {
26+
transportRequirements?: QaTransportScenarioRequirements;
27+
};
28+
29+
export type QaTransportScenarioUnsupportedRequirements = {
30+
actions: QaTransportActionName[];
31+
capabilities: QaTransportCapabilityName[];
32+
};
33+
34+
const TEXT_REPLY_CAPABILITIES = [
35+
"assertNoFailureReplies",
36+
"sendInboundMessage",
37+
"waitForOutboundMessage",
38+
] as const satisfies readonly QaTransportCapabilityName[];
39+
40+
const NO_REPLY_CAPABILITIES = [
41+
"assertNoFailureReplies",
42+
"sendInboundMessage",
43+
"waitForCondition",
44+
] as const satisfies readonly QaTransportCapabilityName[];
45+
46+
const STANDARD_SCENARIO_REQUIREMENTS = {
47+
canary: { capabilities: TEXT_REPLY_CAPABILITIES },
48+
"mention-gating": { capabilities: NO_REPLY_CAPABILITIES },
49+
"allowlist-block": { capabilities: NO_REPLY_CAPABILITIES },
50+
"top-level-reply-shape": { capabilities: TEXT_REPLY_CAPABILITIES },
51+
"quote-reply": { capabilities: TEXT_REPLY_CAPABILITIES },
52+
"restart-resume": {
53+
capabilities: [...TEXT_REPLY_CAPABILITIES, "waitForReady"],
54+
},
55+
"thread-follow-up": { capabilities: TEXT_REPLY_CAPABILITIES },
56+
"thread-isolation": {
57+
capabilities: [...TEXT_REPLY_CAPABILITIES, "waitForCondition"],
58+
},
59+
"reaction-observation": {
60+
capabilities: ["getNormalizedMessageState", "waitForCondition"],
61+
},
62+
"help-command": { capabilities: TEXT_REPLY_CAPABILITIES },
63+
} as const satisfies Record<LiveTransportStandardScenarioId, QaTransportScenarioRequirements>;
64+
65+
function uniqueInOrder<T extends string>(values: readonly T[]) {
66+
const seen = new Set<T>();
67+
const unique: T[] = [];
68+
for (const value of values) {
69+
if (!seen.has(value)) {
70+
seen.add(value);
71+
unique.push(value);
72+
}
73+
}
74+
return unique;
75+
}
76+
77+
export function mergeQaTransportScenarioRequirements(
78+
requirements: readonly QaTransportScenarioRequirements[],
79+
): QaTransportScenarioRequirements {
80+
return {
81+
actions: uniqueInOrder(requirements.flatMap((requirement) => requirement.actions ?? [])),
82+
capabilities: uniqueInOrder(
83+
requirements.flatMap((requirement) => requirement.capabilities ?? []),
84+
),
85+
};
86+
}
87+
88+
export function qaTransportRequirementsForStandardScenario(
89+
standardId: LiveTransportStandardScenarioId,
90+
): QaTransportScenarioRequirements {
91+
return STANDARD_SCENARIO_REQUIREMENTS[standardId];
92+
}
93+
94+
export function defineQaTransportScenario<TId extends string>(
95+
input: QaTransportScenarioDefinitionInput<TId>,
96+
): QaTransportScenarioDefinition<TId> {
97+
const standardRequirements = input.standardId
98+
? qaTransportRequirementsForStandardScenario(input.standardId)
99+
: {};
100+
return {
101+
...input,
102+
transportRequirements: mergeQaTransportScenarioRequirements([
103+
standardRequirements,
104+
input.transportRequirements ?? {},
105+
]),
106+
};
107+
}
108+
109+
export function findUnsupportedQaTransportScenarioRequirements(params: {
110+
scenario: QaTransportScenarioDefinition;
111+
transport: Pick<QaTransportAdapter, "capabilities" | "supportedActions">;
112+
}): QaTransportScenarioUnsupportedRequirements {
113+
const supportedActions = new Set(params.transport.supportedActions);
114+
return {
115+
actions: (params.scenario.transportRequirements.actions ?? []).filter(
116+
(action) => !supportedActions.has(action),
117+
),
118+
capabilities: (params.scenario.transportRequirements.capabilities ?? []).filter(
119+
(capability) => typeof params.transport.capabilities[capability] !== "function",
120+
),
121+
};
122+
}
123+
124+
export function assertQaTransportSupportsScenario(params: {
125+
scenario: QaTransportScenarioDefinition;
126+
transport: Pick<QaTransportAdapter, "capabilities" | "id" | "supportedActions">;
127+
}) {
128+
const unsupported = findUnsupportedQaTransportScenarioRequirements(params);
129+
const problems = [
130+
unsupported.capabilities.length > 0
131+
? `missing capabilities: ${unsupported.capabilities.join(", ")}`
132+
: undefined,
133+
unsupported.actions.length > 0
134+
? `unsupported actions: ${unsupported.actions.join(", ")}`
135+
: undefined,
136+
].filter((problem): problem is string => Boolean(problem));
137+
if (problems.length > 0) {
138+
throw new Error(
139+
`QA transport ${params.transport.id} cannot run scenario ${params.scenario.id}; ${problems.join("; ")}`,
140+
);
141+
}
142+
}
143+
144+
export function filterQaTransportScenariosForTransport<
145+
TScenario extends QaTransportScenarioDefinition,
146+
>(params: {
147+
scenarios: readonly TScenario[];
148+
transport: Pick<QaTransportAdapter, "capabilities" | "supportedActions">;
149+
}): TScenario[] {
150+
return params.scenarios.filter((scenario) => {
151+
const unsupported = findUnsupportedQaTransportScenarioRequirements({
152+
scenario,
153+
transport: params.transport,
154+
});
155+
return unsupported.actions.length === 0 && unsupported.capabilities.length === 0;
156+
});
157+
}

‎extensions/qa-lab/src/qa-transport.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ type QaTransportFailureAssertionOptions = {
5656
cursorSpace?: QaTransportFailureCursorSpace;
5757
};
5858

59-
type QaTransportCommonCapabilities = {
59+
export type QaTransportCapabilities = {
6060
sendInboundMessage: QaTransportState["addInboundMessage"];
6161
injectOutboundMessage: QaTransportState["addOutboundMessage"];
6262
waitForOutboundMessage: (input: QaBusWaitForInput) => Promise<unknown>;
@@ -162,8 +162,9 @@ export type QaTransportAdapter = {
162162
label: string;
163163
accountId: string;
164164
requiredPluginIds: readonly string[];
165+
supportedActions: readonly QaTransportActionName[];
165166
state: QaTransportState;
166-
capabilities: QaTransportCommonCapabilities;
167+
capabilities: QaTransportCapabilities;
167168
createGatewayConfig: (params: { baseUrl: string }) => QaTransportGatewayConfig;
168169
waitReady: (params: {
169170
gateway: QaTransportGatewayClient;
@@ -192,20 +193,23 @@ export abstract class QaStateBackedTransportAdapter implements QaTransportAdapte
192193
readonly label: string;
193194
readonly accountId: string;
194195
readonly requiredPluginIds: readonly string[];
196+
readonly supportedActions: readonly QaTransportActionName[];
195197
readonly state: QaTransportState;
196-
readonly capabilities: QaTransportCommonCapabilities;
198+
readonly capabilities: QaTransportCapabilities;
197199

198200
protected constructor(params: {
199201
id: string;
200202
label: string;
201203
accountId: string;
202204
requiredPluginIds: readonly string[];
205+
supportedActions?: readonly QaTransportActionName[];
203206
state: QaTransportState;
204207
}) {
205208
this.id = params.id;
206209
this.label = params.label;
207210
this.accountId = params.accountId;
208211
this.requiredPluginIds = params.requiredPluginIds;
212+
this.supportedActions = params.supportedActions ?? [];
209213
this.state = params.state;
210214
this.capabilities = {
211215
sendInboundMessage: this.state.addInboundMessage.bind(this.state),

0 commit comments

Comments
 (0)