Skip to content

Commit 24ce672

Browse files
RomneyDaRomneyDa
authored
refactor(qa): use transport-native actions in flow scenarios (#97962)
* Add QA channel scenario driver contracts * Route DM baseline through channel scenario driver * Simplify channel behavior flow runtime helper * test(qa): use native YAML for DM channel scenario * test(qa): add conversation flow action * test(qa): break channel scenario import cycle * test(qa): fix channel scenario lint * Unify QA channel scenario transport adapter * Fix QA transport test type expectations * refactor(qa): use native transport flow actions * test(qa): derive direct reply session from transport
1 parent 477b27b commit 24ce672

24 files changed

Lines changed: 472 additions & 313 deletions

‎extensions/qa-lab/src/crabline-transport.test.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ describe("crabline transport", () => {
107107
});
108108

109109
try {
110-
await transport.state.addInboundMessage({
110+
await transport.sendInbound({
111111
conversation: {
112112
id: "D12345678",
113113
kind: "direct",
@@ -139,6 +139,17 @@ describe("crabline transport", () => {
139139
await release();
140140
expect(response.ok).toBe(true);
141141

142+
await expect(
143+
transport.waitForOutbound({
144+
conversation: { id: "D12345678", kind: "direct" },
145+
textIncludes: "assistant via fake slack",
146+
timeoutMs: 1_000,
147+
}),
148+
).resolves.toMatchObject({
149+
conversation: { id: "D12345678", kind: "direct" },
150+
text: "assistant via fake slack",
151+
});
152+
142153
await expect(
143154
transport.state.waitFor({
144155
direction: "outbound",

‎extensions/qa-lab/src/qa-channel-transport.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,27 @@ describe("qa channel transport", () => {
119119
expect(message.text).toBe("hello from the operator");
120120
});
121121

122+
it("implements the portable scenario transport actions", async () => {
123+
const transport = createQaChannelTransport(createQaBusState());
124+
const conversation = { id: "alice", kind: "direct" as const };
125+
126+
await transport.sendInbound({
127+
conversation,
128+
senderId: "alice",
129+
text: "hello",
130+
});
131+
await transport.state.addOutboundMessage({
132+
to: "dm:alice",
133+
text: "QA-PORTABLE-OK",
134+
});
135+
136+
await expect(
137+
transport.waitForOutbound({ conversation, textIncludes: "QA-PORTABLE-OK" }),
138+
).resolves.toMatchObject({ text: "QA-PORTABLE-OK" });
139+
await transport.reset();
140+
expect(transport.state.getSnapshot().messages).toEqual([]);
141+
});
142+
122143
it("inherits the shared failure-aware wait helper", async () => {
123144
const transport = createQaChannelTransport(createQaBusState());
124145
let injected = false;

‎extensions/qa-lab/src/qa-transport.ts‎

Lines changed: 72 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ import type {
88
QaBusInboundMessageInput,
99
QaBusMessage,
1010
QaBusOutboundMessageInput,
11-
QaBusSearchMessagesInput,
1211
QaBusReadMessageInput,
12+
QaBusSearchMessagesInput,
1313
QaBusStateSnapshot,
1414
QaBusWaitForInput,
1515
} from "./runtime-api.js";
@@ -56,6 +56,20 @@ type QaTransportFailureAssertionOptions = {
5656
cursorSpace?: QaTransportFailureCursorSpace;
5757
};
5858

59+
export type QaTransportOutboundMatch = {
60+
conversation?: QaBusInboundMessageInput["conversation"];
61+
senderId?: string;
62+
sinceIndex?: number;
63+
textIncludes?: string;
64+
threadId?: string;
65+
timeoutMs?: number;
66+
};
67+
68+
export type QaTransportWaitForNoOutboundInput = {
69+
quietMs?: number;
70+
sinceIndex?: number;
71+
};
72+
5973
export type QaTransportCapabilities = {
6074
sendInboundMessage: QaTransportState["addInboundMessage"];
6175
injectOutboundMessage: QaTransportState["addOutboundMessage"];
@@ -165,6 +179,10 @@ export type QaTransportAdapter = {
165179
supportedActions: readonly QaTransportActionName[];
166180
state: QaTransportState;
167181
capabilities: QaTransportCapabilities;
182+
reset: () => Promise<void>;
183+
sendInbound: (input: QaBusInboundMessageInput) => Promise<QaBusMessage>;
184+
waitForNoOutbound: (input?: QaTransportWaitForNoOutboundInput) => Promise<void>;
185+
waitForOutbound: (input: QaTransportOutboundMatch) => Promise<QaBusMessage>;
168186
createGatewayConfig: (params: { baseUrl: string }) => QaTransportGatewayConfig;
169187
waitReady: (params: {
170188
gateway: QaTransportGatewayClient;
@@ -248,4 +266,57 @@ export abstract class QaStateBackedTransportAdapter implements QaTransportAdapte
248266
accountId?: string | null;
249267
}) => Promise<unknown>;
250268
abstract createReportNotes: (params: QaTransportReportParams) => string[];
269+
270+
async reset() {
271+
await this.state.reset();
272+
}
273+
274+
async sendInbound(input: QaBusInboundMessageInput) {
275+
return await this.state.addInboundMessage(input);
276+
}
277+
278+
async waitForNoOutbound(input: QaTransportWaitForNoOutboundInput = {}) {
279+
const quietMs = resolveTimerTimeoutMs(input.quietMs, 1_200, 0);
280+
await sleep(quietMs);
281+
assertNoFailureReplies(this.state, {
282+
sinceIndex: input.sinceIndex,
283+
cursorSpace: "outbound",
284+
});
285+
const observed = this.outboundSince(input.sinceIndex);
286+
if (observed.length > 0) {
287+
const summary = observed.map((message) => `${message.id}:${message.text}`).join("\n");
288+
throw new Error(`expected no outbound messages for ${quietMs}ms, saw:\n${summary}`);
289+
}
290+
}
291+
292+
async waitForOutbound(input: QaTransportOutboundMatch) {
293+
return await waitForQaTransportCondition(() => {
294+
assertNoFailureReplies(this.state, {
295+
sinceIndex: input.sinceIndex,
296+
cursorSpace: "outbound",
297+
});
298+
return this.outboundSince(input.sinceIndex).find((message) => {
299+
if (input.conversation && message.conversation.id !== input.conversation.id) {
300+
return false;
301+
}
302+
if (input.conversation && message.conversation.kind !== input.conversation.kind) {
303+
return false;
304+
}
305+
if (input.senderId && message.senderId !== input.senderId) {
306+
return false;
307+
}
308+
if (input.threadId && message.threadId !== input.threadId) {
309+
return false;
310+
}
311+
return !input.textIncludes || message.text.includes(input.textIncludes);
312+
});
313+
}, input.timeoutMs);
314+
}
315+
316+
private outboundSince(sinceIndex = 0) {
317+
return this.state
318+
.getSnapshot()
319+
.messages.filter((message) => message.direction === "outbound")
320+
.slice(sinceIndex);
321+
}
251322
}

‎extensions/qa-lab/src/scenario-catalog.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,23 @@ const qaFlowCallActionSchema = z.object({
150150
saveAs: z.string().trim().min(1).optional(),
151151
});
152152

153+
const qaFlowTransportActionSchema = z.union([
154+
z.object({
155+
resetTransport: z.literal(true),
156+
}),
157+
z.object({
158+
sendInbound: z.unknown(),
159+
saveAs: z.string().trim().min(1).optional(),
160+
}),
161+
z.object({
162+
waitForOutbound: z.unknown(),
163+
saveAs: z.string().trim().min(1).optional(),
164+
}),
165+
z.object({
166+
waitForNoOutbound: z.unknown(),
167+
}),
168+
]);
169+
153170
const qaFlowSetActionSchema = z.object({
154171
set: z.string().trim().min(1),
155172
value: z.unknown(),
@@ -185,6 +202,7 @@ qaFlowIfShapeBase[qaFlowThenKey] = z.array(z.unknown()).min(1);
185202
const qaFlowActionSchema: z.ZodType = z.lazy(() =>
186203
z.union([
187204
qaFlowCallActionSchema,
205+
qaFlowTransportActionSchema,
188206
qaFlowSetActionSchema,
189207
qaFlowAssertActionSchema,
190208
qaFlowThrowActionSchema,

‎extensions/qa-lab/src/scenario-flow-runner.test.ts‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,39 @@ async function runLoadedScenarioFlow(
3333

3434
const state = createQaBusState();
3535
let waitCount = 0;
36+
const transport = {
37+
state,
38+
reset: async () => {
39+
state.reset();
40+
},
41+
sendInbound: async (input: Parameters<typeof state.addInboundMessage>[0]) =>
42+
state.addInboundMessage(input),
43+
waitForNoOutbound: async () => undefined,
44+
waitForOutbound: async (input: {
45+
conversation?: { id: string; kind: string };
46+
textIncludes?: string;
47+
timeoutMs?: number;
48+
}) => {
49+
waitCount += 1;
50+
params.onWaitForOutboundMessage?.({ waitCount, state });
51+
const match = state
52+
.getSnapshot()
53+
.messages.find(
54+
(candidate) =>
55+
candidate.direction === "outbound" &&
56+
(!input.conversation || candidate.conversation.id === input.conversation.id) &&
57+
(!input.conversation || candidate.conversation.kind === input.conversation.kind) &&
58+
(!input.textIncludes || candidate.text.includes(input.textIncludes)),
59+
);
60+
if (match) {
61+
return match;
62+
}
63+
throw new Error(`timed out after ${input.timeoutMs}ms waiting for outbound marker`);
64+
},
65+
};
3666
const api = {
3767
env: {},
68+
transport,
3869
state,
3970
scenario,
4071
config: scenario.execution.config ?? {},

‎extensions/qa-lab/src/scenario-flow-runner.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,21 @@ async function runFlowAction(action: unknown, api: QaFlowApi, vars: QaFlowVars)
161161
}
162162
return;
163163
}
164+
for (const name of ["sendInbound", "waitForOutbound", "waitForNoOutbound"] as const) {
165+
if (name in action) {
166+
const callable = resolveCallable(`transport.${name}`, api, vars);
167+
const result = await callable(await resolveValue(action[name], api, vars));
168+
if (typeof action.saveAs === "string" && action.saveAs.trim()) {
169+
vars[action.saveAs.trim()] = result;
170+
}
171+
return;
172+
}
173+
}
174+
if (action.resetTransport === true) {
175+
const reset = resolveCallable("transport.reset", api, vars);
176+
await reset();
177+
return;
178+
}
164179
if (typeof action.set === "string") {
165180
vars[action.set] = await resolveValue(action.value, api, vars);
166181
return;

‎extensions/qa-lab/src/scenario-runtime-api.test.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,15 @@ describe("createQaScenarioRuntimeApi", () => {
133133
lab: { baseUrl: "http://127.0.0.1:1234" },
134134
transport: {
135135
state,
136+
reset: async () => {
137+
state.reset();
138+
},
139+
sendInbound: async (input: Parameters<typeof state.addInboundMessage>[0]) =>
140+
state.addInboundMessage(input),
141+
waitForNoOutbound: vi.fn(async () => undefined),
142+
waitForOutbound: vi.fn(async () => {
143+
throw new Error("not used");
144+
}),
136145
capabilities: {
137146
waitForCondition,
138147
getNormalizedMessageState: state.getSnapshot.bind(state),

‎extensions/qa-lab/src/scenario-runtime-api.ts‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,22 @@
11
// Qa Lab API module exposes the plugin public contract.
22
import type * as NodeFs from "node:fs/promises";
33
import type * as NodePath from "node:path";
4-
import type { QaTransportCapabilities, QaTransportState } from "./qa-transport.js";
4+
import type { QaTransportAdapter } from "./qa-transport.js";
55
import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
66

77
type QaScenarioRuntimeFunction = (...args: never[]) => unknown;
88

9+
type QaScenarioTransport = Pick<
10+
QaTransportAdapter,
11+
"capabilities" | "reset" | "sendInbound" | "state" | "waitForNoOutbound" | "waitForOutbound"
12+
>;
13+
914
export type QaScenarioRuntimeEnv<
1015
TLab = unknown,
11-
TTransportState extends QaTransportState = QaTransportState,
16+
TTransport extends QaScenarioTransport = QaScenarioTransport,
1217
> = {
1318
lab: TLab;
14-
transport: {
15-
state: TTransportState;
16-
capabilities: QaTransportCapabilities;
17-
};
19+
transport: TTransport;
1820
};
1921

2022
export type QaScenarioRuntimeDeps = {
@@ -107,6 +109,7 @@ type QaScenarioRuntimeApi<
107109
> = {
108110
env: TEnv;
109111
lab: TEnv["lab"];
112+
transport: TEnv["transport"];
110113
state: TEnv["transport"]["state"];
111114
scenario: QaSeedScenarioWithSource;
112115
config: Record<string, unknown>;
@@ -216,6 +219,7 @@ export function createQaScenarioRuntimeApi<
216219
return {
217220
env: params.env,
218221
lab: params.env.lab,
222+
transport: params.env.transport,
219223
state: params.env.transport.state,
220224
scenario: params.scenario,
221225
config: params.scenario.execution.config ?? {},

‎extensions/qa-lab/src/suite-runtime-flow.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,10 @@ describe("qa suite runtime flow", () => {
179179
supportedActions: [],
180180
handleAction: vi.fn(),
181181
createReportNotes: vi.fn(),
182+
reset: vi.fn(),
183+
sendInbound: vi.fn(),
184+
waitForNoOutbound: vi.fn(),
185+
waitForOutbound: vi.fn(),
182186
state: {
183187
reset: vi.fn(),
184188
getSnapshot: vi.fn(),

‎qa/scenarios/channels/channel-chat-baseline.yaml‎

Lines changed: 21 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -46,21 +46,18 @@ flow:
4646
- set: outboundStartIndex
4747
value:
4848
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
49-
- call: state.addInboundMessage
50-
args:
51-
- conversation:
52-
id: qa-room
53-
kind: channel
54-
title: QA Room
55-
senderId: alice
56-
senderName: Alice
57-
text: hello team, no bot ping here
58-
- call: waitForNoOutbound
59-
args:
60-
- ref: state
61-
- 1200
62-
- sinceIndex:
63-
ref: outboundStartIndex
49+
- sendInbound:
50+
conversation:
51+
id: qa-room
52+
kind: channel
53+
title: QA Room
54+
senderId: alice
55+
senderName: Alice
56+
text: hello team, no bot ping here
57+
- waitForNoOutbound:
58+
quietMs: 1200
59+
sinceIndex:
60+
ref: outboundStartIndex
6461
- name: replies when mentioned in channel
6562
actions:
6663
- call: waitForGatewayHealthy
@@ -71,16 +68,15 @@ flow:
7168
args:
7269
- ref: env
7370
- 60000
74-
- call: state.addInboundMessage
75-
args:
76-
- conversation:
77-
id: qa-room
78-
kind: channel
79-
title: QA Room
80-
senderId: alice
81-
senderName: Alice
82-
text:
83-
expr: config.mentionPrompt
71+
- sendInbound:
72+
conversation:
73+
id: qa-room
74+
kind: channel
75+
title: QA Room
76+
senderId: alice
77+
senderName: Alice
78+
text:
79+
expr: config.mentionPrompt
8480
- call: waitForOutboundMessage
8581
saveAs: message
8682
args:

0 commit comments

Comments
 (0)