Skip to content

Commit f6b3191

Browse files
committed
test(server): replay fx through the shared ACP fixtures
Adds a recorder modeled on the Grok recorder and fixtures recorded from fx 0.0.13: simple, multi_turn, queued_turn, turn_interrupt, message_steering, the Supervised approve and decline cases, and an early Stop recorded as one live session. The shared harness gains three general options: a fixture-level runtimeMode, the flavor cancelBehavior in the ACP replay runtime, and an optional release label on the interrupt step.
1 parent a06dd47 commit f6b3191

21 files changed

Lines changed: 1438 additions & 8 deletions

‎apps/server/scripts/record-fx-acp-replay-fixture.ts‎

Lines changed: 532 additions & 0 deletions
Large diffs are not rendered by default.

‎apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.testkit.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ export function makeAcpReplayRuntime(input: {
188188
readonly childProcessSpawner: ChildProcessSpawner.ChildProcessSpawner["Service"];
189189
readonly fileSystem: FileSystem.FileSystem;
190190
readonly cancelMeta?: AcpSessionRuntime.AcpSessionRuntimeOptions["cancelMeta"];
191+
readonly cancelBehavior?: AcpSessionRuntime.AcpSessionRuntimeOptions["cancelBehavior"];
191192
readonly initializeMeta?: AcpSessionRuntime.AcpSessionRuntimeOptions["initializeMeta"];
192193
readonly replayGate?: ProviderReplayGate;
193194
}): (
@@ -229,6 +230,7 @@ export function makeAcpReplayRuntime(input: {
229230
},
230231
authMethodId: "replay",
231232
...(input.cancelMeta === undefined ? {} : { cancelMeta: input.cancelMeta }),
233+
...(input.cancelBehavior === undefined ? {} : { cancelBehavior: input.cancelBehavior }),
232234
...(input.initializeMeta === undefined ? {} : { initializeMeta: input.initializeMeta }),
233235
...(input.replayGate === undefined
234236
? {}
Lines changed: 340 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,340 @@
1+
import * as NodeServices from "@effect/platform-node/NodeServices";
2+
import { assert, describe, it } from "@effect/vitest";
3+
import {
4+
ProviderInstanceId,
5+
ProviderSessionId,
6+
ThreadId,
7+
type RuntimeMode,
8+
} from "@t3tools/contracts";
9+
import { ACP_PROTOCOL } from "@t3tools/provider-acp/server/adapter";
10+
import {
11+
FX_ACP_CANCEL_BEHAVIOR,
12+
FX_DRIVER_KIND,
13+
FxProviderCapabilitiesV2,
14+
makeFxAdapterV2,
15+
} from "@t3tools/provider-fx/testing";
16+
import { FxSettings } from "@t3tools/provider-fx/settings";
17+
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
18+
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
19+
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
20+
import { resolveSelfInvocation } from "@t3tools/shared/nodeRuntime";
21+
import * as Effect from "effect/Effect";
22+
import * as FileSystem from "effect/FileSystem";
23+
import * as Layer from "effect/Layer";
24+
import * as Path from "effect/Path";
25+
import * as Schema from "effect/Schema";
26+
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
27+
import type * as EffectAcpSchema from "effect-acp/compat";
28+
import * as TestProviderHost from "@t3tools/provider-testing/TestProviderHost";
29+
30+
import {
31+
decodeAcpReplayTranscript,
32+
makeAcpReplayCompletenessAssertion,
33+
makeAcpReplayRuntime,
34+
} from "./AcpAdapterV2.testkit.ts";
35+
36+
const decodeFxSettings = Schema.decodeSync(FxSettings);
37+
const recordedConfigOptions = [
38+
{
39+
id: "provider",
40+
name: "Provider",
41+
category: "model",
42+
type: "select",
43+
currentValue: "gateway",
44+
options: [{ value: "gateway", name: "Vercel AI Gateway" }],
45+
},
46+
{
47+
id: "model",
48+
name: "Model",
49+
category: "model",
50+
type: "select",
51+
currentValue: "anthropic/claude-opus-5.5",
52+
options: [
53+
{ value: "anthropic/claude-sonnet-5.5", name: "anthropic/claude-sonnet-5.5" },
54+
{ value: "anthropic/claude-opus-5.5", name: "anthropic/claude-opus-5.5" },
55+
],
56+
},
57+
{
58+
id: "mode",
59+
name: "Session Mode",
60+
category: "mode",
61+
type: "select",
62+
currentValue: "ask",
63+
options: [
64+
{ value: "code", name: "Code" },
65+
{ value: "ask", name: "Ask" },
66+
],
67+
},
68+
{
69+
id: "effort",
70+
name: "Reasoning Effort",
71+
category: "thought_level",
72+
type: "select",
73+
currentValue: "xhigh",
74+
options: [
75+
{ value: "medium", name: "medium" },
76+
{ value: "high", name: "high" },
77+
{ value: "xhigh", name: "xhigh" },
78+
],
79+
},
80+
] as const satisfies ReadonlyArray<EffectAcpSchema.SessionConfigOption>;
81+
82+
const testLayer = TestProviderHost.layer().pipe(
83+
Layer.provide(NodeServices.layer),
84+
Layer.merge(IdAllocator.layer),
85+
Layer.merge(McpProviderSessions.layer),
86+
Layer.merge(NodeServices.layer),
87+
);
88+
89+
type ReplayEntry = {
90+
readonly type: "expect_outbound" | "emit_inbound";
91+
readonly frame: {
92+
readonly kind: "request" | "response";
93+
readonly method: string;
94+
readonly params?: unknown;
95+
readonly result?: unknown;
96+
};
97+
};
98+
99+
const outbound = (method: string, params: unknown = "<any>"): ReplayEntry => ({
100+
type: "expect_outbound",
101+
frame: { kind: "request", method, ...(params === undefined ? {} : { params }) },
102+
});
103+
const answer = (method: string, result: unknown): ReplayEntry => ({
104+
type: "emit_inbound",
105+
frame: { kind: "response", method, result },
106+
});
107+
108+
function withCurrent(
109+
configId: string,
110+
value: string,
111+
base: ReadonlyArray<EffectAcpSchema.SessionConfigOption> = recordedConfigOptions,
112+
): ReadonlyArray<EffectAcpSchema.SessionConfigOption> {
113+
return base.map((option) =>
114+
option.id === configId && option.type === "select"
115+
? { ...option, currentValue: value }
116+
: option,
117+
);
118+
}
119+
120+
const initializeResult = {
121+
protocolVersion: 1,
122+
agentCapabilities: {
123+
loadSession: true,
124+
promptCapabilities: { image: true, audio: false, embeddedContext: true },
125+
},
126+
agentInfo: { name: "fx", title: "fx", version: "0.0.13" },
127+
authMethods: [],
128+
};
129+
130+
const openReplay = Effect.fn("openFxReplay")(function* (input: {
131+
readonly scenario: string;
132+
readonly entries: ReadonlyArray<ReplayEntry>;
133+
readonly initialNativeThreadId?: string;
134+
readonly runtimeMode?: RuntimeMode;
135+
readonly model?: string;
136+
readonly options?: ReadonlyArray<{ readonly id: string; readonly value: string }>;
137+
readonly fxDefaultModel?: string;
138+
}) {
139+
const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;
140+
const fileSystem = yield* FileSystem.FileSystem;
141+
const path = yield* Path.Path;
142+
const replayDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-fx-adapter-" });
143+
const statusPath = path.join(replayDir, "status.json");
144+
const transcript = yield* decodeAcpReplayTranscript(
145+
{
146+
provider: FX_DRIVER_KIND,
147+
protocol: ACP_PROTOCOL,
148+
version: "1",
149+
scenario: input.scenario,
150+
entries: input.entries,
151+
},
152+
FX_DRIVER_KIND,
153+
);
154+
const instanceId = ProviderInstanceId.make("fx");
155+
const adapter = yield* makeFxAdapterV2({
156+
instanceId,
157+
settings: decodeFxSettings({ enabled: true }),
158+
environment: {},
159+
selfInvocation: yield* resolveSelfInvocation(),
160+
...(input.fxDefaultModel === undefined
161+
? {}
162+
: { currentFxDefaultModel: Effect.succeed(input.fxDefaultModel) }),
163+
makeRuntime: makeAcpReplayRuntime({
164+
transcript,
165+
statusPath,
166+
scriptPath: yield* path.fromFileUrl(
167+
new URL("../../../scripts/acp-replay-agent.ts", import.meta.url),
168+
),
169+
childProcessSpawner,
170+
fileSystem,
171+
cancelBehavior: FX_ACP_CANCEL_BEHAVIOR,
172+
}),
173+
});
174+
yield* adapter
175+
.openSession({
176+
threadId: ThreadId.make("thread-fx"),
177+
providerSessionId: ProviderSessionId.make("provider-session-fx"),
178+
modelSelection: {
179+
instanceId,
180+
model: input.model ?? "anthropic/claude-sonnet-5.5",
181+
...(input.options === undefined
182+
? { options: [{ id: "effort", value: "high" }] }
183+
: { options: input.options }),
184+
},
185+
runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({
186+
runtimeMode: input.runtimeMode ?? "auto",
187+
interactionMode: "default",
188+
cwd: replayDir,
189+
}),
190+
...(input.initialNativeThreadId === undefined
191+
? {}
192+
: { initialNativeThreadId: input.initialNativeThreadId }),
193+
})
194+
.pipe(Effect.scoped);
195+
yield* makeAcpReplayCompletenessAssertion(fileSystem, statusPath, transcript);
196+
});
197+
198+
describe("FxAdapterV2", () => {
199+
it("does not switch runtime mode or steer in session", () => {
200+
assert.equal(FxProviderCapabilitiesV2.sessions.supportsRuntimeModeSwitchInSession, false);
201+
assert.equal(FxProviderCapabilitiesV2.turns.supportsActiveSteering, false);
202+
});
203+
204+
it.effect("writes model, effort, and the advertised auto mode after session/new", () =>
205+
openReplay({
206+
scenario: "fx-session-new-config",
207+
entries: [
208+
outbound("initialize"),
209+
answer("initialize", initializeResult),
210+
outbound("session/new"),
211+
answer("session/new", { sessionId: "fx-session", configOptions: recordedConfigOptions }),
212+
outbound("session/set_config_option", {
213+
sessionId: "fx-session",
214+
configId: "model",
215+
value: "anthropic/claude-sonnet-5.5",
216+
}),
217+
answer("session/set_config_option", {
218+
configOptions: withCurrent("model", "anthropic/claude-sonnet-5.5"),
219+
}),
220+
outbound("session/set_config_option", {
221+
sessionId: "fx-session",
222+
configId: "effort",
223+
value: "high",
224+
}),
225+
answer("session/set_config_option", {
226+
configOptions: withCurrent("effort", "high"),
227+
}),
228+
outbound("session/set_config_option", {
229+
sessionId: "fx-session",
230+
configId: "mode",
231+
value: "code",
232+
}),
233+
answer("session/set_config_option", {
234+
configOptions: withCurrent("mode", "code"),
235+
}),
236+
],
237+
}).pipe(Effect.provide(testLayer), Effect.scoped),
238+
);
239+
240+
it.effect("writes medium effort when the thread stored none", () =>
241+
openReplay({
242+
scenario: "fx-session-new-default-effort",
243+
options: [],
244+
entries: [
245+
outbound("initialize"),
246+
answer("initialize", initializeResult),
247+
outbound("session/new"),
248+
answer("session/new", { sessionId: "fx-session", configOptions: recordedConfigOptions }),
249+
outbound("session/set_config_option", {
250+
sessionId: "fx-session",
251+
configId: "model",
252+
value: "anthropic/claude-sonnet-5.5",
253+
}),
254+
answer("session/set_config_option", {
255+
configOptions: withCurrent("model", "anthropic/claude-sonnet-5.5"),
256+
}),
257+
outbound("session/set_config_option", {
258+
sessionId: "fx-session",
259+
configId: "effort",
260+
value: "medium",
261+
}),
262+
answer("session/set_config_option", {
263+
configOptions: withCurrent(
264+
"effort",
265+
"medium",
266+
withCurrent("model", "anthropic/claude-sonnet-5.5"),
267+
),
268+
}),
269+
outbound("session/set_config_option", {
270+
sessionId: "fx-session",
271+
configId: "mode",
272+
value: "code",
273+
}),
274+
answer("session/set_config_option", { configOptions: withCurrent("mode", "code") }),
275+
],
276+
}).pipe(Effect.provide(testLayer), Effect.scoped),
277+
);
278+
279+
it.effect("writes ask on a supervised session/load", () =>
280+
openReplay({
281+
scenario: "fx-session-load-ask",
282+
runtimeMode: "approval-required",
283+
initialNativeThreadId: "fx-session",
284+
entries: [
285+
outbound("initialize"),
286+
answer("initialize", initializeResult),
287+
outbound("session/load"),
288+
answer("session/load", { sessionId: "fx-session", configOptions: recordedConfigOptions }),
289+
outbound("session/set_config_option", {
290+
sessionId: "fx-session",
291+
configId: "model",
292+
value: "anthropic/claude-sonnet-5.5",
293+
}),
294+
answer("session/set_config_option", {
295+
configOptions: withCurrent("model", "anthropic/claude-sonnet-5.5"),
296+
}),
297+
outbound("session/set_config_option", {
298+
sessionId: "fx-session",
299+
configId: "effort",
300+
value: "high",
301+
}),
302+
answer("session/set_config_option", { configOptions: withCurrent("effort", "high") }),
303+
outbound("session/set_config_option", {
304+
sessionId: "fx-session",
305+
configId: "mode",
306+
value: "ask",
307+
}),
308+
answer("session/set_config_option", { configOptions: withCurrent("mode", "ask") }),
309+
],
310+
}).pipe(Effect.provide(testLayer), Effect.scoped),
311+
);
312+
313+
it.effect("resolves default to the probed fx model and never sends the sentinel", () =>
314+
openReplay({
315+
scenario: "fx-session-default-model",
316+
model: "default",
317+
options: [],
318+
fxDefaultModel: "anthropic/claude-opus-5.5",
319+
runtimeMode: "approval-required",
320+
entries: [
321+
outbound("initialize"),
322+
answer("initialize", initializeResult),
323+
outbound("session/new"),
324+
answer("session/new", { sessionId: "fx-session", configOptions: recordedConfigOptions }),
325+
outbound("session/set_config_option", {
326+
sessionId: "fx-session",
327+
configId: "effort",
328+
value: "medium",
329+
}),
330+
answer("session/set_config_option", { configOptions: withCurrent("effort", "medium") }),
331+
outbound("session/set_config_option", {
332+
sessionId: "fx-session",
333+
configId: "mode",
334+
value: "ask",
335+
}),
336+
answer("session/set_config_option", { configOptions: withCurrent("mode", "ask") }),
337+
],
338+
}).pipe(Effect.provide(testLayer), Effect.scoped),
339+
);
340+
});

0 commit comments

Comments
 (0)