Skip to content

Commit ed63c24

Browse files
committed
test(client): benchmark remote setup and history contention
1 parent eeb09e8 commit ed63c24

1 file changed

Lines changed: 249 additions & 1 deletion

File tree

‎packages/client-runtime/src/remotePerformance.bench.ts‎

Lines changed: 249 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,22 @@ import {
22
EnvironmentId,
33
EventId,
44
MessageId,
5+
ORCHESTRATION_PROTOCOL_VERSION,
6+
ORCHESTRATION_V2_WS_METHODS,
7+
RunId,
8+
type OrchestrationV2Command,
59
type OrchestrationV2DomainEvent,
610
type OrchestrationV2ThreadProjection,
711
} from "@t3tools/contracts";
812
import * as Effect from "effect/Effect";
13+
import * as Crypto from "effect/Crypto";
14+
import * as Deferred from "effect/Deferred";
15+
import * as Layer from "effect/Layer";
916
import * as Option from "effect/Option";
17+
import * as SubscriptionRef from "effect/SubscriptionRef";
1018
import { HttpClient, HttpClientResponse } from "effect/http";
11-
import { describe, test } from "vite-plus/test";
19+
import { Atom, AtomRegistry } from "effect/reactivity";
20+
import { describe, expect, test } from "vite-plus/test";
1221

1322
import { issueRemoteWebSocketTicket } from "./authorization/remote.ts";
1423
import { PrimaryConnectionTarget } from "./connection/model.ts";
@@ -17,6 +26,18 @@ import type { RemoteEnvironmentRequestError } from "./rpc/http.ts";
1726
import { fetchEnvironmentThreadSnapshot } from "./state/threadSnapshotHttp.ts";
1827
import { applyOrchestrationV2ProjectionEvent } from "./state/orchestrationV2Projection.ts";
1928
import { v2Projection, v2Now } from "./state/orchestrationV2TestFixtures.ts";
29+
import * as RemoteAuthorization from "./authorization/service.ts";
30+
import { RemoteDpopAccessTokenStore } from "./authorization/tokenStore.ts";
31+
import { ConnectionCredentialStore } from "./connection/credentialStore.ts";
32+
import { ConnectionProfileStore } from "./connection/profileStore.ts";
33+
import * as EnvironmentRegistry from "./connection/registry.ts";
34+
import * as ConnectionResolver from "./connection/resolver.ts";
35+
import * as EnvironmentSupervisor from "./connection/supervisor.ts";
36+
import * as Capabilities from "./platform/capabilities.ts";
37+
import { ManagedRelayClient, ManagedRelayDpopSigner } from "./relay/managedRelay.ts";
38+
import type { RpcSession } from "./rpc/session.ts";
39+
import { createThreadEnvironmentAtoms } from "./state/threadCommands.ts";
40+
import { ThreadHistoryController } from "./state/threadHistoryController.ts";
2041

2142
const timestamp = "2026-09-01T00:00:00.000Z";
2243
const runOptions = { warmupTime: 1_000, time: 1_500 };
@@ -49,9 +70,11 @@ const responses = {
4970
label: target.label,
5071
platform: { os: "linux", arch: "x64" },
5172
serverVersion: "0.0.0-test",
73+
orchestrationProtocolVersion: ORCHESTRATION_PROTOCOL_VERSION,
5274
capabilities: { repositoryIdentity: true },
5375
},
5476
"/api/auth/websocket-ticket": { ticket: "test-ticket", expiresAt: timestamp },
77+
"/api/auth/websocket-ticket/rpc": { ticket: "test-ticket", expiresAt: timestamp },
5578
"/api/orchestration/threads/thread-v2": { snapshotSequence: 1, projection: thread },
5679
};
5780
const httpClient = HttpClient.make((request) =>
@@ -97,6 +120,231 @@ describe("remote HTTP processing with an in-memory transport", () => {
97120
}
98121
});
99122

123+
const controlledRunOptions = { warmup: false, time: 0, iterations: 3 };
124+
const unused = () => Effect.die("Unexpected benchmark dependency");
125+
const presentation = Layer.succeed(Capabilities.ClientPresentation, {
126+
metadata: { label: "Benchmark", deviceType: "mobile", os: "test" },
127+
});
128+
129+
const prepareSample = (delayMs: number) =>
130+
Effect.scoped(
131+
Effect.gen(function* () {
132+
const paths: string[] = [];
133+
let observedHttpWaitSeconds = 0;
134+
const transport = HttpClient.make((request) =>
135+
Effect.gen(function* () {
136+
const path = new URL(request.url).pathname as keyof typeof responses;
137+
paths.push(path);
138+
if (delayMs > 0) {
139+
const started = performance.now();
140+
yield* Effect.sleep(delayMs);
141+
observedHttpWaitSeconds += (performance.now() - started) / 1_000;
142+
}
143+
return HttpClientResponse.fromWeb(request, Response.json(responses[path]));
144+
}),
145+
);
146+
const dependencies = Layer.mergeAll(
147+
Layer.succeed(HttpClient.HttpClient, transport),
148+
presentation,
149+
Layer.succeed(ManagedRelayDpopSigner, { thumbprint: unused(), createProof: unused }),
150+
Layer.succeed(ManagedRelayClient, {
151+
relayUrl: "https://relay.example.test",
152+
listEnvironments: unused,
153+
listDevices: unused,
154+
createEnvironmentLinkChallenge: unused,
155+
linkEnvironment: unused,
156+
unlinkEnvironment: unused,
157+
getEnvironmentStatus: unused,
158+
connectEnvironment: unused,
159+
registerDevice: unused,
160+
unregisterDevice: unused,
161+
registerLiveActivity: unused,
162+
getAgentActivitySnapshot: unused,
163+
resetTokenCache: Effect.void,
164+
}),
165+
Layer.succeed(Capabilities.CloudSession, { identity: unused(), clerkToken: unused() }),
166+
Layer.succeed(Capabilities.RelayDeviceIdentity, { deviceId: unused() }),
167+
Layer.succeed(RemoteDpopAccessTokenStore, { get: unused, put: unused, remove: unused }),
168+
Layer.succeed(Capabilities.PrimaryEnvironmentAuth, {
169+
bearerToken: Effect.succeedSome("test-token"),
170+
}),
171+
Layer.succeed(Capabilities.SshEnvironmentGateway, {
172+
provision: unused,
173+
prepare: unused,
174+
disconnect: unused,
175+
}),
176+
Layer.succeed(ConnectionProfileStore, { get: unused, put: unused, remove: unused }),
177+
Layer.succeed(ConnectionCredentialStore, { get: unused, put: unused, remove: unused }),
178+
);
179+
const layer = ConnectionResolver.layer.pipe(
180+
Layer.provide(RemoteAuthorization.layer.pipe(Layer.provideMerge(dependencies))),
181+
);
182+
return yield* Effect.gen(function* () {
183+
const resolver = yield* ConnectionResolver.ConnectionResolver;
184+
const samples = [];
185+
for (const phase of ["cold", "warm"] as const) {
186+
paths.length = 0;
187+
observedHttpWaitSeconds = 0;
188+
const started = performance.now();
189+
const prepared = yield* resolver.prepare({
190+
target,
191+
profile: Option.none(),
192+
enabled: true,
193+
});
194+
expect(prepared.socketUrl).toContain("test-ticket");
195+
samples.push({
196+
phase,
197+
seconds: (performance.now() - started) / 1_000,
198+
observedHttpWaitSeconds,
199+
paths: [...paths],
200+
});
201+
}
202+
return samples;
203+
}).pipe(Effect.provide(layer));
204+
}),
205+
);
206+
207+
const commandSample = (action: "startTurn" | "interruptTurn", historyDelayMs: number) =>
208+
Effect.scoped(
209+
Effect.gen(function* () {
210+
const historyStarted = yield* Deferred.make<void>();
211+
let dispatchedAt = 0;
212+
let observedHistoryWaitSeconds = 0;
213+
let observedReplyWaitSeconds = 0;
214+
const supervisor = EnvironmentSupervisor.EnvironmentSupervisor.of({
215+
target,
216+
session: yield* SubscriptionRef.make(
217+
Option.some({
218+
client: {
219+
[ORCHESTRATION_V2_WS_METHODS.dispatchCommand]: Effect.fn(
220+
"RemotePerformance.dispatch",
221+
)(function* (command: OrchestrationV2Command) {
222+
dispatchedAt = performance.now();
223+
expect(command.type).toBe(
224+
action === "startTurn" ? "message.dispatch" : "run.interrupt",
225+
);
226+
const started = performance.now();
227+
yield* Effect.sleep(50);
228+
observedReplyWaitSeconds = (performance.now() - started) / 1_000;
229+
return { sequence: 1 };
230+
}),
231+
},
232+
} as unknown as RpcSession),
233+
),
234+
} as EnvironmentSupervisor.EnvironmentSupervisor["Service"]);
235+
const runtime = Atom.runtime(
236+
Layer.mergeAll(
237+
Layer.succeed(EnvironmentRegistry.EnvironmentRegistry, {
238+
run: (_id, effect) =>
239+
Effect.provideService(
240+
effect,
241+
EnvironmentSupervisor.EnvironmentSupervisor,
242+
supervisor,
243+
),
244+
} as EnvironmentRegistry.EnvironmentRegistry["Service"]),
245+
Layer.succeed(
246+
Crypto.Crypto,
247+
Crypto.make({
248+
randomBytes: (size) => new Uint8Array(size),
249+
digest: (_algorithm, data) => Effect.succeed(data),
250+
}),
251+
),
252+
Layer.succeed(ThreadHistoryController, {
253+
register: unused,
254+
unregister: Effect.fn("ThreadHistoryController.unregister")(() => Effect.void),
255+
reconcilePendingWork: Effect.fn("ThreadHistoryController.reconcilePendingWork")(
256+
() => Effect.void,
257+
),
258+
loadEarlier: Effect.fn("RemotePerformance.loadEarlier")(function* () {
259+
yield* Deferred.succeed(historyStarted, undefined);
260+
if (historyDelayMs > 0) {
261+
const started = performance.now();
262+
yield* Effect.sleep(historyDelayMs);
263+
observedHistoryWaitSeconds = (performance.now() - started) / 1_000;
264+
}
265+
return { _tag: "loaded" } as const;
266+
}),
267+
}),
268+
),
269+
);
270+
const registry = AtomRegistry.make();
271+
yield* Effect.addFinalizer(() => Effect.sync(() => registry.dispose()));
272+
const commands = createThreadEnvironmentAtoms(runtime, () => Atom.make(null));
273+
const history = commands.loadEarlierHistory.run(registry, {
274+
environmentId: target.environmentId,
275+
input: { threadId: thread.thread.id },
276+
});
277+
yield* Deferred.await(historyStarted);
278+
const started = performance.now();
279+
const result = yield* Effect.promise(() =>
280+
commands[action].run(registry, {
281+
environmentId: target.environmentId,
282+
input: {
283+
threadId: thread.thread.id,
284+
runId: RunId.make("running-run"),
285+
message: {
286+
messageId: MessageId.make("send"),
287+
role: "user",
288+
text: "Send",
289+
attachments: [],
290+
},
291+
runtimeMode: "full-access",
292+
interactionMode: "default",
293+
dispatchMode: "start",
294+
},
295+
}),
296+
);
297+
const completeSeconds = (performance.now() - started) / 1_000;
298+
expect(result._tag).toBe("Success");
299+
expect(dispatchedAt).toBeGreaterThan(0);
300+
expect((yield* Effect.promise(() => history))._tag).toBe("Success");
301+
return {
302+
dispatchSeconds: (dispatchedAt - started) / 1_000,
303+
completeSeconds,
304+
observedHistoryWaitSeconds,
305+
observedReplyWaitSeconds,
306+
};
307+
}),
308+
);
309+
310+
describe("controlled fixture latency in seconds, not phone or Wi-Fi measurements", () => {
311+
for (const delayMs of [0, 100]) {
312+
test(`prepare with ${delayMs}ms injected per HTTP response`, async ({ bench }) => {
313+
const samples: Effect.Success<ReturnType<typeof prepareSample>>[] = [];
314+
await bench("actual resolver cold and warm prepare", async () => {
315+
samples.push(await Effect.runPromise(prepareSample(delayMs)));
316+
}).run(controlledRunOptions);
317+
Effect.runSync(
318+
Effect.log(
319+
JSON.stringify({ scenario: "prepare", injectedHttpSeconds: delayMs / 1_000, samples }),
320+
),
321+
);
322+
});
323+
}
324+
for (const action of ["startTurn", "interruptTurn"] as const) {
325+
for (const historyDelayMs of [0, 250]) {
326+
test(`${action} with ${historyDelayMs}ms injected history and 50ms command reply`, async ({
327+
bench,
328+
}) => {
329+
const samples: Effect.Success<ReturnType<typeof commandSample>>[] = [];
330+
await bench(`actual thread command scheduler ${action}`, async () => {
331+
samples.push(await Effect.runPromise(commandSample(action, historyDelayMs)));
332+
}).run(controlledRunOptions);
333+
Effect.runSync(
334+
Effect.log(
335+
JSON.stringify({
336+
scenario: action,
337+
injectedHistorySeconds: historyDelayMs / 1_000,
338+
injectedCommandSeconds: 0.05,
339+
samples,
340+
}),
341+
),
342+
);
343+
});
344+
}
345+
}
346+
});
347+
100348
const delta: Extract<OrchestrationV2DomainEvent, { type: "message.updated" }> = {
101349
id: EventId.make("delta"),
102350
type: "message.updated",

0 commit comments

Comments
 (0)