Skip to content

Commit 7aff2b1

Browse files
t3dotggclaude
andcommitted
test(server): measure shell, second client, and reconnect transfer
The transfer budget measured one thread subscription. Real clients also hold a sidebar subscription, often run on two devices, and reconnect. Measure all three on the same turn and print them next to the capped thread rows, plus a SQL statement count from sql.execute spans. Co-Authored-By: Claude Fable 5.1 <[email protected]>
1 parent 80c708a commit 7aff2b1

6 files changed

Lines changed: 436 additions & 65 deletions

‎apps/server/integration/NetworkTransferMeasurement.integration.ts‎

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@ import * as NodeZlib from "node:zlib";
55
import * as NodeSocket from "@effect/platform-node/NodeSocket";
66
import { WsRpcGroup } from "@t3tools/contracts";
77
import * as Effect from "effect/Effect";
8+
import * as Exit from "effect/Exit";
89
import * as Layer from "effect/Layer";
910
import * as Schema from "effect/Schema";
11+
import * as Scope from "effect/Scope";
1012
import { RpcClient, RpcSerialization } from "effect/unstable/rpc";
1113
import * as Socket from "effect/unstable/socket/Socket";
1214

@@ -101,6 +103,8 @@ export interface WebSocketTransferRecorder {
101103
) => globalThis.WebSocket;
102104
readonly totals: () => WebSocketTransferTotals;
103105
readonly negotiatedExtensions: () => string;
106+
/** Resolves once the upgrade completes, so totals taken after it exclude the upgrade response. */
107+
readonly awaitOpen: Effect.Effect<void>;
104108
}
105109

106110
interface NodeWebSocketWithTransport extends NodeSocket.NodeWS.WebSocket {
@@ -118,8 +122,15 @@ function rawDataBytes(data: NodeSocket.NodeWS.RawData): number {
118122

119123
export function makeWebSocketTransferRecorder(): WebSocketTransferRecorder {
120124
let socket: NodeWebSocketWithTransport | null = null;
125+
// Held separately from the WebSocket so wire totals survive a close, which
126+
// is when a reconnect measurement reads them.
127+
let transport: NodeWebSocketWithTransport["_socket"] | null = null;
121128
let decodedBytes = 0;
122129
let messages = 0;
130+
let resolveOpen: () => void = () => {};
131+
const opened = new Promise<void>((resolve) => {
132+
resolveOpen = resolve;
133+
});
123134

124135
return {
125136
connect: (url, protocols, cookie) => {
@@ -128,6 +139,10 @@ export function makeWebSocketTransferRecorder(): WebSocketTransferRecorder {
128139
perMessageDeflate: true,
129140
}) as NodeWebSocketWithTransport;
130141
socket = nextSocket;
142+
nextSocket.once("open", () => {
143+
transport = nextSocket._socket ?? null;
144+
resolveOpen();
145+
});
131146
nextSocket.on("message", (data) => {
132147
const bytes = rawDataBytes(data);
133148
decodedBytes += bytes;
@@ -136,11 +151,17 @@ export function makeWebSocketTransferRecorder(): WebSocketTransferRecorder {
136151
return nextSocket as unknown as globalThis.WebSocket;
137152
},
138153
totals: () => ({
139-
wireBytes: socket?._socket?.bytesRead ?? 0,
154+
wireBytes: transport?.bytesRead ?? socket?._socket?.bytesRead ?? 0,
140155
decodedBytes,
141156
messages,
142157
}),
143158
negotiatedExtensions: () => socket?.extensions ?? "",
159+
awaitOpen: Effect.promise(() => opened).pipe(
160+
Effect.timeoutOrElse({
161+
duration: "10 seconds",
162+
orElse: () => Effect.die(new Error("Timed out waiting for the WebSocket to open")),
163+
}),
164+
),
144165
};
145166
}
146167

@@ -175,3 +196,40 @@ export function countingWsRpcProtocolLayer(input: {
175196

176197
export const makeCountingWsRpcClient = RpcClient.make(WsRpcGroup);
177198
export type CountingWsRpcClient = Effect.Success<typeof makeCountingWsRpcClient>;
199+
200+
export interface MeasuredWsClient {
201+
readonly client: CountingWsRpcClient;
202+
readonly recorder: WebSocketTransferRecorder;
203+
/** Fork subscription consumers here so they stop before the socket closes. */
204+
readonly scope: Scope.Scope;
205+
/** Closes the socket now. The enclosing scope closes it otherwise. */
206+
readonly close: Effect.Effect<void>;
207+
}
208+
209+
/**
210+
* Opens one WebSocket RPC client on a child of the current scope. Several
211+
* clients can share one test scope and still disconnect independently, which
212+
* a reconnect measurement needs.
213+
*/
214+
export const openMeasuredWsClient = Effect.fn("TransferBudget.openMeasuredWsClient")(
215+
function* (input: { readonly url: string; readonly cookie: string }) {
216+
const recorder = makeWebSocketTransferRecorder();
217+
const parent = yield* Effect.scope;
218+
const scope = yield* Scope.fork(parent);
219+
const protocol = yield* Layer.buildWithScope(
220+
countingWsRpcProtocolLayer({ url: input.url, cookie: input.cookie, recorder }),
221+
scope,
222+
);
223+
const client = yield* makeCountingWsRpcClient.pipe(
224+
Effect.provide(protocol),
225+
Scope.provide(scope),
226+
);
227+
yield* recorder.awaitOpen;
228+
return {
229+
client,
230+
recorder,
231+
scope,
232+
close: Scope.close(scope, Exit.void),
233+
} satisfies MeasuredWsClient;
234+
},
235+
);

‎apps/server/integration/OrchestrationEngineHarness.integration.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import * as Schedule from "effect/Schedule";
2222
import * as Schema from "effect/Schema";
2323
import * as Scope from "effect/Scope";
2424
import * as Stream from "effect/Stream";
25+
import * as Tracer from "effect/Tracer";
2526

2627
import * as CheckpointStore from "../src/checkpointing/CheckpointStore.ts";
2728
import { TextGeneration, type TextGenerationShape } from "../src/textGeneration/TextGeneration.ts";
@@ -231,6 +232,8 @@ export interface OrchestrationIntegrationHarness {
231232
interface MakeOrchestrationIntegrationHarnessOptions {
232233
readonly provider?: ProviderDriverKind;
233234
readonly realCodex?: boolean;
235+
/** Tracer for every fiber the harness runtime runs, including reactors. */
236+
readonly tracer?: Tracer.Tracer;
234237
}
235238

236239
export const makeOrchestrationIntegrationHarness = (
@@ -399,6 +402,9 @@ export const makeOrchestrationIntegrationHarness = (
399402
Layer.provideMerge(ServerSettingsService.layerTest()),
400403
Layer.provideMerge(ServerConfig.layerTest(workspaceDir, rootDir)),
401404
Layer.provideMerge(NodeServices.layer),
405+
Layer.provideMerge(
406+
options?.tracer ? Layer.succeed(Tracer.Tracer, options.tracer) : Layer.empty,
407+
),
402408
);
403409

404410
const runtime = ManagedRuntime.make(layer);
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import * as Tracer from "effect/Tracer";
2+
3+
export interface SqlStatementCounter {
4+
readonly tracer: Tracer.Tracer;
5+
/** Statements executed so far. Read before and after a phase and subtract. */
6+
readonly count: () => number;
7+
}
8+
9+
/**
10+
* Counts `sql.execute` spans, which the Effect SQL client opens once per
11+
* statement. Spans behave exactly as with the native tracer. Install the same
12+
* counter on every runtime under test, otherwise statements run by background
13+
* reactors and statements run by request handlers land in different counters.
14+
*/
15+
export function makeSqlStatementCounter(): SqlStatementCounter {
16+
let statements = 0;
17+
const tracer = Tracer.make({
18+
span: (options) => {
19+
if (options.name === "sql.execute") statements += 1;
20+
return new Tracer.NativeSpan(options);
21+
},
22+
});
23+
return { tracer, count: () => statements };
24+
}

‎apps/server/integration/TransferBudgetReport.integration.ts‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,28 @@ import {
1212
TRANSFER_MEASURED_TOOLS,
1313
} from "./fixtures/transferBudget.ts";
1414

15+
/** Catch-up delivered to a resubscribing client, and which path the server chose. */
16+
export interface WebSocketCatchUpMeasurement extends WebSocketTransferTotals {
17+
readonly mode: "replay" | "snapshot";
18+
}
19+
1520
export interface TransferBudgetRun {
1621
readonly provider: ProviderDriverKind;
1722
readonly threadSnapshot: HttpTransferMeasurement;
23+
/** One socket holding only the thread subscription. This is the capped measurement. */
1824
readonly measuredTurnWebSocket: WebSocketTransferTotals;
25+
readonly shellSnapshot: HttpTransferMeasurement;
26+
/** One socket holding only the shell (sidebar) subscription during the same turn. */
27+
readonly measuredTurnShellWebSocket: WebSocketTransferTotals;
28+
/** A second client: one socket holding both the thread and shell subscriptions. */
29+
readonly measuredTurnSecondClientWebSocket: WebSocketTransferTotals;
30+
/** The second client resubscribes after the turn from the cursor it held before it. */
31+
readonly reconnectThread: WebSocketCatchUpMeasurement;
32+
readonly reconnectShell: WebSocketCatchUpMeasurement;
33+
/** `sql.execute` spans opened server-wide during the measured turn. */
34+
readonly measuredTurnSqlStatements: number;
35+
/** `sql.execute` spans opened while serving both reconnect catch-ups. */
36+
readonly reconnectSqlStatements: number;
1937
}
2038

2139
interface ProviderTransferBudget {
@@ -47,6 +65,15 @@ function totalWireBytes(run: TransferBudgetRun): number {
4765
return run.threadSnapshot.wireBytes + run.measuredTurnWebSocket.wireBytes;
4866
}
4967

68+
/** Bytes the server wrote to every measured socket during the turn. */
69+
function serverEgressWireBytes(run: TransferBudgetRun): number {
70+
return (
71+
run.measuredTurnWebSocket.wireBytes +
72+
run.measuredTurnShellWebSocket.wireBytes +
73+
run.measuredTurnSecondClientWebSocket.wireBytes
74+
);
75+
}
76+
5077
function observedTransfer(run: TransferBudgetRun) {
5178
return {
5279
totalWireBytes: totalWireBytes(run),
@@ -105,6 +132,33 @@ function row(
105132
return `| ${provider} | ${phase} | ${metric} | ${format(observed)} | ${format(maximum)} | ${status} |`;
106133
}
107134

135+
// Shell, second-client, reconnect, and SQL rows are reported without a cap.
136+
// Shell delivery coalesces on a 50 ms window, so message counts and bytes move
137+
// with scheduler timing between runs, and the reconnect and SQL figures follow
138+
// the same batches. The rows exist so CI shows the numbers next to the capped
139+
// thread measurement.
140+
function infoRow(
141+
provider: ProviderDriverKind,
142+
phase: string,
143+
metric: string,
144+
observed: number,
145+
format: (value: number) => string = formatBytes,
146+
): string {
147+
return `| ${provider} | ${phase} | ${metric} | ${format(observed)} | none | INFO |`;
148+
}
149+
150+
function webSocketRows(
151+
provider: ProviderDriverKind,
152+
phase: string,
153+
totals: WebSocketTransferTotals,
154+
): string[] {
155+
return [
156+
infoRow(provider, phase, "WebSocket wire", totals.wireBytes),
157+
infoRow(provider, phase, "WebSocket decoded", totals.decodedBytes),
158+
infoRow(provider, phase, "WebSocket messages", totals.messages, String),
159+
];
160+
}
161+
108162
export function transferBudgetViolations(runs: ReadonlyArray<TransferBudgetRun>): string[] {
109163
const violations: string[] = [];
110164
for (const run of runs) {
@@ -146,6 +200,7 @@ export function formatTransferBudgetReport(runs: ReadonlyArray<TransferBudgetRun
146200
"# T3 Code thread transfer budget",
147201
"",
148202
"Wire values are thread data bytes read from local HTTP and WebSocket sockets. HTTP includes response headers; WebSocket measurement starts after the resumed thread subscription synchronizes. TCP/IP, TLS framing, and the WebSocket upgrade are excluded. WebSocket permessage-deflate is negotiated.",
203+
"The measured turn is observed on three sockets at once: one with only the thread subscription (the capped rows), one with only the shell subscription, and a second client holding both. Server egress is the sum of the three. After the turn the second client disconnects and resubscribes from the cursor it held before the turn, which is the cursor a backgrounded phone would hold. SQL statements are `sql.execute` spans counted across the orchestration runtime and the WebSocket handlers.",
149204
`Scenario: ${TRANSFER_HISTORY_TURN_COUNT} historical turns with ${TRANSFER_HISTORY_TOOLS_PER_TURN} command tools and one retained ${formatBytes(TRANSFER_HISTORY_MCP_RESULT_BYTES)} MCP result each, followed by one measured turn with ${TRANSFER_MEASURED_TOOLS} command tools and a retained ${formatBytes(TRANSFER_MEASURED_MCP_RESULT_BYTES)} MCP result. Payload sizes are calibrated from heavy local Codex and Claude histories and contain no user data.`,
150205
"",
151206
"| Provider | Total thread wire | Budget | Result |",
@@ -198,13 +253,40 @@ export function formatTransferBudgetReport(runs: ReadonlyArray<TransferBudgetRun
198253
budget.measuredTurnWebSocketMessages,
199254
String,
200255
),
256+
infoRow(run.provider, "shell snapshot", "HTTP wire", run.shellSnapshot.wireBytes),
257+
...webSocketRows(run.provider, "measured turn, shell", run.measuredTurnShellWebSocket),
258+
...webSocketRows(
259+
run.provider,
260+
"measured turn, second client",
261+
run.measuredTurnSecondClientWebSocket,
262+
),
263+
infoRow(run.provider, "measured turn", "server egress wire", serverEgressWireBytes(run)),
264+
infoRow(
265+
run.provider,
266+
"measured turn",
267+
"SQL statements",
268+
run.measuredTurnSqlStatements,
269+
String,
270+
),
271+
...webSocketRows(
272+
run.provider,
273+
`reconnect, thread (${run.reconnectThread.mode})`,
274+
run.reconnectThread,
275+
),
276+
...webSocketRows(
277+
run.provider,
278+
`reconnect, shell (${run.reconnectShell.mode})`,
279+
run.reconnectShell,
280+
),
281+
infoRow(run.provider, "reconnect", "SQL statements", run.reconnectSqlStatements, String),
201282
);
202283
}
203284

204285
lines.push("", "## Compression diagnostics", "");
205286
for (const run of runs) {
206287
lines.push(
207288
`- ${run.provider}: thread snapshot ${formatBytes(run.threadSnapshot.decodedBodyBytes)} decoded to ${formatBytes(run.threadSnapshot.encodedBodyBytes)} gzip.`,
289+
`- ${run.provider}: shell snapshot ${formatBytes(run.shellSnapshot.decodedBodyBytes)} decoded to ${formatBytes(run.shellSnapshot.encodedBodyBytes)} gzip.`,
208290
);
209291
}
210292

‎apps/server/integration/TransferBudgetScenario.integration.ts‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,20 @@ import {
55
DEFAULT_MODEL_BY_PROVIDER,
66
DEFAULT_PROVIDER_INTERACTION_MODE,
77
MessageId,
8+
ORCHESTRATION_WS_METHODS,
9+
type OrchestrationShellStreamItem,
10+
type OrchestrationThreadStreamItem,
811
ProjectId,
912
ProviderDriverKind,
1013
ThreadId,
1114
} from "@t3tools/contracts";
1215
import * as Effect from "effect/Effect";
16+
import * as Queue from "effect/Queue";
17+
import * as Scope from "effect/Scope";
18+
import * as Stream from "effect/Stream";
1319

1420
import type { TurnProcessingQuiescedReceipt } from "../src/orchestration/Services/RuntimeReceiptBus.ts";
21+
import type { MeasuredWsClient } from "./NetworkTransferMeasurement.integration.ts";
1522
import type { OrchestrationIntegrationHarness } from "./OrchestrationEngineHarness.integration.ts";
1623
import {
1724
expectedRecordedAssistantText,
@@ -125,4 +132,81 @@ export function expectedMeasuredAssistantText(provider: ProviderDriverKind): str
125132
return expectedRecordedAssistantText(provider, TRANSFER_MEASURED_TURN_INDEX);
126133
}
127134

135+
/** Takes from the queue until one value matches, and returns everything taken. */
136+
export const collectQueueUntil = Effect.fn("TransferBudget.collectQueueUntil")(function* <A>(
137+
queue: Queue.Queue<A>,
138+
predicate: (value: A) => boolean,
139+
waitDescription: string,
140+
) {
141+
return yield* Effect.gen(function* () {
142+
const values: A[] = [];
143+
while (true) {
144+
const value = yield* Queue.take(queue);
145+
values.push(value);
146+
if (predicate(value)) return values;
147+
}
148+
}).pipe(
149+
Effect.timeoutOrElse({
150+
duration: "10 seconds",
151+
orElse: () => Effect.die(new Error(`Timed out waiting for ${waitDescription}`)),
152+
}),
153+
);
154+
});
155+
156+
/**
157+
* Resumes the thread subscription from a cursor on a measured client. The
158+
* consumer runs in the client's scope, so closing the client stops it. Callers
159+
* read items from the returned queue.
160+
*/
161+
export const subscribeThreadItems = Effect.fn("TransferBudget.subscribeThreadItems")(function* (
162+
measured: MeasuredWsClient,
163+
afterSequence: number,
164+
) {
165+
const items = yield* Queue.unbounded<OrchestrationThreadStreamItem>();
166+
yield* measured.client[ORCHESTRATION_WS_METHODS.subscribeThread]({
167+
threadId: TRANSFER_THREAD_ID,
168+
afterSequence,
169+
requestCompletionMarker: true,
170+
}).pipe(
171+
Stream.runForEach((item) => Queue.offer(items, item).pipe(Effect.asVoid)),
172+
Scope.provide(measured.scope),
173+
Effect.forkIn(measured.scope),
174+
);
175+
return items;
176+
});
177+
178+
/** Shell counterpart of subscribeThreadItems. */
179+
export const subscribeShellItems = Effect.fn("TransferBudget.subscribeShellItems")(function* (
180+
measured: MeasuredWsClient,
181+
afterSequence: number,
182+
) {
183+
const items = yield* Queue.unbounded<OrchestrationShellStreamItem>();
184+
yield* measured.client[ORCHESTRATION_WS_METHODS.subscribeShell]({
185+
afterSequence,
186+
requestCompletionMarker: true,
187+
}).pipe(
188+
Stream.runForEach((item) => Queue.offer(items, item).pipe(Effect.asVoid)),
189+
Scope.provide(measured.scope),
190+
Effect.forkIn(measured.scope),
191+
);
192+
return items;
193+
});
194+
195+
/** Waits for the initial catch-up and reports whether it was a replay or a snapshot reset. */
196+
export const awaitSubscriptionSynchronized = Effect.fn(
197+
"TransferBudget.awaitSubscriptionSynchronized",
198+
)(function* <Item extends OrchestrationThreadStreamItem | OrchestrationShellStreamItem>(
199+
items: Queue.Queue<Item>,
200+
waitDescription: string,
201+
) {
202+
const initial = yield* collectQueueUntil(
203+
items,
204+
(item) => item.kind === "synchronized",
205+
waitDescription,
206+
);
207+
return initial.some((item) => item.kind === "snapshot")
208+
? ("snapshot" as const)
209+
: ("replay" as const);
210+
});
211+
128212
export { TRANSFER_HISTORY_TURN_COUNT, waitForTurnQuiesced };

0 commit comments

Comments
 (0)