Skip to content

Commit fb2b8c5

Browse files
committed
fix: keep reconnect recovery authoritative
1 parent 4e8e3a1 commit fb2b8c5

9 files changed

Lines changed: 94 additions & 78 deletions

File tree

‎apps/server/src/ws.ts‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -294,12 +294,10 @@ const PROVIDER_STATUS_DEBOUNCE_MS = 200;
294294

295295
// When a resuming client's cursor is more than this many events behind the
296296
// current head, skip the per-event catch-up replay and send a fresh shell
297-
// snapshot instead. Past this gap a single O(active-threads) snapshot is
298-
// cheaper and bounded. The shell replay is coalesced per aggregate (see
299-
// coalesceShellStream), so its cost stays bounded well past the event store's
300-
// default page size and a wider gap keeps wake/resume on the cheap replay
301-
// path instead of forcing a full snapshot.
302-
const SHELL_RESUME_MAX_GAP = 5_000;
297+
// snapshot instead. Replaying each intervening event costs a shell refetch;
298+
// past this gap a single O(active-threads) snapshot is cheaper and bounded.
299+
// Matches the event store's default page size (DEFAULT_READ_FROM_SEQUENCE_LIMIT).
300+
const SHELL_RESUME_MAX_GAP = 1_000;
303301

304302
// Same bound for thread resume. The replay reads the *global* event range and
305303
// filters per-thread afterwards, so a stale cursor far behind the head would

‎apps/web/src/components/ChatView.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4741,7 +4741,7 @@ function ChatViewContent(props: ChatViewProps) {
47414741
toastManager.add(
47424742
stackedThreadToast({
47434743
type: "warning",
4744-
title: "Not connected — message not sent",
4744+
title: "Not connected: message not sent",
47454745
description: "Reconnecting to the environment. Try again once it is connected.",
47464746
}),
47474747
);

‎apps/web/src/components/chat/ChatComposer.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1270,6 +1270,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
12701270
[activePendingIsResponding, activePendingProgress, activePendingResolvedAnswers],
12711271
);
12721272
const collapsedComposerPrimaryActionDisabled =
1273+
phase === "running" ||
12731274
isSendBusy ||
12741275
isSendDisabled ||
12751276
isConnecting ||

‎apps/web/src/components/chat/ComposerPrimaryActions.tsx‎

Lines changed: 11 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -134,48 +134,17 @@ export const ComposerPrimaryActions = memo(function ComposerPrimaryActions({
134134

135135
if (isRunning) {
136136
return (
137-
<div className={cn("flex items-center justify-end", compact ? "gap-1.5" : "gap-2")}>
138-
<Button
139-
type="submit"
140-
size="icon"
141-
variant="outline"
142-
className="size-8 rounded-full"
143-
{...pointerFocusProps}
144-
disabled={
145-
isSendBusy ||
146-
isSendDisabled ||
147-
isConnecting ||
148-
isEnvironmentUnavailable ||
149-
!hasSendableContent
150-
}
151-
aria-label="Send message to running agent"
152-
>
153-
{isConnecting || isSendBusy ? (
154-
<Spinner className="size-3.5" aria-hidden="true" />
155-
) : (
156-
<svg className="size-3.5" viewBox="0 0 14 14" fill="none" aria-hidden="true">
157-
<path
158-
d="M7 11.5V2.5M7 2.5L3 6.5M7 2.5L11 6.5"
159-
stroke="currentColor"
160-
strokeWidth="1.8"
161-
strokeLinecap="round"
162-
strokeLinejoin="round"
163-
/>
164-
</svg>
165-
)}
166-
</Button>
167-
<button
168-
type="button"
169-
className="flex size-8 cursor-pointer items-center justify-center rounded-full bg-destructive/90 text-white shadow-xs shadow-destructive/24 inset-shadow-[0_1px_--theme(--color-white/16%)] transition-all duration-150 hover:bg-destructive hover:scale-105 active:inset-shadow-[0_1px_--theme(--color-black/8%)] active:shadow-none sm:h-8 sm:w-8"
170-
{...pointerFocusProps}
171-
onClick={onInterrupt}
172-
aria-label="Stop generation"
173-
>
174-
<svg width="12" height="12" viewBox="0 0 12 12" fill="currentColor" aria-hidden="true">
175-
<rect x="2" y="2" width="8" height="8" rx="1.5" />
176-
</svg>
177-
</button>
178-
</div>
137+
<button
138+
type="button"
139+
className="flex size-8 cursor-pointer items-center justify-center rounded-full bg-destructive/90 text-white shadow-xs shadow-destructive/24 inset-shadow-[0_1px_--theme(--color-white/16%)] transition-all duration-150 hover:bg-destructive hover:scale-105 active:inset-shadow-[0_1px_--theme(--color-black/8%)] active:shadow-none sm:h-8 sm:w-8"
140+
{...pointerFocusProps}
141+
onClick={onInterrupt}
142+
aria-label="Stop generation"
143+
>
144+
<svg width="12" height="12" viewBox="0 0 12 12" fill="currentColor" aria-hidden="true">
145+
<rect x="2" y="2" width="8" height="8" rx="1.5" />
146+
</svg>
147+
</button>
179148
);
180149
}
181150

‎packages/client-runtime/src/connection/supervisor.test.ts‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -927,7 +927,12 @@ describe("EnvironmentSupervisor", () => {
927927

928928
it.effect("reconnects immediately when the foreground liveness probe fails", () =>
929929
Effect.gen(function* () {
930+
const allowReconnect = yield* Deferred.make<void>();
930931
const harness = yield* makeHarness({
932+
prepare: (attempt) =>
933+
attempt === 2
934+
? Deferred.await(allowReconnect).pipe(Effect.as(PREPARED_CONNECTION))
935+
: Effect.succeed(PREPARED_CONNECTION),
931936
probe: (attempt) =>
932937
attempt === 1 ? Effect.fail(transient("The live session is stale.")) : Effect.void,
933938
});
@@ -937,8 +942,15 @@ describe("EnvironmentSupervisor", () => {
937942

938943
yield* awaitState(supervisor.state, (state) => state.phase === "connected");
939944
yield* harness.wake("application-active");
940-
// No TestClock advance: a failed wake probe skips the first backoff rung
941-
// and reconnects immediately.
945+
const reconnecting = yield* awaitState(
946+
supervisor.state,
947+
(state) => state.phase === "connecting",
948+
);
949+
expect(reconnecting.attempt).toBe(1);
950+
expect(Option.isNone(yield* SubscriptionRef.get(supervisor.session))).toBe(true);
951+
952+
// No TestClock advance: a failed wake probe skips the first backoff rung.
953+
yield* Deferred.succeed(allowReconnect, undefined);
942954
yield* awaitState(
943955
supervisor.state,
944956
(state) => state.phase === "connected" && state.generation === 2 && state.attempt === 1,
@@ -981,7 +993,7 @@ describe("EnvironmentSupervisor", () => {
981993
}).pipe(Effect.provide(TestClock.layer())),
982994
);
983995

984-
it.effect("quickly times out a stalled desktop foreground liveness probe", () =>
996+
it.effect("uses the full tolerance window for a stalled desktop foreground probe", () =>
985997
Effect.gen(function* () {
986998
const harness = yield* makeHarness({
987999
probe: (attempt) => (attempt === 1 ? Effect.never : Effect.void),
@@ -992,7 +1004,9 @@ describe("EnvironmentSupervisor", () => {
9921004

9931005
yield* awaitState(supervisor.state, (state) => state.phase === "connected");
9941006
yield* harness.wake("application-active");
995-
yield* TestClock.adjust("3 seconds");
1007+
yield* TestClock.adjust("14999 millis");
1008+
expect(yield* Ref.get(harness.sessionCount)).toBe(1);
1009+
yield* TestClock.adjust("1 milli");
9961010
yield* awaitState(
9971011
supervisor.state,
9981012
(state) => state.phase === "connected" && state.generation === 2 && state.attempt === 1,

‎packages/client-runtime/src/connection/supervisor.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ import * as ConnectionWakeups from "./wakeups.ts";
3131

3232
const RETRY_DELAYS_MS = [3_000, 4_000, 8_000, 16_000] as const;
3333
const CONNECTION_ESTABLISHMENT_TIMEOUT = "15 seconds";
34-
const CONNECTION_PROBE_TIMEOUT = "3 seconds";
34+
const CONNECTION_PROBE_TIMEOUT = "15 seconds";
3535
const MOBILE_CONNECTION_PROBE_TIMEOUT = "3 seconds";
3636
const BACKOFF_RESET_AFTER_MS = 30_000;
3737

@@ -725,6 +725,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* (
725725
// backoff rung. Only this first attempt skips the ladder; if it fails
726726
// too, normal backoff resumes.
727727
resetRetryLadder();
728+
yield* setState(connectingState(yield* Ref.get(intent), generation, 1, error));
728729
continue;
729730
}
730731

‎packages/client-runtime/src/rpc/session.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,33 @@ describe("RpcSessionFactory", () => {
287287
}),
288288
);
289289

290+
it.effect("tolerates two missed pong windows before closing the session", () =>
291+
Effect.gen(function* () {
292+
const { factory, sockets } = yield* makeFactory();
293+
const session = yield* factory.connect(PREPARED);
294+
const readyFiber = yield* Effect.forkChild(session.ready);
295+
const closedFiber = yield* Effect.forkChild(Effect.flip(session.closed));
296+
const socket = yield* awaitSocket(sockets);
297+
298+
socket.open();
299+
yield* completeInitialConfig(socket);
300+
yield* Fiber.join(readyFiber);
301+
302+
yield* TestClock.adjust("15 seconds");
303+
expect(closedFiber.pollUnsafe()).toBeUndefined();
304+
expect(socket.sent.slice(1).map((request) => decodeJson(request))).toEqual([
305+
{ _tag: "Ping" },
306+
{ _tag: "Ping" },
307+
{ _tag: "Ping" },
308+
]);
309+
310+
yield* TestClock.adjust("5 seconds");
311+
const error = yield* Fiber.join(closedFiber);
312+
expect(error).toBeInstanceOf(ConnectionTransientError);
313+
expect(error).toMatchObject({ reason: "transport" });
314+
}).pipe(Effect.scoped, Effect.provide(TestClock.layer())),
315+
);
316+
290317
it.effect("reaches ready when a newer server sends unknown config members", () =>
291318
Effect.gen(function* () {
292319
const { factory, sockets } = yield* makeFactory();

‎packages/client-runtime/src/state/shell-sync.test.ts‎

Lines changed: 22 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ describe("environment shell synchronization", () => {
150150
}),
151151
);
152152

153-
it.effect("resumes from a warm shell cache without reloading the HTTP snapshot", () =>
153+
it.effect("replaces a warm shell cache with an authoritative snapshot on a new session", () =>
154154
Effect.gen(function* () {
155155
const cachedSnapshot: OrchestrationShellSnapshot = {
156156
snapshotSequence: 5,
@@ -219,22 +219,19 @@ describe("environment shell synchronization", () => {
219219
Effect.provideService(ShellSnapshotLoader, snapshotLoader),
220220
);
221221

222-
// Wait until the subscription is established from the warm cache.
222+
// A new session refreshes the cache before opening the stream.
223223
yield* SubscriptionRef.changes(capturedAfterSequence).pipe(
224224
Stream.filter((value) => value !== undefined),
225225
Stream.runHead,
226226
);
227227

228-
expect(yield* SubscriptionRef.get(capturedAfterSequence)).toBe(5);
228+
expect(yield* SubscriptionRef.get(capturedAfterSequence)).toBe(9_999);
229229
expect(yield* Ref.get(capturedCompletionMarker)).toBe(true);
230-
expect(yield* SubscriptionRef.get(loaderCalls)).toBe(0);
230+
expect(yield* SubscriptionRef.get(loaderCalls)).toBe(1);
231231
const synchronizing = yield* SubscriptionRef.get(shellState);
232232
expect(synchronizing.status).toBe("synchronizing");
233-
expect(Option.getOrThrow(synchronizing.snapshot)).toEqual(cachedSnapshot);
233+
expect(Option.getOrThrow(synchronizing.snapshot)).toEqual(resetSnapshot);
234234

235-
// When the resume gap overflows SHELL_RESUME_MAX_GAP the server resets
236-
// the stream with a fresh snapshot; the client applies it as usual.
237-
yield* Queue.offer(events, { kind: "snapshot", snapshot: resetSnapshot });
238235
yield* Queue.offer(events, { kind: "synchronized" });
239236
yield* SubscriptionRef.changes(shellState).pipe(
240237
Stream.filter((value) => value.status === "live"),
@@ -243,7 +240,7 @@ describe("environment shell synchronization", () => {
243240

244241
const live = yield* SubscriptionRef.get(shellState);
245242
expect(Option.getOrThrow(live.snapshot)).toEqual(resetSnapshot);
246-
expect(yield* SubscriptionRef.get(loaderCalls)).toBe(0);
243+
expect(yield* SubscriptionRef.get(loaderCalls)).toBe(1);
247244
}),
248245
);
249246

@@ -263,10 +260,11 @@ describe("environment shell synchronization", () => {
263260
),
264261
} as unknown as WsRpcProtocolClient;
265262
const supervisorState = yield* SubscriptionRef.make(AVAILABLE_CONNECTION_STATE);
263+
const activeSession = yield* SubscriptionRef.make(Option.some(session(client)));
266264
const supervisor = EnvironmentSupervisor.EnvironmentSupervisor.of({
267265
target: TARGET,
268266
state: supervisorState,
269-
session: yield* SubscriptionRef.make(Option.some(session(client))),
267+
session: activeSession,
270268
prepared: yield* SubscriptionRef.make(Option.some(PREPARED)),
271269
connect: Effect.void,
272270
disconnect: Effect.void,
@@ -304,12 +302,12 @@ describe("environment shell synchronization", () => {
304302
),
305303
);
306304

307-
// The warm cache resumes at its own cursor without an HTTP load.
305+
// A new session starts from an authoritative HTTP snapshot.
308306
for (let attempt = 0; attempt < 100; attempt += 1) {
309307
if ((yield* Ref.get(capturedAfterSequences)).length >= 1) break;
310308
yield* Effect.yieldNow;
311309
}
312-
expect(yield* Ref.get(capturedAfterSequences)).toEqual([1]);
310+
expect(yield* Ref.get(capturedAfterSequences)).toEqual([10]);
313311
yield* Queue.offer(events, { kind: "synchronized" });
314312
yield* SubscriptionRef.changes(shellState).pipe(
315313
Stream.filter((value) => value.status === "live"),
@@ -333,24 +331,31 @@ describe("environment shell synchronization", () => {
333331
if ((yield* Ref.get(capturedAfterSequences)).length >= 2) break;
334332
yield* Effect.yieldNow;
335333
}
336-
expect(yield* Ref.get(capturedAfterSequences)).toEqual([1, 40]);
334+
expect(yield* Ref.get(capturedAfterSequences)).toEqual([10, 40]);
337335
yield* Queue.offer(events, { kind: "synchronized" });
338336

339337
yield* Queue.offer(wakeups, "application-active-probe");
340338
for (let attempt = 0; attempt < 100; attempt += 1) {
341339
if ((yield* Ref.get(capturedAfterSequences)).length >= 3) break;
342340
yield* Effect.yieldNow;
343341
}
344-
expect(yield* Ref.get(capturedAfterSequences)).toEqual([1, 40, 40]);
342+
expect(yield* Ref.get(capturedAfterSequences)).toEqual([10, 40, 40]);
345343

346344
yield* Queue.offer(wakeups, "application-active-reconnect");
347345
for (let attempt = 0; attempt < 10; attempt += 1) {
348346
yield* Effect.yieldNow;
349347
}
350348
expect((yield* Ref.get(capturedAfterSequences)).length).toBe(3);
351-
// The in-memory cursor made every resubscription warm: the HTTP
352-
// snapshot loader is never consulted.
353-
expect(yield* Ref.get(loaderCalls)).toBe(0);
349+
expect(yield* Ref.get(loaderCalls)).toBe(1);
350+
351+
// Replacing the session performs another authoritative refresh.
352+
yield* SubscriptionRef.set(activeSession, Option.some(session(client)));
353+
for (let attempt = 0; attempt < 100; attempt += 1) {
354+
if ((yield* Ref.get(capturedAfterSequences)).length >= 4) break;
355+
yield* Effect.yieldNow;
356+
}
357+
expect(yield* Ref.get(capturedAfterSequences)).toEqual([10, 40, 40, 20]);
358+
expect(yield* Ref.get(loaderCalls)).toBe(2);
354359
}),
355360
);
356361
});

‎packages/client-runtime/src/state/shell.ts‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import * as ConnectionWakeups from "../connection/wakeups.ts";
2121
import { safeErrorLogAttributes } from "../errors/safeLog.ts";
2222
import { EnvironmentCacheStore } from "../platform/persistence.ts";
2323
import { subscribeDynamic } from "../rpc/client.ts";
24+
import type { RpcSession } from "../rpc/session.ts";
2425
import { ShellSnapshotLoader } from "./shellSnapshotHttp.ts";
2526
import { applyShellStreamEvent } from "./shellReducer.ts";
2627
import type { EnvironmentCatalogState } from "./connections.ts";
@@ -71,6 +72,7 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
7172
error: Option.none(),
7273
});
7374
const awaitingCompletion = yield* Ref.make(false);
75+
const lastAuthoritativeSession = yield* Ref.make<RpcSession | null>(null);
7476
const persistence = yield* Queue.sliding<OrchestrationShellSnapshot>(1);
7577

7678
const persist = Effect.fn("EnvironmentShellState.persist")(function* (
@@ -187,14 +189,12 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
187189
yield* Ref.set(awaitingCompletion, supportsCompletionMarker);
188190
yield* setSynchronizing;
189191

190-
// Warm resume: a snapshot already in memory (restored from the cache
191-
// or left over from a previous subscription) carries a resume cursor,
192-
// so subscribe with it directly instead of re-fetching the full shell
193-
// snapshot over HTTP. When the cursor is too far behind, the server
194-
// resets the stream with a fresh snapshot (see SHELL_RESUME_MAX_GAP
195-
// in the server), which applyItem applies like any other item.
192+
// Foreground resubscriptions on the same live session can resume from
193+
// the in-memory cursor. A new session reloads the authoritative HTTP
194+
// snapshot so a valid cursor cannot preserve incomplete cached data.
195+
const hasAuthoritativeSnapshot = (yield* Ref.get(lastAuthoritativeSession)) === session;
196196
let current = yield* SubscriptionRef.get(state);
197-
if (Option.isNone(current.snapshot)) {
197+
if (!hasAuthoritativeSnapshot || Option.isNone(current.snapshot)) {
198198
const prepared = yield* SubscriptionRef.get(supervisor.prepared).pipe(
199199
Effect.flatMap(
200200
Option.match({
@@ -212,6 +212,7 @@ export const makeEnvironmentShellState = Effect.fn("EnvironmentShellState.make")
212212
const httpSnapshot = yield* snapshotLoader.load(prepared);
213213
if (Option.isSome(httpSnapshot)) {
214214
yield* applyItem({ kind: "snapshot", snapshot: httpSnapshot.value });
215+
yield* Ref.set(lastAuthoritativeSession, session);
215216
current = yield* SubscriptionRef.get(state);
216217
}
217218
}

0 commit comments

Comments
 (0)