Skip to content

Commit f5cb98a

Browse files
PR Batch Testerjuliusmarmingeclaude
committed
fix(client-runtime): threads no longer stay on "Syncing messages..." after a failed resume
A saved thread resumes by asking the server for the events after its cursor. When that replay dies, the same cursor dies again on every reconnect and relaunch, so the thread stays on "Syncing messages...". A failed resume now retries once with the full snapshot, which moves the cursor past the bad events. The sync error is published only where the attempt is given up, so a recovered resume never flashes it and a defect on a snapshot attempt stays terminal. Refs #19563 Co-Authored-By: Julius Marminge <[email protected]> Co-Authored-By: Claude Fable 5.1 <[email protected]> Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 05b697f commit f5cb98a

2 files changed

Lines changed: 114 additions & 8 deletions

File tree

‎packages/client-runtime/src/state/threads-sync.test.ts‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,15 @@ const BASE_PROJECTION: OrchestrationV2ThreadProjection = {
6767
thread: { ...v2Projection.thread, title: "Cached thread" },
6868
};
6969

70-
type TestThreadInput = OrchestrationV2ThreadStreamItem | Error;
70+
/** Ends the stream with a defect, like a chunk the client cannot decode. */
71+
class TestStreamDefect {
72+
readonly message: string;
73+
constructor(message: string) {
74+
this.message = message;
75+
}
76+
}
77+
78+
type TestThreadInput = OrchestrationV2ThreadStreamItem | Error | TestStreamDefect;
7179

7280
function testSession(
7381
client: WsRpcProtocolClient,
@@ -118,6 +126,7 @@ const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (o
118126
const latest = yield* Ref.make<EnvironmentThreadState>(EMPTY_ENVIRONMENT_THREAD_STATE);
119127
const retryCount = yield* Ref.make(0);
120128
const subscriptionCount = yield* Ref.make(0);
129+
const subscriptions = yield* Queue.unbounded<number | undefined>();
121130
const loaderCalls = yield* Ref.make(0);
122131
const lastSubscribeAfterSequence = yield* Ref.make<number | undefined>(undefined);
123132
const lastRequestCompletionMarker = yield* Ref.make(false);
@@ -132,7 +141,11 @@ const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (o
132141
const streamFrom = (queue: Queue.Queue<TestThreadInput>) =>
133142
Stream.fromQueue(queue).pipe(
134143
Stream.mapEffect((input) =>
135-
input instanceof Error ? Effect.fail(input) : Effect.succeed(input),
144+
input instanceof Error
145+
? Effect.fail(input)
146+
: input instanceof TestStreamDefect
147+
? Effect.die(new Error(input.message))
148+
: Effect.succeed(input),
136149
),
137150
);
138151
const client = {
@@ -150,6 +163,7 @@ const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (o
150163
),
151164
Effect.andThen(Ref.set(lastAcceptBoundedSnapshot, input.acceptBoundedSnapshot)),
152165
Effect.andThen(Ref.set(lastAcceptCompactTurnItems, input.acceptCompactTurnItems)),
166+
Effect.andThen(Queue.offer(subscriptions, input.afterSequence)),
153167
Effect.as(streamFrom(inputs)),
154168
),
155169
),
@@ -261,6 +275,7 @@ const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (o
261275
latest,
262276
retryCount,
263277
subscriptionCount,
278+
subscriptions,
264279
loaderCalls,
265280
lastSubscribeAfterSequence,
266281
lastRequestCompletionMarker,
@@ -1251,6 +1266,54 @@ describe("EnvironmentThreads", () => {
12511266
}),
12521267
);
12531268

1269+
it.effect("reloads the full snapshot after a resumed replay hits a defect", () =>
1270+
Effect.gen(function* () {
1271+
const harness = yield* makeHarness({ cached: BASE_PROJECTION, completionMarker: true });
1272+
expect(yield* Queue.take(harness.subscriptions)).toBe(CACHED_SNAPSHOT_SEQUENCE);
1273+
1274+
yield* Queue.offer(harness.inputs, new TestStreamDefect("Replayed event did not decode"));
1275+
expect(yield* Queue.take(harness.subscriptions)).toBeUndefined();
1276+
1277+
yield* Queue.offerAll(harness.inputs, [
1278+
snapshot(
1279+
{ ...BASE_PROJECTION, thread: { ...BASE_PROJECTION.thread, title: "Reloaded thread" } },
1280+
CACHED_SNAPSHOT_SEQUENCE + 5,
1281+
),
1282+
synchronized(),
1283+
]);
1284+
// The failed resume is retried silently: no state on the way to live
1285+
// reports an error.
1286+
let live = yield* Queue.take(harness.observed);
1287+
while (live.status !== "live") {
1288+
expect(Option.isNone(live.error)).toBe(true);
1289+
live = yield* Queue.take(harness.observed);
1290+
}
1291+
expect(Option.getOrThrow(live.data).thread.title).toBe("Reloaded thread");
1292+
expect(Option.isNone(live.error)).toBe(true);
1293+
1294+
yield* harness.replaceSession;
1295+
expect(yield* Queue.take(harness.subscriptions)).toBe(CACHED_SNAPSHOT_SEQUENCE + 5);
1296+
}),
1297+
);
1298+
1299+
it.effect("stops when the snapshot reload after a failed resume also hits a defect", () =>
1300+
Effect.gen(function* () {
1301+
const harness = yield* makeHarness({ cached: BASE_PROJECTION, completionMarker: true });
1302+
expect(yield* Queue.take(harness.subscriptions)).toBe(CACHED_SNAPSHOT_SEQUENCE);
1303+
yield* Queue.offer(harness.inputs, new TestStreamDefect("Replayed event did not decode"));
1304+
expect(yield* Queue.take(harness.subscriptions)).toBeUndefined();
1305+
1306+
// A failed resume never publishes an error, so this one is the snapshot attempt's.
1307+
yield* Queue.offer(harness.inputs, new TestStreamDefect("Snapshot did not decode"));
1308+
const failed = yield* awaitThreadState(harness.observed, (value) =>
1309+
Option.isSome(value.error),
1310+
);
1311+
expect(failed.status).not.toBe("live");
1312+
expect(Option.getOrThrow(failed.error)).toBe("Could not synchronize the thread.");
1313+
expect(yield* Queue.size(harness.subscriptions)).toBe(0);
1314+
}),
1315+
);
1316+
12541317
it.effect("keeps replayed updates synchronizing until the completion marker arrives", () =>
12551318
Effect.gen(function* () {
12561319
const harness = yield* makeHarness({ cached: BASE_PROJECTION, completionMarker: true });

‎packages/client-runtime/src/state/threads.ts‎

Lines changed: 49 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,11 @@ import * as ConnectionWakeups from "../connection/wakeups.ts";
2929
import * as Persistence from "../platform/persistence.ts";
3030
import { runCachePersistence } from "./cachePersistence.ts";
3131
import * as ManagedRelay from "../relay/managedRelay.ts";
32-
import { subscribeDynamic } from "../rpc/client.ts";
32+
import {
33+
subscribeDynamic,
34+
type EnvironmentRpcStreamFailure,
35+
type EnvironmentRpcStreamValue,
36+
} from "../rpc/client.ts";
3337
import { parseThreadKey, threadKey } from "./entities.ts";
3438
import { applyOrchestrationV2ProjectionEvent } from "./orchestrationV2Projection.ts";
3539
import { THREAD_SNAPSHOT_IDLE_TTL_MS } from "./threadRetention.ts";
@@ -763,11 +767,23 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
763767
);
764768
});
765769

770+
// A resume replays the events after the held cursor. When that replay dies
771+
// (an event this client cannot decode), the same cursor dies the same way on
772+
// every attempt, including after a relaunch from the saved cache. The next
773+
// attempt loads the full snapshot instead, which moves the cursor past it.
774+
const attempt = yield* Ref.make<"snapshot" | "resume">("snapshot");
775+
const forceSnapshot = yield* Ref.make(false);
776+
766777
yield* markSynchronizing;
767-
yield* Effect.forkScoped(
778+
const subscribeThread = (): Stream.Stream<
779+
EnvironmentRpcStreamValue<typeof ORCHESTRATION_V2_WS_METHODS.subscribeThread>,
780+
EnvironmentRpcStreamFailure<typeof ORCHESTRATION_V2_WS_METHODS.subscribeThread>,
781+
EnvironmentSupervisor.EnvironmentSupervisor
782+
> =>
768783
subscribeDynamic(
769784
ORCHESTRATION_V2_WS_METHODS.subscribeThread,
770785
Effect.fn("EnvironmentThreadState.makeSubscribeInput")(function* (session) {
786+
yield* Ref.set(attempt, "snapshot");
771787
let current = yield* SubscriptionRef.get(state);
772788
// A prior definitive miss (or delete event) already cleared this thread.
773789
// Park the subscription attempt without opening the socket so we do not
@@ -841,7 +857,9 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
841857
}
842858

843859
const sequence = yield* SubscriptionRef.get(lastSequence);
844-
const canResume = Option.isSome(current.data);
860+
const canResume =
861+
Option.isSome(current.data) && !(yield* Ref.getAndSet(forceSnapshot, false));
862+
if (canResume) yield* Ref.set(attempt, "resume");
845863
const acceptBoundedSnapshot = yield* Ref.get(acceptsBoundedSocketSnapshots);
846864
if (!supportsCompletionMarker && canResume) {
847865
yield* SubscriptionRef.update(state, (value) => ({
@@ -861,13 +879,38 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
861879
};
862880
}),
863881
{
864-
onDefect: () => setStreamError("Could not synchronize the thread."),
882+
// A die fails the stream, so the catch below reports or retries it. A
883+
// transport defect waits for the next session instead: report it now,
884+
// and make that session load the snapshot when it broke a resume.
885+
onDefect: (cause) =>
886+
Cause.hasDies(cause)
887+
? Effect.void
888+
: Effect.flatMap(Ref.get(attempt), (kind) =>
889+
kind === "resume" ? Ref.set(forceSnapshot, true) : Effect.void,
890+
).pipe(Effect.andThen(setStreamError("Could not synchronize the thread."))),
865891
onExpectedFailure: (cause) => setStreamError(formatThreadError(cause)),
866892
retryExpectedFailureAfter: "250 millis",
867893
resubscribe: foregroundResubscriptions,
868894
},
869-
).pipe(Stream.runForEachArray(applyItems)),
870-
);
895+
).pipe(
896+
// Only a failed resume retries, and it retries as a snapshot. A defect on
897+
// a snapshot attempt is reported here and stays terminal, so this cannot
898+
// loop.
899+
Stream.catchCause((cause) =>
900+
Stream.unwrap(
901+
Effect.gen(function* () {
902+
if (Cause.hasInterruptsOnly(cause)) return Stream.failCause(cause);
903+
if ((yield* Ref.get(attempt)) === "resume") {
904+
yield* Ref.set(forceSnapshot, true);
905+
return subscribeThread();
906+
}
907+
yield* setStreamError("Could not synchronize the thread.");
908+
return Stream.failCause(cause);
909+
}),
910+
),
911+
),
912+
);
913+
yield* Effect.forkScoped(subscribeThread().pipe(Stream.runForEachArray(applyItems)));
871914

872915
return state;
873916
});

0 commit comments

Comments
 (0)