Skip to content

Commit de21fbc

Browse files
fix(client): a thread whose subscription dies on a defect resubscribes instead of staying blank (pingdotgg#25919)
1 parent 0ad9c9a commit de21fbc

3 files changed

Lines changed: 141 additions & 27 deletions

File tree

‎packages/client-runtime/src/rpc/client.ts‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,11 @@ export const isAuthorizationFailure = (cause: Cause.Cause<unknown>) =>
114114

115115
const MAX_EXPECTED_FAILURE_RETRY_DELAY_MS = 30_000;
116116

117+
export const subscriptionRetryDelay = (base: Duration.Input, retries: number) =>
118+
Duration.millis(
119+
Math.min(Duration.toMillis(base) * 2 ** retries, MAX_EXPECTED_FAILURE_RETRY_DELAY_MS),
120+
);
121+
117122
export type EnvironmentRpcInput<TTag extends EnvironmentRpcTag> = Parameters<RpcMethod<TTag>>[0];
118123

119124
export type EnvironmentRpcSuccess<TTag extends EnvironmentUnaryRpcTag> =
@@ -378,12 +383,9 @@ function subscribeDynamicMapped<TTag extends EnvironmentSubscriptionRpcTag, A>(
378383
) {
379384
return handled;
380385
}
381-
const retryDelay = Duration.millis(
382-
Math.min(
383-
Duration.toMillis(options.retryExpectedFailureAfter) *
384-
2 ** expectedFailureRetries,
385-
MAX_EXPECTED_FAILURE_RETRY_DELAY_MS,
386-
),
386+
const retryDelay = subscriptionRetryDelay(
387+
options.retryExpectedFailureAfter,
388+
expectedFailureRetries,
387389
);
388390
expectedFailureRetries += 1;
389391
return handled.pipe(

‎packages/client-runtime/src/state/threads-sync.test.ts‎

Lines changed: 66 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,8 @@ const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (o
118118
readonly latestLocalTurnOrdinal?: number | null;
119119
};
120120
readonly httpSnapshot?: ThreadSnapshotLoadResult;
121+
/** Answers per load call; the last one repeats. Overrides `httpSnapshot`. */
122+
readonly httpSnapshots?: readonly ThreadSnapshotLoadResult[];
121123
/** The test sends the completion marker itself instead of the fake server sending it first. */
122124
readonly completionMarker?: boolean;
123125
readonly resumeCache?: NonNullable<Parameters<typeof makeEnvironmentThreadState>[1]>;
@@ -193,11 +195,12 @@ const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (o
193195
);
194196
const snapshotLoader = ThreadSnapshotLoader.ThreadSnapshotLoader.of({
195197
load: (_prepared, threadId) =>
196-
Ref.update(loaderCalls, (count) => count + 1).pipe(
197-
Effect.as(
198+
Ref.updateAndGet(loaderCalls, (count) => count + 1).pipe(
199+
Effect.map((count) =>
198200
threadId === THREAD_ID
199-
? (options?.httpSnapshot ??
200-
({ _tag: "unavailable" } satisfies ThreadSnapshotLoadResult))
201+
? (options?.httpSnapshots?.[Math.min(count, options.httpSnapshots.length) - 1] ??
202+
options?.httpSnapshot ??
203+
({ _tag: "unavailable" } satisfies ThreadSnapshotLoadResult))
201204
: ({ _tag: "unavailable" } satisfies ThreadSnapshotLoadResult),
202205
),
203206
),
@@ -798,6 +801,27 @@ describe("EnvironmentThreads", () => {
798801
}),
799802
);
800803

804+
it.effect("keeps asking a mounted thread over HTTP after a miss until it exists", () =>
805+
Effect.gen(function* () {
806+
const harness = yield* makeHarness({
807+
httpSnapshots: [
808+
{ _tag: "missing" },
809+
{ _tag: "missing" },
810+
{
811+
_tag: "present",
812+
snapshot: { snapshotSequence: CACHED_SNAPSHOT_SEQUENCE, projection: BASE_PROJECTION },
813+
},
814+
],
815+
});
816+
yield* awaitThreadState(harness.observed, (value) => value.status === "deleted");
817+
yield* TestClock.adjust("250 millis");
818+
yield* TestClock.adjust("500 millis");
819+
const state = yield* awaitThreadState(harness.observed, (value) => value.status === "live");
820+
expect(Option.getOrThrow(state.data)).toEqual(BASE_PROJECTION);
821+
expect(yield* Ref.get(harness.loaderCalls)).toBe(3);
822+
}),
823+
);
824+
801825
it.effect.each([
802826
// The HTTP retry decides, so a missing thread never flashes an error.
803827
{
@@ -869,7 +893,7 @@ describe("EnvironmentThreads", () => {
869893
let configReads = 0;
870894
const harness = yield* makeHarness({
871895
httpSnapshot: { _tag: "missing" },
872-
// The retry's config read dies too, so a stale flag would resubscribe forever.
896+
// The retry's config read dies too, so a stale flag would resubscribe without backing off.
873897
initialConfig: Effect.suspend(() =>
874898
++configReads === 1
875899
? Effect.succeed({ threadSnapshotPagination: true })
@@ -2382,6 +2406,43 @@ describe("EnvironmentThreads", () => {
23822406
}),
23832407
);
23842408

2409+
it.effect(
2410+
"resubscribes after a snapshot decode defect ends the thread subscription (#23041)",
2411+
() =>
2412+
Effect.gen(function* () {
2413+
const harness = yield* makeHarness({ httpSnapshot: { _tag: "unavailable" } });
2414+
yield* SubscriptionRef.changes(harness.subscriptionCount).pipe(
2415+
Stream.filter((count) => count >= 1),
2416+
Stream.runHead,
2417+
);
2418+
yield* Queue.offer(harness.inputs, TEST_DEFECT);
2419+
const failed = yield* awaitThreadState(harness.observed, (value) =>
2420+
Option.isSome(value.error),
2421+
);
2422+
expect(failed.status).toBe("empty");
2423+
expect(Option.getOrThrow(failed.error)).toBe("Could not synchronize the thread.");
2424+
2425+
yield* TestClock.adjust("60 seconds");
2426+
expect(yield* SubscriptionRef.get(harness.subscriptionCount)).toBe(2);
2427+
2428+
yield* Queue.offer(
2429+
harness.inputs,
2430+
snapshot({
2431+
...BASE_PROJECTION,
2432+
thread: { ...BASE_PROJECTION.thread, title: "Recovered thread" },
2433+
}),
2434+
);
2435+
const recovered = yield* awaitThreadState(
2436+
harness.observed,
2437+
(value) =>
2438+
value.status === "live" &&
2439+
Option.isSome(value.data) &&
2440+
value.data.value.thread.title === "Recovered thread",
2441+
);
2442+
expect(Option.isNone(recovered.error)).toBe(true);
2443+
}),
2444+
);
2445+
23852446
it.effect("keeps replayed updates synchronizing until the completion marker arrives", () =>
23862447
Effect.gen(function* () {
23872448
const harness = yield* makeHarness({ cached: BASE_PROJECTION, completionMarker: true });

‎packages/client-runtime/src/state/threads.ts‎

Lines changed: 67 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,12 @@ import * as ConnectionWakeups from "../connection/wakeups.ts";
3232
import * as Persistence from "../platform/persistence.ts";
3333
import { runCachePersistence } from "./cachePersistence.ts";
3434
import * as ManagedRelay from "../relay/managedRelay.ts";
35-
import { isAuthorizationFailure, request, subscribeDynamic } from "../rpc/client.ts";
35+
import {
36+
isAuthorizationFailure,
37+
request,
38+
subscribeDynamic,
39+
subscriptionRetryDelay,
40+
} from "../rpc/client.ts";
3641
import { parseThreadKey, threadKey } from "./entities.ts";
3742
import { applyOrchestrationV2ProjectionEvent } from "./orchestrationV2Projection.ts";
3843
import { THREAD_SNAPSHOT_IDLE_TTL_MS } from "./threadRetention.ts";
@@ -55,6 +60,8 @@ import {
5560
type EnvironmentThreadStatus,
5661
} from "./threadState.ts";
5762

63+
const THREAD_SUBSCRIPTION_RETRY_DELAY = "250 millis";
64+
5865
function statusWithoutLiveData(
5966
data: Option.Option<OrchestrationV2ThreadProjection>,
6067
): EnvironmentThreadStatus {
@@ -967,8 +974,8 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
967974
(yield* Ref.get(socketOnlyOpen)) && Option.isNone((yield* SubscriptionRef.get(state)).data)
968975
);
969976
});
970-
// A defect parks the subscription, so a socket-only open resubscribes once.
971-
const httpRetries = yield* Queue.unbounded<void>();
977+
// A stream defect parks the subscription until something resubscribes it.
978+
const resubscribes = yield* Queue.unbounded<void>();
972979
const markSynchronizing = Effect.gen(function* () {
973980
if (yield* Ref.get(resumingLive)) return;
974981
yield* SubscriptionRef.update(state, (current) =>
@@ -979,15 +986,27 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
979986
});
980987

981988
yield* markSynchronizing;
982-
yield* Effect.forkScoped(
989+
// A 404 can arrive before the thread exists, so a miss asks HTTP again while mounted.
990+
let httpMisses = 0;
991+
let defectRetries = 0;
992+
const backOffAfterDefect = Effect.suspend(() => {
993+
const retryDelay = subscriptionRetryDelay(THREAD_SUBSCRIPTION_RETRY_DELAY, defectRetries);
994+
defectRetries += 1;
995+
return setStreamError("Could not synchronize the thread.").pipe(
996+
Effect.andThen(Effect.sleep(retryDelay)),
997+
);
998+
});
999+
const followThread = (): Effect.Effect<
1000+
void,
1001+
never,
1002+
EnvironmentSupervisor.EnvironmentSupervisor
1003+
> =>
9831004
subscribeDynamic(
9841005
ORCHESTRATION_V2_WS_METHODS.subscribeThread,
9851006
Effect.fn("EnvironmentThreadState.makeSubscribeInput")(function* (session) {
9861007
let current = yield* SubscriptionRef.get(state);
987-
// A prior definitive miss (or delete event) already cleared this thread.
988-
// Park the subscription attempt without opening the socket so we do not
989-
// retry forever against a known-missing id.
990-
if (current.status === "deleted") {
1008+
// A delete event already cleared this thread. Park without opening the socket.
1009+
if (current.status === "deleted" && httpMisses === 0) {
9911010
return yield* Effect.never;
9921011
}
9931012

@@ -1024,6 +1043,7 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
10241043
yield* snapshotLoader.load(prepared, threadId);
10251044
switch (httpResult._tag) {
10261045
case "present": {
1046+
httpMisses = 0;
10271047
if (canLoadHistory && httpResult.history !== undefined) {
10281048
yield* Ref.set(acceptsBoundedSocketSnapshots, true);
10291049
}
@@ -1052,9 +1072,13 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
10521072
break;
10531073
}
10541074
case "missing": {
1055-
yield* setDeleted();
1075+
if (current.status !== "deleted") yield* setDeleted();
10561076
// A 404 also means "not created yet". Only a delete event may outlive this mount.
10571077
if (resumeCache?.owner === owner) resumeCache.snapshot = undefined;
1078+
yield* Effect.sleep(
1079+
subscriptionRetryDelay(THREAD_SUBSCRIPTION_RETRY_DELAY, httpMisses++),
1080+
);
1081+
yield* Queue.offer(resubscribes, undefined);
10581082
return yield* Effect.never;
10591083
}
10601084
case "unavailable": {
@@ -1085,22 +1109,49 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make
10851109
{
10861110
// A socket-only open that fails retries over HTTP, which tells a
10871111
// missing thread apart from a failure, before showing an error.
1088-
onDefect: Effect.fn("EnvironmentThreadState.onDefect")(function* () {
1112+
onDefect: Effect.fn("EnvironmentThreadState.onDefect")(function* (cause) {
10891113
if (yield* failedSocketOnlyOpen) {
10901114
yield* Ref.set(socketOnlyOpen, false);
1091-
return yield* Queue.offer(httpRetries, undefined);
1115+
return yield* Queue.offer(resubscribes, undefined);
1116+
}
1117+
if (!Cause.hasDies(cause)) {
1118+
return yield* setStreamError("Could not synchronize the thread.");
10921119
}
1093-
yield* setStreamError("Could not synchronize the thread.");
1120+
yield* backOffAfterDefect;
1121+
yield* Queue.offer(resubscribes, undefined);
10941122
}),
10951123
onExpectedFailure: Effect.fn("EnvironmentThreadState.onExpectedFailure")(function* (cause) {
10961124
const retryOverHttp = !isAuthorizationFailure(cause) && (yield* failedSocketOnlyOpen);
10971125
if (!retryOverHttp) yield* setStreamError(formatThreadError(cause));
10981126
}),
1099-
retryExpectedFailureAfter: "250 millis",
1100-
resubscribe: Stream.merge(foregroundResubscriptions, Stream.fromQueue(httpRetries)),
1127+
retryExpectedFailureAfter: THREAD_SUBSCRIPTION_RETRY_DELAY,
1128+
resubscribe: Stream.merge(foregroundResubscriptions, Stream.fromQueue(resubscribes)),
11011129
},
1102-
).pipe(Stream.runForEachArray(applyItems)),
1103-
);
1130+
).pipe(
1131+
Stream.runForEachArray((items) =>
1132+
applyItems(items).pipe(
1133+
Effect.andThen(
1134+
Effect.sync(() => {
1135+
defectRetries = 0;
1136+
}),
1137+
),
1138+
),
1139+
),
1140+
// A defect while applying items ends the stream; resubscribe for a fresh snapshot.
1141+
Effect.catchCause((cause) =>
1142+
Cause.hasInterrupts(cause)
1143+
? Effect.interrupt
1144+
: Effect.logWarning("Thread subscription failed; resubscribing.", {
1145+
cause: Cause.pretty(cause),
1146+
environmentId,
1147+
threadId,
1148+
}).pipe(
1149+
Effect.andThen(backOffAfterDefect),
1150+
Effect.andThen(Effect.suspend(followThread)),
1151+
),
1152+
),
1153+
);
1154+
yield* Effect.forkScoped(followThread());
11041155

11051156
return state;
11061157
});

0 commit comments

Comments
 (0)